added brave image+video support

about.php

@@ -81,7 +81,7 @@ $left =
 	<div class="code-inline">Escape</div> to exit the image viewer.
 	
 	<a href="#instances"><h2 id="instances">Instances</h2></a>
-	4get is open source, anyone can create their own 4get instance! If you wish to add your website to this list, please <a href="https://lolcat.ca/contact">contact me</a>.
+	4get is open source, anyone can create their own 4get instance! If you wish to add your website to this list, please <a href="https://lolcat.ca">contact me</a>.
 	 
 	<table>
 		<tr>
@@ -102,7 +102,7 @@ $left =
 	
 	<b>Message to all DMCA enforcers:</b> I don\'t host any of the content. Everything you see here is <u>proxied</u> trough my shitbox with no moderation. Please reach out to the people hosting the infringing content instead.<br><br>
 	
-	<a href="https://lolcat.ca/contact" rel="dofollow" class="link">Click here to contact me!</a><br><br>
+	<a href="https://lolcat.ca" rel="dofollow" class="link">Click here to contact me!</a><br><br>
 	
 	<a href="https://validator.w3.org/nu/?doc=https%3A%2F%2F4get.ca" title="W3 Valid!">
 		<img src="/static/icon/w3html.png" alt="Valid W3C HTML 4.01" width="88" height="31">

api/v1/images.php

@@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
 
 try{
 	echo json_encode(
-		$scraper->image($get)
+		$scraper->image($get),
+		JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
 	);
 	
 }catch(Exception $e){

api/v1/news.php

@@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
 
 try{
 	echo json_encode(
-		$scraper->news($get)
+		$scraper->news($get),
+		JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
 	);
 	
 }catch(Exception $e){

api/v1/videos.php

@@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
 
 try{
 	echo json_encode(
-		$scraper->video($get)
+		$scraper->video($get),
+		JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
 	);
 	
 }catch(Exception $e){

api/v1/web.php

@@ -21,7 +21,8 @@ if(!isset($_GET["extendedsearch"])){
 
 try{
 	echo json_encode(
-		$scraper->web($get)
+		$scraper->web($get),
+		JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
 	);
 	
 }catch(Exception $e){

images.php

@@ -62,8 +62,13 @@ foreach($results["image"] as $image){
 		'<div class="image-wrapper" title="' . htmlspecialchars($image["title"]) .'" data-json="' . htmlspecialchars(json_encode($image["source"])) . '">' .
 			'<div class="image">' .
 				'<a href="' . htmlspecialchars($image["source"][0]["url"]) . '" rel="noreferrer nofollow" class="thumb">' .
-					'<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "thumb") . '" alt="thumbnail">' .
+					'<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "thumb") . '" alt="thumbnail">';
-					'<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>' .
+				
+				if($image["source"][0]["width"] !== null){
+					$payload["images"] .= '<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>';
+				}
+				
+			$payload["images"] .=
 				'</a>' .
 				'<a href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow">' .
 					'<div class="title">' . htmlspecialchars(parse_url($image["url"], PHP_URL_HOST)) . '</div>' .

lib/frontend.php

@@ -892,6 +892,7 @@ class frontend{
 					"option" => [
 						"ddg" => "DuckDuckGo",
 						"yandex" => "Yandex",
+						"brave" => "Brave"//,
 						//"google" => "Google"
 					]
 				];
@@ -903,6 +904,7 @@ class frontend{
 					"option" => [
 						"yt" => "YouTube",
 						"ddg" => "DuckDuckGo",
+						"brave" => "Brave"//,
 						//"google" => "Google"
 					]
 				];
@@ -1285,7 +1287,7 @@ class frontend{
 			return htmlspecialchars($image);
 		}
 		
-		return "/proxy?i=" . urlencode($image) . "&s=" . $format;
+		return "/proxy.php?i=" . urlencode($image) . "&s=" . $format;
 	}
 	
 	public function htmlnextpage($gets, $npt, $page){

lib/fuckhtml.php

@@ -356,6 +356,91 @@ class fuckhtml{
 		
 		return $out;
 	}
+	
+	public function parseJsObject(string $json){
+		
+		$bracket = false;
+		$is_close_bracket = false;
+		$escape = false;
+		$json_out = null;
+		$last_char = null;
+		
+		for($i=0; $i<strlen($json); $i++){
+			
+			switch($json[$i]){
+				
+				case "\"":
+				case "'":
+					if($escape === true){
+						
+						break;
+					}
+					
+					if($json[$i] == $bracket){
+						
+						$bracket = false;
+						$is_close_bracket = true;
+					}else{
+						
+						if($bracket === false){
+							
+							$bracket = $json[$i];
+						}
+					}
+					break;
+				
+				default:
+					$is_close_bracket = false;
+					break;
+			}
+			
+			$escape = $json[$i] == "\\" ? true : false;
+			
+			if(
+				$bracket === false &&
+				$is_close_bracket === false
+			){
+				
+				// here we know we're not iterating over a quoted string
+				switch($json[$i]){
+					
+					case "[":
+					case "{":
+						// dont execute whats in "default"
+						$json_out .= $json[$i];
+						break;
+					
+					case "]":
+					case "}":
+					case ",":
+					case ":":
+						if(!in_array($last_char, ["[", "{", "}", "]", "\""])){
+							
+							$json_out .= "\"";
+						}
+						
+						$json_out .= $json[$i];
+						break;
+					
+					default:
+						if(in_array($last_char, ["{", "[", ",", ":"])){
+							
+							$json_out .= "\"";
+						}
+						
+						$json_out .= $json[$i];
+						break;
+				}
+			}else{
+				
+				$json_out .= $json[$i];
+			}
+			
+			$last_char = $json[$i];
+		}
+		
+		return json_decode($json_out, true);
+	}
 }
 
 ?>

proxy.php

@@ -57,6 +57,7 @@ try{
 		}
 		
 		$image->readImageBlob($payload["body"]);
+		
 		$image_width = $image->getImageWidth();
 		$image_height = $image->getImageHeight();
 		
@@ -102,16 +103,16 @@ try{
 			$image_width = $image_height * $ratio;
 		}
 		
-		$image->setImageBackgroundColor(new ImagickPixel("#504945"));
+		$image->setImageBackgroundColor("#504945");
-		$image->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN);
+		$image->setImageAlphaChannel(Imagick::ALPHACHANNEL_REMOVE);
-		
-		$image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1);
 		
 		$image->stripImage();
 		$image->setFormat("jpeg");
 		$image->setImageCompressionQuality(90);
 		$image->setImageCompression(Imagick::COMPRESSION_JPEG2000);
 		
+		$image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1);
+		
 		$proxy->getfilenameheader($payload["headers"], $_GET["i"]);
 		
 		header("Content-Type: image/jpeg");

scraper/brave.php

@@ -86,6 +86,8 @@ class brave{
 				];
 				break;
 			
+			case "images":
+			case "videos":
 			case "news":
 				return [
 					"country" => [
@@ -143,7 +145,7 @@ class brave{
 		}
 	}
 	
-	private function get($url, $get = [], $nsfw, $country/*, $is_post = false, $additional_cookies = null*/){
+	private function get($url, $get = [], $nsfw, $country){
 		
 		switch($nsfw){
 			
@@ -152,13 +154,6 @@ class brave{
 			case "no": $nsfw = "strict"; break;
 		}
 		
-		//$cookie = "safesearch={$nsfw}; country={$country}; useLocation=0";
-		/*
-		if($additional_cookies !== null){
-			
-			$cookie = $additional_cookies . "; " . $cookie;
-		}*/
-		
 		$headers = [
 			"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0",
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@@ -171,8 +166,7 @@ class brave{
 			"Sec-Fetch-Dest: document",
 			"Sec-Fetch-Mode: navigate",
 			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"//,
+			"Sec-Fetch-User: ?1"
-			//"Content-Type: application/json"
 		];
 		
 		if($country == "any"){
@@ -182,22 +176,10 @@ class brave{
 		
 		$curlproc = curl_init();
 		
-		/*if($is_post){
-			
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt(
-				$curlproc,
-				CURLOPT_POSTFIELDS,
-				json_encode($get)
-			);
-			
-		}else{
-			*/
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
 		}
-		//}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
@@ -1950,18 +1932,24 @@ class brave{
 		return $out;
 	}
 	
-	/*
+	public function image($get){
-	public function bypasscaptcha($html, $nsfw, $country){
 		
-		// @TODO figure out why I still cant go trough
+		$search = $get["s"];
-		// the captcha wall even after breaking it
+		$country = $get["country"];
+		$nsfw = $get["nsfw"];
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
 		
 		try{
 			$html =
 				$this->get(
-					"https://search.brave.com/goggles",
+					"https://search.brave.com/images",
 					[
-						"q" => "site:dailymotion.com my bloody valentine"
+						"q" => $search
 					],
 					$nsfw,
 					$country
@@ -1969,177 +1957,192 @@ class brave{
 			
 		}catch(Exception $error){
 			
-			throw new Exception("Could not fetch html");
+			throw new Exception("Could not fetch search page");
+		}
+		/*
+		$handle = fopen("scraper/brave-image.html", "r");
+		$html = fread($handle, filesize("scraper/brave-image.html"));
+		fclose($handle);*/
+		
+		preg_match(
+			'/const data = (\[{.*}\]);/',
+			$html,
+			$json
+		);
+		
+		if(!isset($json[1])){
+			
+			throw new Exception("Failed to get data object");
 		}
 		
-		// Bypass brave search captcha
+		$json =
-		// this captcha only appears on the goggles page
+			$this->fuckhtml
-		preg_match(
+			->parseJsObject(
-			'/this\.img\.src = "(.*)"/',
+				$json[1]
-			$html,
-			$image
 			);
 		
-		$image =
-			base64_decode(
-				explode(
-					"data:image/png;base64,",
-					$image[1]
-				)[1]
-			);
-		
-		$im = new Imagick();
-		$im->readImageBlob($image);
-		
-		$im->blurImage(20, 20);
-		$im->posterizeImage(2, imagick::IMGTYPE_COLORSEPARATION);
-		
-		// if we encounter a white line thats longer than 45px
-		// we found the circle position
-		$iterator = $im->getPixelRegionIterator(0, 77, 310, 1);
-		
-		$found = null;
 		foreach(
-			$iterator as $row
+			$json[1]
+			["data"]
+			["body"]
+			["response"]
+			["results"]
+			as $result
 		){
 			
-			$whitecount = 0;
+			$out["image"][] = [
-			$count = 0;
+				"title" => $result["title"],
-			
+				"source" => [
-			foreach($row as $pixel){
-				
-				if($pixel->getColor()["r"] === 255){
-					
-					$whitecount++;
-					$pixel->setColor("rgba(255,0,0,0)");
-					
-					if($whitecount === 45){
-						
-						$found = $count - 45;
-						break 2;
-					}
-				}else{
-					
-					$whitecount = 0;
-				}
-				
-				$count++;
-				$iterator->syncIterator();
-			}
-		}
-		
-		$found = $found + 10;
-		
-		//header("Content-Type: image/png");
-		//echo $im;
-		//die();
-		
-		if($found === null){
-			
-			throw new Exception("Could not bypass captcha");
-		}
-		
-		preg_match(
-			'/data="{"captcha_id":"([0-9A-z-]+)"}"/',
-			$html,
-			$key
-		);
-		
-		$key = $key[1];
-		// we bypassed captcha, send POST data
-		$order =
-			$this->get(
-				"https://search.brave.com/api/captcha?brave=0&captcha_id={$key}",
 					[
-					"solution" => (string)$found
+						"url" => $result["properties"]["url"],
+						"width" => null,
+						"height" => null
 					],
-				$nsfw,
-				$country,
-				true
-			);
-		
-		$order = json_decode($order, true)["orderId"];
-		
-		$orderpayload =
-			$this->get(
-				"https://search.brave.com/api/rewards/v1/orders/{$order}",
-				[],
-				$nsfw,
-				$country
-			);
-		
-		$orderpayload = json_decode($orderpayload, true);
-		
-		$creds =
-			$this->get(
-				"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
 					[
-					"itemId" => $orderpayload["items"][0]["id"],
+						"url" => $result["thumbnail"]["src"],
-					"blindedCreds" => [
+						"width" => null,
-						"fuYAVcB/m7BU66vf3wkNGxJCSaRhshB9o+8km3F1h2c=",
+						"height" => null
-						"uswvcWJuPK/1qFlVdzBP3eQd0+V1EQgfAtnEoMIK+Uk=",
-						"fJWKGLBxl3Gyn4n9FjTLq1PjupfABT7Ni8MeB+iGzUs=",
-						"Aq9enJ/VZP9GxQIza3n65ZK7xQhY4VwDxv53BCb/Txg=",
-						"FMJA9eSLHq71K+Pcwgm4gIQOmdR/6KMy5cMgXhpd5Ro=",
-						"2NVhIAbvI317SP9/xXbVe/U57eWgvHyqVbHL/5+Gdmw=",
-						"6mpjsjSCmYEzK2xlbL8DI2P4LuhWUOxjTLvsTAL9l24=",
-						"kAn4wuHvIlKWhfuFfPTSfD4tZ5le9t7/61YbdEc/L3k=",
-						"BjjUyG16aTfd1c0h4oBzgQQOekrH1f+a5CmcXqMPTR4=",
-						"SBNgpCt4/V44yaQTfh+D027Yv1GJFHkjUEpPw6rAwRI=",
-						"XDENAtdQ7PyYx+Qx1wQGQtDWgg8WpIMgWGmd4RDOVWE=",
-						"tF7rB4sqamsiUk3K7fojdQSI0Q6iip72yKyhnvg/bC0=",
-						"VsAqflirAd/u4VsLdfRS2UvnH24ZNkFh6YN3DctLjzQ=",
-						"MntLbXkoI0LdcisCbNazmooiHXJyX91L1KERDAu1JRU=",
-						"TH6Zs8JBvFDbTDWgKbfGE4M5/cSwCtHD8ms5Y/U8zHQ=",
-						"jsZg0Z+qDPHymrbhdnesodhLNJ26QdunyMko1aVe4So=",
-						"rpKsyj6/vdnuMgLI2BApeijtGq9g5USRDL0w6X2bnlQ=",
-						"vCzliGT8A9vcLXj2sFf2kavOuYw69d70NpfgA22B4lI=",
-						"7OWoxSCtYXWcaBSifF7AXNBif/sjcuO0IelzXG/3PFk=",
-						"iiXtByNlT6nDMN9De5B58Jl8J0p6LCjnZ9aS3w2FEQU=",
-						"zDhd7gsJ4h4JkDeGK0Y0mfFd8IBdkLhMOANzwO+4Dig=",
-						"qANZ+AikwFReEA61JF009d/c3IHM/aSfIYwljckhJWE=",
-						"nNC30pDLxtXvUr+WDwfDSrAInNBpfSZkPsV2JlpheWI=",
-						"kGXE1pkt25P71kdJzmKIg4+yMR1VA5wNmbpBb/FhJQ8=",
-						"aLqPsY1Qiz2UCa2Jx3YNNt8r4JINMphks/43EiyZfXU=",
-						"bHGYZoQARZEM5LdFF6B74PkRqNd9EKxzuTvGYxjq+hk=",
-						"JOsYQjfE/9Y1u29hR+GvEkNyxUI8blgLhX1iJI/aGRQ=",
-						"yKjHjH5j600TJD/3WPsA1N3OmItDLifdjlysq4H6NV0=",
-						"9lTnUbsPp7BJ7XVN5/T4yGfzD9DJdqWB7xk72s19MAA=",
-						"5KHG8iY45em7zDhO/HlI0ydcZ0Ubn+XSyjifMmy7qXM="
 					]
 				],
-				$nsfw,
+				"url" => $result["url"]
-				$country,
+			];
-				true
+		}
-			);
 		
-		var_dump($creds);
+		return $out;
+	}
 	
-		sleep(2);
+	public function video($get){
-		$test =
+		
+		$search = $get["s"];
+		$country = $get["country"];
+		$nsfw = $get["nsfw"];
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		try{
+			$html =
 				$this->get(
-				"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
+					"https://search.brave.com/videos",
-				[],
+					[
+						"q" => $search
+					],
 					$nsfw,
 					$country
 				);
 			
-		var_dump($test);
+		}catch(Exception $error){
 			
-		$html =
+			throw new Exception("Could not fetch search page");
-			$this->get(
+		}
-				"https://search.brave.com/goggles",
+		/*
-				[
+		$handle = fopen("scraper/brave-video.html", "r");
-					"q" => "site:dailymotion.com my bloody valentine"
+		$html = fread($handle, filesize("scraper/brave-video.html"));
-				],
+		fclose($handle);*/
-				$nsfw,
+		
-				$country,
+		preg_match(
-				false,
+			'/const data = (\[{.*}\]);/',
-				"__Secure-sku#brave-search-captcha=eyJ0eXBlIjoic2luZ2xlLXVzZSIsInZlcnNpb24iOjEsInNrdSI6ImJyYXZlLXNlYXJjaC1jYXB0Y2hhIiwicHJlc2VudGF0aW9uIjoiZXlKcGMzTjFaWElpT2lKaWNtRjJaUzVqYjIwL2MydDFQV0p5WVhabExYTmxZWEpqYUMxallYQjBZMmhoSWl3aWMybG5ibUYwZFhKbElqb2lNRzl0VDBneWQxZ3dTazkzU0VFMVJ6QTJaR1V5WjFOQ1dDdGhSM3B2Y2xsTVQwVTJZVVJtTUc5a1IweG1Wa3RhZEd0cU4xbHdia3BPT0VOVGNGbE5lVWR2YmpGRlNTOUhhMlZYU1RWNGQxTjJPWGxJTTNjOVBTSXNJblFpT2lKWlJWWldaVzR5TTJwQ01tSnZkakJ2U1hGNGJtSndUMGxEUW5Kd1drRjBRbWQxVnpoRlNURTNVREY2UVRaQlpUTXJSVGRFYm5NeVFqUmhka0pGYTFWM2FGY3JWRVZJVjNWcE9TdFllRU1yYlVSTVkyMTBRVDA5SW4wPSJ9"
+			$html,
+			$json
 		);
 		
-		var_dump($html);
+		if(!isset($json[1])){
-	}*/
+			
+			throw new Exception("Failed to get data object");
+		}
+		
+		$json =
+			$this->fuckhtml
+			->parseJsObject(
+				$json[1]
+			);
+		
+		foreach(
+			$json
+			[1]
+			["data"]
+			["body"]
+			["response"]
+			["results"]
+			as $result
+		){
+			
+			if($result["video"]["author"] != "null"){
+				
+				$author = [
+					"name" => $result["video"]["author"]["name"] == "null" ? null : $result["video"]["author"]["name"],
+					"url" => $result["video"]["author"]["url"] == "null" ? null : $result["video"]["author"]["url"],
+					"avatar" => $result["video"]["author"]["img"] == "null" ? null : $result["video"]["author"]["img"]
+				];
+			}else{
+				
+				$author = [
+					"name" => null,
+					"url" => null,
+					"avatar" => null
+				];
+			}
+			
+			if($result["thumbnail"] != "null"){
+				
+				$thumb = [
+					"url" => $result["thumbnail"]["original"],
+					"ratio" => "16:9"
+				];
+			}else{
+				
+				$thumb = [
+					"url" => null,
+					"ratio" => null
+				];
+			}
+			
+			$out["video"][] = [
+				"title" => $result["title"],
+				"description" => $result["description"] == "null" ? null : $this->titledots($result["description"]),
+				"author" => $author,
+				"date" => $result["age"] == "null" ? null : strtotime($result["age"]),
+				"duration" => $result["video"]["duration"] == "null" ? null : $this->hms2int($result["video"]["duration"]),
+				"views" => $result["video"]["views"] == "null" ? null : (int)$result["video"]["views"],
+				"thumb" => $thumb,
+				"url" => $result["url"]
+			];
+		}
+
+		return $out;
+	}
+	
+	private function hms2int($time){
+		
+		$parts = explode(":", $time, 3);
+		$time = 0;
+		
+		if(count($parts) === 3){
+			
+			// hours
+			$time = $time + ((int)$parts[0] * 3600);
+			array_shift($parts);
+		}
+		
+		if(count($parts) === 2){
+			
+			// minutes
+			$time = $time + ((int)$parts[0] * 60);
+			array_shift($parts);
+		}
+		
+		// seconds
+		$time = $time + (int)$parts[0];
+		
+		return $time;
+	}
 	
 	private function appendtext($payload, &$text, &$index){
 		

scraper/google.php

@@ -808,6 +808,7 @@ class google{
 			->getElementsByTagName("style");
 		
 		$this->computedstyle = [];
+		$this->ask = [];
 		
 		foreach($styles as $style){
 			
@@ -860,6 +861,22 @@ class google{
 						$image_grep[1][0]
 					);
 			}
+			
+			// even more javascript crap
+			// "People also ask" node is loaded trough javascript
+			preg_match_all(
+				'/window\.jsl\.dh\(\'([^\']+)\',\'(.+)\'\);/',
+				$script["innerHTML"],
+				$ask_grep
+			);
+			
+			for($i=0; $i<count($ask_grep[0]); $i++){
+				
+				$this->ask[trim($ask_grep[1][$i])] =
+					stripcslashes(
+						$ask_grep[2][$i]
+					);
+			}
 		}
 		
 		// get nodes
@@ -926,6 +943,8 @@ class google{
 					"div"
 				);
 			
+			if(count($carousel) !== 0){
+				
 				$carousel_title =
 					$this->fuckhtml
 					->getElementsByClassName(
@@ -940,8 +959,6 @@ class google{
 						"div"
 					);
 				
-			if(count($carousel) !== 0){
-				
 				$sublink = []; // twitter carousel sublinks
 				foreach($carousel as $item){
 					
@@ -1212,6 +1229,136 @@ class google{
 				continue;
 			}
 			
+			$people_title =
+				$this->fuckhtml
+				->getElementsByClassName(
+					$this->findstyles(
+						[
+							"font-weight" => "bold",
+							"font-size" => "16px",
+							"color" => "#000",
+							"margin" => "0",
+							"padding" => "12px 16px 0 16px"
+						],
+						self::is_class
+					),
+					"div"
+				);
+			
+			if(
+				count($people_title) !== 0 &&
+				strtolower(
+					$this->fuckhtml
+					->getTextContent(
+						$people_title[0]
+					)
+				) == "people also ask"
+			){
+				/*
+					Parse "people also ask" node
+				*/
+				
+				$div =
+					$this->fuckhtml
+					->getElementsByTagName("div");
+				
+				// add suggestions
+				$suggestions =
+					$this->fuckhtml
+					->getElementsByClassName(
+						$this->findstyles(
+							[
+								"display" => "inline-block",
+								"padding-right" => "26px"
+							],
+							self::is_class
+						),
+						$div
+					);
+				
+				foreach($suggestions as $suggestion){
+					
+					$out["related"][] =
+						$this->fuckhtml
+						->getTextContent($suggestion);
+				}
+				
+				// parse websites
+				foreach($div as $d){
+					
+					if(
+						isset($d["attributes"]["id"]) &&
+						strpos(
+							$d["attributes"]["id"],
+							"accdef_"
+						) !== false
+					){
+						
+						$this->fuckhtml->load(
+							$this->ask[
+								$d["attributes"]["id"]
+							]
+						);
+						
+						$description =
+							$this->titledots(
+								$this->fuckhtml
+								->getTextContent(
+									$this->fuckhtml
+									->getElementsByClassName(
+										$this->findstyles(
+											[
+												"white-space" => "pre-line",
+												"word-wrap" => "break-word"
+											],
+											self::is_class
+										),
+										"div"
+									)[0]
+								)
+							);
+						
+						$a =
+							$this->fuckhtml
+							->getElementsByTagName("a")
+							[0];
+						
+						$this->fuckhtml->load($a);
+						
+						$out["web"][] = [
+							"title" =>
+								$this->titledots(
+									$this->fuckhtml
+									->getTextContent(
+										$this->fuckhtml
+										->getElementsByTagName("span")[0]
+									)
+								),
+							"description" => $description,
+							"url" =>
+								$this->decodeurl(
+									$this->fuckhtml
+									->getTextContent(
+										$a
+										["attributes"]
+										["href"]
+									)
+								),
+							"date" => null,
+							"type" => "web",
+							"thumb" => [
+								"url" => null,
+								"ratio" => null
+							],
+							"sublink" => [],
+							"table" => []
+						];
+					}
+				}
+				
+				continue;
+			}
+			
 			if(count($title) !== 0){
 				
 				/*
@@ -1231,6 +1378,19 @@ class google{
 						"url" => $this->getimage($thumb[0]["attributes"]["id"]),
 						"ratio" => "1:1"
 					];
+					
+					if(parse_url($thumb["url"], PHP_URL_HOST) == "i.ytimg.com"){
+						
+						$thumb = [
+							"url" =>
+								str_replace(
+									"default.jpg",
+									"maxresdefault.jpg",
+									$thumb["url"]
+								),
+							"ratio" => "16:9"
+						];
+					}
 				}else{
 					
 					$thumb = [
@@ -1287,18 +1447,33 @@ class google{
 							
 							$cat = explode(":", $cat, 2);
 							
-							$table[
+							$name =
 								$this->fuckhtml
 								->getTextContent(
 									$cat[0]
-								)
+								);
-							] =
+							
+							if(strtolower($name) != "posted"){
+								
+								$table[$name] =
 									$this->titledots(
 										$this->fuckhtml
 										->getTextContent(
 											$cat[1]
 										)
 									);
+							}else{
+								
+								$date =
+									strtotime(
+										$this->titledots(
+											$this->fuckhtml
+											->getTextContent(
+												$cat[1]
+											)
+										)
+									);
+							}
 						}
 						continue;
 					}
@@ -1307,6 +1482,7 @@ class google{
 						$this->fuckhtml
 						->getElementsByTagName("span");
 					
+					$encounter_rating = false;
 					foreach($spans as $span){
 						
 						// replace element with nothing
@@ -1319,10 +1495,53 @@ class google{
 								);
 						}
 						
+						if($encounter_rating !== false){
+							
+							switch($encounter_rating){
+								
+								case 3:
+									$table["Votes"] =
+										number_format(
+											str_replace(
+												[
+													"(",
+													")",
+													","
+												],
+												"",
+												$this->fuckhtml
+												->getTextContent(
+													$span["innerHTML"]
+												)
+											)
+										);
+									break;
+								
+								case 6:
+									$table["Price"] =
+										$this->fuckhtml
+										->getTextContent(
+											$span["innerHTML"]
+										);
+									break;
+								
+								case 8:
+									$table["Support"] =
+										$this->fuckhtml
+										->getTextContent(
+											$span["innerHTML"]
+										);
+									break;
+							}
+							
+							$encounter_rating++;
+						}
+						
 						// get rating
 						if(isset($span["attributes"]["aria-hidden"])){
 							
 							$table["Rating"] = $span["innerHTML"];
+							$encounter_rating = 0;
 							continue;
 						}
 					}
@@ -1565,16 +1784,7 @@ class google{
 			}
 			
 			/*
-				Detect if its a wikipedia thing
+				Parse instant answers with parts
-			*/
-			$h3 =
-				$this->fuckhtml
-				->getElementsByTagName("h3");
-			
-			
-			
-			/*
-				Fallback to parsing the word definitions
 			*/
 			$parts =
 				$this->fuckhtml
@@ -1588,14 +1798,7 @@ class google{
 					"div"
 				);
 			
-			if(count($parts) === 0){
+			if(count($parts) !== 0){
-				
-				continue;
-			}
-			
-			$head = $parts[0];
-			
-			if(count($h3) !== 0){
 			
 				$table = [
 					"title" => null,
@@ -1606,19 +1809,66 @@ class google{
 					"sublink" => []
 				];
 				
-				$h3 = $h3[0];
+				// get thumb
+				$thumb =
+					$this->fuckhtml
+					->getElementsByClassName(
+						$this->findstyles(
+							[
+								"float" => "right",
+								"padding-left" => "16px"
+							],
+							self::is_class
+						),
+						"div"
+					);
+					
+				if(count($thumb) !== 0){
+					
+					$this->fuckhtml->load($thumb[0]);
+					
+					$img =
+						$this->fuckhtml
+						->getElementsByTagName("img");
+					
+					if(count($img) !== 0){
+						
+						$table["thumb"] =
+							$this->getimage(
+								$img[0]["attributes"]["id"]
+							);
+					}
+					
+					$this->fuckhtml->load($container);
+				}
+				
+				$h =
+					$this->fuckhtml
+					->getElementsByTagName("h3");
+				
+				if(count($h) === 0){
+					
+					$h =
+						$this->fuckhtml
+						->getElementsByTagName("h2");
+				}
+				
+				if(count($h) !== 0){
+					// set title + subtext for when a word definition
+					// appears
+					$h = $h[0];
 					
 					$table["title"] =
 						$this->fuckhtml
 						->getTextContent(
-						$h3
+							$h
 						);
 					
-				$head["innerHTML"] =
+					$parts[0]["innerHTML"] =
 						str_replace(
-						$h3["outerHTML"],
+							$h["outerHTML"],
 							"",
-						$head["innerHTML"]
+							$parts[0]["innerHTML"]
 						);
 					
 					$table["description"][] =
@@ -1627,9 +1877,62 @@ class google{
 							"value" =>
 								$this->fuckhtml
 								->getTextContent(
-								$head
+									$parts[0]
 								)
 						];
+				}else{
+					
+					// parse it as a wikipedia header
+					
+				}
+				
+				// get table elements
+				$tables =
+					$this->fuckhtml
+					->getElementsByClassName(
+						$this->findstyles(
+							[
+								"display" => "table",
+								"width" => "100%",
+								"padding-right" => "16px",
+								"-webkit-box-sizing" => "border-box"
+							],
+							self::is_class
+						),
+						"div"
+					);
+				
+				foreach($tables as $tbl){
+					
+					$this->fuckhtml->load($tbl);
+					
+					$images =
+						$this->fuckhtml
+						->getElementsByTagName("img");
+					
+					if(count($images) !== 0){
+						
+						$image = $this->getimage($images[0]["attributes"]["id"]);
+						
+						$text =
+							$this->fuckhtml
+							->getTextContent(
+								$tbl
+							);
+						
+						$table["description"][] = [
+							"type" => "link",
+							"value" => $text,
+							"url" => "?s=" . urlencode($text) . "&scraper=google"
+						];
+						
+						$table["description"][] = [
+							"type" => "image",
+							"url" => $image
+						];
+					}
+					
+				}
 				
 				$audio =
 					$this->fuckhtml
@@ -1828,10 +2131,10 @@ class google{
 						}
 					}
 				}
-			}
 				
 				$out["answer"][] = $table;
 			}
+		}
 		
 		if($dmca_table){
 			
@@ -2136,22 +2439,67 @@ class google{
 			$match
 		);
 		
-		if(count($match) !== 0){
+		if(count($match) === 0){
-			
-			if(!empty($match[1])){
-				
-				return urldecode($match[1]);
-			}
-			
-			if(!empty($match[2])){
-					
-				return urldecode($match[2]);
-			}
-		}
 			
 			return null;
 		}
 		
+		$url = empty($match[1]) ? urldecode($match[2]) : urldecode($match[1]);
+		
+		$domain = parse_url($url, PHP_URL_HOST);
+		
+		if(
+			preg_match(
+				'/wikipedia.org$/',
+				$domain
+			)
+		){
+			
+			// rewrite wikipedia mobile URLs to desktop
+			$url =
+				$this->replacedomain(
+					$url,
+					preg_replace(
+						'/([a-z0-9]+)(\.m\.)/',
+						'$1.',
+						$domain
+					)
+				);
+		}
+		
+		if(
+			preg_match(
+				'/imdb\.com$|youtube\.[^.]+$/',
+				$domain
+			)
+		){
+			
+			// rewrite imdb and youtube mobile URLs too
+			$url =
+				$this->replacedomain(
+					$url,
+					preg_replace(
+						'/^m\./',
+						"",
+						$domain
+					)
+				);
+			
+		}
+		
+		return $url;
+	}
+	
+	private function replacedomain($url, $domain){
+		
+		return
+			preg_replace(
+				'/(https?:\/\/)([^\/]+)/',
+				'$1' . $domain,
+				$url
+			);
+	}
+	
 	private function titledots($title){
 		
 		return rtrim($title, ".… \t\n\r\0\x0B");

scraper/mojeek.php

@@ -909,6 +909,23 @@ class mojeek{
 				
 				$a = $a[0];
 				
+				$date =
+					explode(
+						" - ",
+						$this->fuckhtml
+						->getTextContent(
+							$this->fuckhtml
+							->getElementsByTagName(
+								"span"
+							)[0]
+						)
+					);
+				
+				$date =
+					strtotime(
+						$date[count($date) - 1]
+					);
+				
 				$out["news"][] = [
 					"title" =>
 						html_entity_decode(
@@ -918,20 +935,7 @@ class mojeek{
 							)
 						),
 					"description" => null,
-					"date" =>
+					"date" => $date,
-						strtotime(
-							explode(
-								" - ",
-								$this->fuckhtml
-								->getTextContent(
-									$this->fuckhtml
-									->getElementsByTagName(
-										"span"
-									)[0]
-								),
-								2
-							)[1]
-						),
 					"thumb" => [
 						"url" => null,
 						"ratio" => null

settings.php

@@ -70,10 +70,10 @@ $settings = [
 						"value" => "brave",
 						"text" => "Brave"
 					],
-					//[
+					/*[
-					//	"value" => "google",
+						"value" => "google",
-					//	"text" => "Google"
+						"text" => "Google"
-					//],
+					],*/
 					[
 						"value" => "mojeek",
 						"text" => "Mojeek"
@@ -99,11 +99,15 @@ $settings = [
 					[
 						"value" => "yandex",
 						"text" => "Yandex"
-					]//,
+					],
-					//[
+					[
-					//	"value" => "google",
+						"value" => "brave",
-					//	"text" => "Google"
+						"text" => "Brave"
-					//]
+					]/*,
+					[
+						"value" => "google",
+						"text" => "Google"
+					]*/
 				]
 			],
 			[
@@ -117,11 +121,15 @@ $settings = [
 					[
 						"value" => "ddg",
 						"text" => "DuckDuckGo"
-					]//,
+					],
-					//[
+					[
-					//	"value" => "google",
+						"value" => "brave",
-					//	"text" => "Google"
+						"text" => "Brave"
-					//]
+					]/*,
+					[
+						"value" => "google",
+						"text" => "Google"
+					]*/
 				]
 			],
 			[
@@ -136,10 +144,10 @@ $settings = [
 						"value" => "brave",
 						"text" => "Brave"
 					],
-					//[
+					/*[
-					//	"value" => "google",
+						"value" => "google",
-					//	"text" => "Google"
+						"text" => "Google"
-					//],
+					],*/
 					[
 						"value" => "mojeek",
 						"text" => "Mojeek"
@@ -219,7 +227,7 @@ echo
 		'<head>' .
 			'<meta http-equiv="Content-Type" content="text/html;charset=utf-8">' .
 			'<title>Settings</title>' .
-			'<link rel="stylesheet" href="/static/style.css?v2">' .
+			'<link rel="stylesheet" href="/static/style.css">' .
 			'<meta name="viewport" content="width=device-width,initial-scale=1">' .
 			'<meta name="robots" content="index,follow">' .
 			'<link rel="icon" type="image/x-icon" href="/favicon.ico">' .

static/client.js

@@ -360,6 +360,23 @@ if(image_class !== null){
 						elem.getAttribute("data-json")
 					);
 				
+				var imagesize = elem.getElementsByTagName("img")[0];
+				
+				if(imagesize.complete){
+					
+					var imagesize_w = imagesize.naturalWidth;
+					var imagesize_h = imagesize.naturalHeight;
+				}
+				
+				for(var i=0; i<collection.length; i++){
+					
+					if(collection[i].width === null){
+						
+						collection[i].width = imagesize_w;
+						collection[i].height = imagesize_h;
+					}
+				}
+				
 				var title = elem.title;
 			}
 			

template/home.html

@@ -14,7 +14,9 @@
 		<div id="center">
 			<form method="GET" autocomplete="off" action="web">
 				<div class="logo">
+					<a href="/">
 						<img src="{%banner%}" alt="4get">
+					</a>
 				</div>
 				<div class="searchbox">
 					<input type="submit" value="Search" tabindex="-1">
@@ -28,9 +30,9 @@
 			<div class="subtext">
 				Clearnet: <a href="https://4get.ca">4get.ca</a><br>
 				Tor: <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion</a><br>
-				Report a problem: <a href="https://lolcat.ca/contact">lolcat.ca/contact</a>
+				Report a problem: <a href="https://lolcat.ca">lolcat.ca</a>
 			</div>
 		</div>
-		<script src="/static/client.js?v2"></script>
+		<script src="/static/client.js?v3"></script>
 	</body>
 </html>

template/images.html

@@ -2,6 +2,6 @@
 			{%images%}
 		</div>
 		{%nextpage%}
-		<script src="/static/client.js?v2"></script>
+		<script src="/static/client.js?v3"></script>
 	</body>
 </html>

template/search.html

@@ -11,6 +11,6 @@
 				{%left%}
 			</div>
 		</div>
-		<script src="/static/client.js?v2"></script>
+		<script src="/static/client.js?v3"></script>
 	</body>
 </html>