bypass anubis bullshit on marginalia

Reviewed-on: lolcat/4get#67

data/config.php

@@ -129,6 +129,7 @@ class config{
 	const PROXY_BRAVE = false;
 	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
+	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
 	const PROXY_STARTPAGE = false;
 	const PROXY_QWANT = false;
@@ -143,6 +144,8 @@ class config{
 	const PROXY_YT = false; // youtube
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
+	const PROXY_SANKAKUCOMPLEX = false;
+	const PROXY_FLICKR = false;
 	const PROXY_FIVEHPX = false;
 	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
@@ -160,7 +163,7 @@ class config{
 	// Scraper-specific parameters
 	//
 	
-	// GOOGLE CSE
+	// GOOGLE CSE & GOOGLE API
 	const GOOGLE_CX_ENDPOINT = "d4e68b99b876541f0";
 	
 	// MARGINALIA

favicon.php

@@ -15,7 +15,12 @@ class favicon{
 		
 		header("Content-Type: image/png");
 		
-		if(substr_count($url, "/") !== 2){
+		if(
+			preg_match(
+				'/^https?:\/\/[A-Za-z0-9.-]+$/',
+				$url
+			) === 0
+		){
 			
 			header("X-Error: Only provide the protocol and domain");
 			$this->defaulticon();

lib/anubis.php

@@ -0,0 +1,100 @@
+<?php
+
+//
+// Reference
+// https://github.com/TecharoHQ/anubis/blob/ecc716940e34ebe7249974f2789a99a2c7115e4e/web/js/proof-of-work.mjs
+//
+
+class anubis{
+	
+	public function __construct(){
+		
+		include_once "fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function scrape($html){
+		
+		$this->fuckhtml->load($html);
+		
+		$script =
+			$this->fuckhtml
+			->getElementById(
+				"anubis_challenge",
+				"script"
+			);
+		
+		if(count($script) === 0){
+			
+			throw new Exception("Failed to scrape anubis challenge data");
+		}
+		
+		$script =
+			json_decode(
+				$this->fuckhtml
+				->getTextContent(
+					$script
+				),
+				true
+			);
+		
+		if($script === null){
+			
+			throw new Exception("Failed to decode anubis challenge data");
+		}
+		
+		if(
+			!isset($script["challenge"]) ||
+			!isset($script["rules"]["difficulty"]) ||
+			!is_int($script["rules"]["difficulty"]) ||
+			!is_string($script["challenge"])
+		){
+			
+			throw new Exception("Found invalid challenge data");
+		}
+		
+		return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
+	}
+	
+	private function is_valid_hash($hash, $difficulty){
+		
+		for ($i=0; $i<$difficulty; $i++) {
+			
+			$index = (int)floor($i / 2);
+			$nibble = $i % 2;
+			
+			$byte = ord($hash[$index]);
+			$nibble = ($byte >> ($nibble === 0 ? 4 : 0)) & 0x0f;
+			
+			if($nibble !== 0){
+				return false;
+			}
+		}
+		
+		return true;
+	}
+	
+	public function rape($data, $difficulty = 5){
+		
+		$nonce = 0;
+		
+		while(true){
+			
+			$hash_binary = hash("sha256", $data . $nonce, true);
+			
+			if($this->is_valid_hash($hash_binary, $difficulty)){
+				
+				$hash_hex = bin2hex($hash_binary);
+				
+				return [
+					"response" => $hash_hex,
+					//"data" => $data,
+					//"difficulty" => $difficulty,
+					"nonce" => $nonce
+				];
+			}
+			
+			$nonce++;
+		}
+	}
+}

lib/frontend.php

@@ -939,6 +939,7 @@ class frontend{
 						"brave" => "Brave",
 						"yandex" => "Yandex",
 						"google" => "Google",
+						//"google_api" => "Google API",
 						"google_cse" => "Google CSE",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
@@ -970,10 +971,12 @@ class frontend{
 						"yep" => "Yep",
 						"solofield" => "Solofield",
 						"pinterest" => "Pinterest",
+						"flickr" => "Flickr",
 						"fivehpx" => "500px",
 						"vsco" => "VSCO",
 						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme"
+						"ftm" => "FindThatMeme",
+						//"sankakucomplex" => "SankakuComplex"
 					]
 				];
 				break;

proxy.php

@@ -34,22 +34,46 @@ try{
 		)
 	){
 		
-		if(
-			!isset($image["query"]) ||
-			!isset($image["path"]) ||
-			$image["path"] != "/th"
-		){
+		if(!isset($image["path"])){
 			
-			header("X-Error: Invalid bing image path");
+			header("X-Error: Missing bing image path");
 			$proxy->do404();
 			die();
 		}
 		
-		parse_str($image["query"], $str);
-		
-		if(!isset($str["id"])){
+		//
+		// get image ID
+		// formations:
+		// https://tse2.mm.bing.net/th/id/OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
+		// https://tse2.mm.bing.net/th?id=OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
+		//
+		$id = null;
+		if(isset($image["query"])){
 			
-			header("X-Error: Missing bing ID");
+			parse_str($image["query"], $str);
+			
+			if(isset($str["id"])){
+				
+				$id = $str["id"];
+			}
+		}
+		
+		if($id === null){
+			
+			$id = explode("/th/id/", $image["path"], 2);
+			
+			if(count($id) !== 2){
+				
+				// malformed
+				return $url;
+			}
+			
+			$id = $id[1];
+		}
+		
+		if(is_array($id)){
+			
+			header("X-Error: Missing bing id parameter");
 			$proxy->do404();
 			die();
 		}
@@ -63,7 +87,7 @@ try{
 			case "cover": $req = "&w=207&h=270&p=0&qlt=90"; break;
 		}
 		
-		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . urlencode($str["id"]) . $req, "https://www.bing.com");
+		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . rawurlencode($id) . $req, "https://www.bing.com");
 		die();
 	}
 	

scraper/ddg.php

@@ -285,6 +285,7 @@ class ddg{
 							"display" => "NSFW",
 							"option" => [
 								"yes" => "Yes",
+								"maybe" => "Maybe",
 								"no" => "No"
 							]
 						],
@@ -1345,7 +1346,7 @@ class ddg{
 				$get_filters["iaf"] = $filters;
 			}
 			
-			$nsfw = $get["nsfw"] == "yes" ? "-2" : "-1";
+			$nsfw = $get["nsfw"] == "yes" ? "-1" : "1";
 			$get_filters["kp"] = $nsfw;
 			
 			try{
@@ -1498,8 +1499,12 @@ class ddg{
 				"ia" => "videos"
 			];
 			
-			$nsfw = $get["nsfw"] == "yes" ? "-2" : "-1";
-			$get_filters["kp"] = $nsfw;
+			switch($get["nsfw"]){
+				
+				case "yes": $nsfw = "-2"; break;
+				case "maybe": $nsfw = "-1"; break;
+				case "no": $nsfw = "1"; break;
+			}
 			
 			$filters = [];
 			
@@ -1938,10 +1943,33 @@ class ddg{
 	
 	private function bingimg($url){
 		
-		$parse = parse_url($url);
-		parse_str($parse["query"], $parts);
+		$image = parse_url($url);
 		
-		return "https://" . $parse["host"] . "/th?id=" . urlencode($parts["id"]);
+		$id = null;
+		if(isset($image["query"])){
+			
+			parse_str($image["query"], $str);
+			
+			if(isset($str["id"])){
+				
+				$id = $str["id"];
+			}
+		}
+		
+		if($id === null){
+			
+			$id = explode("/th/id/", $image["path"], 2);
+			
+			if(count($id) !== 2){
+				
+				// malformed
+				return $url;
+			}
+			
+			$id = $id[1];
+		}
+		
+		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
 	
 	private function bingratio($width, $height){

scraper/flickr.php

@@ -0,0 +1,415 @@
+<?php
+
+class flickr{
+	
+	const req_web = 0;
+	const req_xhr = 1;
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("flickr");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes",
+					"maybe" => "Maybe",
+					"no" => "No",
+				]
+			],
+			"sort" => [
+				"display" => "Sort by",
+				"option" => [
+					"relevance" => "Relevance",
+					"date-posted-desc" => "Newest uploads",
+					"date-posted-asc" => "Oldest uploads",
+					"date-taken-desc" => "Newest taken",
+					"date-taken-asc" => "Oldest taken",
+					"interestingness-desc" => "Interesting"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					// color_codes=
+					"0" => "Red",
+					"1" => "Brown",
+					"2" => "Orange",
+					"b" => "Pink",
+					"4" => "Yellow",
+					"3" => "Golden",
+					"5" => "Lime",
+					"6" => "Green",
+					"7" => "Sky blue",
+					"8" => "Blue",
+					"9" => "Purple",
+					"a" => "Hot pink",
+					"c" => "White",
+					"d" => "Gray",
+					"e" => "Black",
+					// styles= override
+					"blackandwhite" => "Black & white",
+				]
+			],
+			"style" => [ // styles=
+				"display" => "Style",
+				"option" => [
+					"any" => "Any style",
+					"depthoffield" => "Depth of field",
+					"minimalism" => "Minimalism",
+					"pattern" => "Patterns"
+				]
+			],
+			"license" => [
+				"display" => "License",
+				"option" => [
+					"any" => "Any license",
+					"1,2,3,4,5,6,9,11,12,13,14,15,16" => "All creative commons",
+					"4,5,6,9,10,11,12,13" => "Commercial use allowed",
+					"1,2,4,5,9,10,11,12,14,15" => "Modifications allowed",
+					"4,5,9,10,11,12" => "Commercial use & mods allowed",
+					"7,9,10" => "No known copyright restrictions",
+					"8" => "U.S Government works"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $reqtype){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($reqtype === flickr::req_web){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-User: ?1",
+				"Priority: u=0, i",
+				"TE: trailers"]
+			);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"Origin: https://www.flickr.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Referer: https://www.flickr.com/",
+				// Cookie:
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"TE: trailers"]
+			);
+		}
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filters, $proxy] =
+				$this->backend->get(
+					$get["npt"], "images"
+				);
+			
+			$filters = json_decode($filters, true);
+			
+			// Workaround for the future, if flickr deprecates &page argument on html page
+			/*
+			try{
+				
+				$json =
+					$this->get(
+						$proxy,
+						"https://api.flickr.com/services/rest",
+						[
+							"sort" => $data["sort"],
+							"parse_tags" => 1,
+							// url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o
+							"extras" => "can_comment,can_print,count_comments,count_faves,description,isfavorite,license,media,needs_interstitial,owner_name,path_alias,realname,rotation,url_sq,url_q,url_t,url_s,url_n,url_w,url_m,url_z,url_c,url_l",
+							"per_page" => 100,
+							"page" => $data["page"],
+							"lang" => "en-US",
+							"text" => $data["search"],
+							"viewerNSID" => "",
+							"method" => "flickr.photos.search",
+							"csrf" => "",
+							"api_key" => $data["api_key"],
+							"format" => "json",
+							"hermes" => 1,
+							"hermesClient" => 1,
+							"reqId" => $data["reqId"],
+							"nojsoncallback" => 1
+						]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JSON");
+			}*/
+			
+		}else{
+			
+			if(strlen($get["s"]) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+						
+			// compute filters
+			$filters = [
+				"page" => 1,
+				"sort" => $get["sort"]
+			];
+			
+			if($get["style"] != "any"){
+				
+				$filters["styles"] = $get["style"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				if($get["color"] != "blackandwhite"){
+					
+					$filters["color_codes"] = $get["color"];
+				}else{
+					
+					$filters["styles"] = "blackandwhite";
+				}
+			}
+			
+			if($get["license"] != "any"){
+				
+				$filters["license"] = $get["license"];
+			}
+			
+			switch($get["nsfw"]){
+				
+				case "yes": $filters["safe_search"] = 0; break;
+				case "maybe": $filters["safe_search"] = 2; break;
+				case "no": $filters["safe_search"] = 1; break;
+			}
+		}
+		
+		$get_params = [
+			"text" => $get["s"],
+			"per_page" => 50,
+			// scrape highest resolution
+			"extras" => "url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o",
+			"view_all" => 1
+		];
+		
+		$get_params = array_merge($get_params, $filters);
+		
+		$html =
+			$this->get(
+				$proxy,
+				"https://www.flickr.com/search/",
+				$get_params,
+				flickr::req_web
+			);
+		
+		// @TODO
+		// get api_key and reqId, if flickr deprecates &page
+		
+		$this->fuckhtml->load($html);
+		
+		//
+		// get response JSON
+		//
+		$scripts =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"modelExport",
+				"script"
+			);
+		
+		$found = false;
+		foreach($scripts as $script){
+			
+			$json =	
+				preg_split(
+					'/modelExport: ?/',
+					$script["innerHTML"],
+					2
+				);
+			
+			if(count($json) !== 0){
+				
+				$found = true;
+				$json = $json[1];
+				break;
+			}
+		}
+		
+		if($found === false){
+			
+			throw new Exception("Failed to grep JSON");
+		}
+		
+		$json =
+			json_decode(
+				$this->fuckhtml
+				->extract_json(
+					$json
+				),
+				true
+			);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(!isset($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"])){
+			
+			throw new Exception("Failed to access data object");
+		}
+		
+		foreach($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"] as $image){
+			
+			if(!isset($image["data"])){
+				
+				// flickr likes to gives us empty array objects
+				continue;
+			}
+			
+			$image = $image["data"];
+			
+			$title = [];
+			
+			if(isset($image["title"])){
+				
+				$title[] =
+					$this->fuckhtml
+					->getTextContent(
+						$image["title"]
+					);
+			}
+			
+			if(isset($image["description"])){
+				
+				$title[] =
+					$this->fuckhtml
+					->getTextContent(
+						str_replace(
+							"\n",
+							" ",
+							$image["description"]
+						)
+					);
+			}
+			
+			$title = implode(": ", $title);
+			
+			$sources = array_values($image["sizes"]["data"]);
+			
+			$suitable_sizes = ["n", "m", "w", "s"];
+			
+			$thumb = &$sources[0]["data"];
+			foreach($suitable_sizes as $testing_size){
+				
+				if(isset($image["sizes"]["data"][$testing_size])){
+					
+					$thumb = &$image["sizes"]["data"][$testing_size]["data"];
+					break;
+				}
+			}
+			
+			$og = &$sources[count($sources) - 1]["data"];
+			
+			$out["image"][] = [
+				"title" => $title,
+				"source" => [
+					[
+						"url" => "https:" . $og["displayUrl"],
+						"width" => (int)$og["width"],
+						"height" => (int)$og["height"]
+					],
+					[
+						"url" => "https:" . $thumb["displayUrl"],
+						"width" => (int)$thumb["width"],
+						"height" => (int)$thumb["height"]
+					]
+				],
+				"url" => "https://www.flickr.com/photos/" . $image["ownerNsid"] . "/" . $image["id"] . "/"
+			];
+		}
+		
+		$total_items = (int)$json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["totalItems"];
+		
+		if(($filters["page"]) * 50 < $total_items){
+			
+			$filters["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filters),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+}

scraper/imgur.php

@@ -182,6 +182,23 @@ class imgur{
 			throw new Exception("Failed to fetch HTML");
 		}
 		
+		$json = json_decode($html, true);
+		
+		if($json){
+			
+			// {"data":{"error":"Imgur is temporarily over capacity. Please try again later."},"success":false,"status":403}
+			
+			if(isset($json["data"]["error"])){
+				
+				if(stripos($json["data"]["error"], "capacity")){
+					
+					throw new Exception("Imgur IP blocked this 4get instance or request proxy. Try again");
+				}
+			}
+			
+			throw new Exception("Imgur returned an unknown error (IP ban?)");
+		}
+		
 		$this->fuckhtml->load($html);
 		
 		$posts =
@@ -197,7 +214,14 @@ class imgur{
 			
 			$image =
 				$this->fuckhtml
-				->getElementsByTagName("img")[0];
+				->getElementsByTagName("img");
+			
+			if(count($image) === 0){
+				
+				continue;
+			}
+			
+			$image = $image[0];
 			
 			$image_url = "https:" . substr($this->fuckhtml->getTextContent($image["attributes"]["src"]), 0, -5);
 			

scraper/marginalia.php

@@ -3,7 +3,10 @@
 class marginalia{
 	public function __construct(){
 		
-		include "lib/fuckhtml.php";
+		include "lib/anubis.php";
+		$this->anubis = new anubis();
+		
+		include_once "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
 		
 		include "lib/backend.php";
@@ -102,7 +105,40 @@ class marginalia{
 		);
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $get_cookies = 1){
+		
+		$curlproc = curl_init();
+		
+		switch($get_cookies){
+			
+			case 0:
+				$cookies = "";
+				$cookies_tmp = [];
+				curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
+					
+					$length = strlen($header);
+					
+					$header = explode(":", $header, 2);
+					
+					if(trim(strtolower($header[0])) == "set-cookie"){
+						
+						$cookie_tmp = explode("=", trim($header[1]), 2);
+						
+						$cookies_tmp[trim($cookie_tmp[0])] =
+							explode(";", $cookie_tmp[1], 2)[0];
+					}
+					
+					return $length;
+				});
+				break;
+			
+			case 1:
+				$cookies = "";
+				break;
+			
+			default:
+				$cookies = "Cookie: " . $get_cookies;
+		}
 		
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
@@ -110,6 +146,7 @@ class marginalia{
 			"Accept-Language: en-US,en;q=0.5",
 			"Accept-Encoding: gzip",
 			"DNT: 1",
+			$cookies,
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
 			"Sec-Fetch-Dest: document",
@@ -118,8 +155,6 @@ class marginalia{
 			"Sec-Fetch-User: ?1"
 		];
 		
-		$curlproc = curl_init();
-		
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
@@ -145,7 +180,19 @@ class marginalia{
 			throw new Exception(curl_error($curlproc));
 		}
 		
-		curl_close($curlproc);
+		if($get_cookies === 0){
+			
+			$cookie = [];
+			
+			foreach($cookies_tmp as $key => $value){
+				
+				$cookie[] = $key . "=" . $value;
+			}
+			
+			curl_close($curlproc);
+			return implode(";", $cookie);
+		}
+		
 		return $data;
 	}
 	
@@ -267,6 +314,55 @@ class marginalia{
 		// HTML parser
 		$proxy = $this->backend->get_ip();
 		
+		//
+		// Bypass anubis check
+		//
+		if(($anubis_key = apcu_fetch("marginalia_cookie")) === false){
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/"
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get anubis challenge");
+			}
+			
+			try{
+				
+				$anubis_data = $this->anubis->scrape($html);
+			}catch(Exception $error){
+				
+				throw new Exception($error);
+			}
+			
+			// send anubis response & get cookies
+			// https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge?response=0000018966b086834f738bacba6031028adb5aa875974ead197a8b75778baf3a&nonce=39947&redir=https%3A%2F%2Fold-search.marginalia.nu%2F&elapsedTime=1164
+			
+			try{
+				
+				$anubis_key =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge",
+						[
+							"response" => $anubis_data["response"],
+							"nonce" => $anubis_data["nonce"],
+							"redir" => "https://old-search.marginalia.nu/",
+							"elapsedTime" => random_int(1000, 2000)
+						],
+						0
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to submit anubis challenge");
+			}
+			
+			apcu_store("marginalia_cookie", $anubis_key);
+		}
+		
 		if($get["npt"]){
 			
 			[$params, $proxy] =
@@ -279,7 +375,9 @@ class marginalia{
 				$html =
 					$this->get(
 						$proxy,
-						"https://old-search.marginalia.nu/search?" . $params
+						"https://old-search.marginalia.nu/search?" . $params,
+						[],
+						$anubis_key
 					);
 			}catch(Exception $error){
 				
@@ -309,7 +407,8 @@ class marginalia{
 					$this->get(
 						$proxy,
 						"https://old-search.marginalia.nu/search",
-						$params
+						$params,
+						$anubis_key
 					);
 			}catch(Exception $error){
 				

scraper/qwant.php

@@ -410,10 +410,7 @@ class qwant{
 					"thumb" =>
 						$answer["data"]["result"]["thumbnail"]["landscape"] == null ?
 						null :
-						$this->unshitimage(
-							$answer["data"]["result"]["thumbnail"]["landscape"],
-							false
-						),
+						$this->unshitimage($answer["data"]["result"]["thumbnail"]["landscape"]),
 					"table" => [],
 					"sublink" => []
 				];
@@ -770,7 +767,7 @@ class qwant{
 			}else{
 				
 				$thumb = [
-					"url" => $this->unshitimage($video["thumbnail"], false),
+					"url" => $this->unshitimage($video["thumbnail"]),
 					"ratio" => "16:9"
 				];
 			}
@@ -870,7 +867,7 @@ class qwant{
 			}else{
 				
 				$thumb = [
-					"url" => $this->unshitimage($news["media"][0]["pict_big"]["url"], false),
+					"url" => $this->unshitimage($news["media"][0]["pict_big"]["url"]),
 					"ratio" => "16:9"
 				];
 			}
@@ -920,18 +917,77 @@ class qwant{
 		return trim($text, ". ");
 	}
 	
-	private function unshitimage($url, $is_bing = true){
+	private function unshitimage($url){
 		
 		// https://s1.qwant.com/thumbr/0x0/8/d/f6de4deb2c2b12f55d8bdcaae576f9f62fd58a05ec0feeac117b354d1bf5c2/th.jpg?u=https%3A%2F%2Fwww.bing.com%2Fth%3Fid%3DOIP.vvDWsagzxjoKKP_rOqhwrQAAAA%26w%3D160%26h%3D160%26c%3D7%26pid%3D5.1&q=0&b=1&p=0&a=0
-		parse_str(parse_url($url)["query"], $parts);
+		// https://s2.qwant.com/thumbr/474x289/7/f/412d13b3fe3a03eb2b89633c8e88b609b7d0b93cdd9a5e52db3c663e41e65e/th.jpg?u=https%3A%2F%2Ftse.mm.bing.net%2Fth%3Fid%3DOIP.9Tm_Eo6m7V7ltN19mxduDgHaEh%26pid%3DApi&q=0&b=1&p=0&a=0
 		
-		if($is_bing){
-			$parse = parse_url($parts["u"]);
-			parse_str($parse["query"], $parts);
+		$image = parse_url($url);
+		
+		if(
+			!isset($image["host"]) ||
+			!isset($image["query"])
+		){
 			
-			return "https://" . $parse["host"] . "/th?id=" . urlencode($parts["id"]);
+			// cant do anything
+			return $url;
 		}
 		
-		return $parts["u"];
+		$id = null;
+		
+		if(
+			preg_match(
+				'/s[0-9]+\.qwant\.com$/',
+				$image["host"]
+			)
+		){
+			
+			parse_str($image["query"], $str);
+			
+			// we're being served a proxy URL
+			if(isset($str["u"])){
+				
+				$bing_url = $str["u"];
+			}else{
+				
+				// give up
+				return $url;
+			}
+		}
+		
+		// parse bing URL
+		$id = null;
+		$image = parse_url($bing_url);
+		
+		if(isset($image["query"])){
+			
+			parse_str($image["query"], $str);
+			
+			if(isset($str["id"])){
+				
+				$id = $str["id"];
+			}
+		}
+		
+		if($id === null){
+			
+			$id = explode("/th/id/", $image["path"], 2);
+			
+			if(count($id) !== 2){
+				
+				// malformed
+				return $url;
+			}
+			
+			$id = $id[1];
+		}
+		
+		if(is_array($id)){
+			
+			// fuck off, let proxy.php deal with it
+			return $url;
+		}
+		
+		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
 }

scraper/startpage.php

@@ -1226,7 +1226,12 @@ class startpage{
 		// get results
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
-			if($category["display_type"] == "video-youtube"){
+			if(
+				preg_match(
+					'/^video-/i',
+					$category["display_type"]
+				)
+			){
 				
 				foreach($category["results"] as $video){
 					
@@ -1248,7 +1253,7 @@ class startpage{
 					}
 					
 					$out["video"][] = [
-						"title" => $video["title"],
+						"title" => str_replace(["", ""], "", $video["title"]),
 						"description" => $this->limitstrlen($video["description"]),
 						"author" => [
 							"name" => $video["channelTitle"],
@@ -1256,7 +1261,7 @@ class startpage{
 							"avatar" => null
 						],
 						"date" => strtotime($video["publishDate"]),
-						"duration" => $this->hms2int($video["duration"]),
+						"duration" => $this->hms2int($category["display_type"] == "video-youtube" ? $video["duration"] : $video["duration"] / 1000),
 						"views" => (int)$video["viewCount"],
 						"thumb" => $thumb,
 						"url" => $video["clickUrl"]

scraper/yandex.php

@@ -14,7 +14,7 @@ class yandex{
 		// backend included in the scraper functions
 	}
 	
-	private function get($proxy, $url, $get = [], $nsfw){
+	private function get($proxy, $url, $get = [], $nsfw, $get_cookie = 1){
 		
 		$curlproc = curl_init();
 		
@@ -25,19 +25,55 @@ class yandex{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
+		// extract "i" cookie
+		if($get_cookie === 0){
+			
+			$cookies_tmp = [];
+			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
+				
+				$length = strlen($header);
+				
+				$header = explode(":", $header, 2);
+				
+				if(trim(strtolower($header[0])) == "set-cookie"){
+					
+					$cookie_tmp = explode("=", trim($header[1]), 2);
+					
+					$cookies_tmp[trim($cookie_tmp[0])] =
+						explode(";", $cookie_tmp[1], 2)[0];
+				}
+				
+				return $length;
+			});
+		}
+		
 		switch($nsfw){
 			case "yes": $nsfw = "0"; break;
 			case "maybe": $nsfw = "1"; break;
 			case "no": $nsfw = "2"; break;
 		}
 		
+		switch($get_cookie){
+			
+			case 0:
+				$cookie = "";
+				break;
+			
+			case 1:
+				$cookie = "Cookie: yp=" . (time() - 4000033) . ".szm.1:1920x1080:876x1000#" . time() . ".sp.family:" . $nsfw;
+				break;
+			
+			default:
+				$cookie = "Cookie: i=" . $get_cookie;
+		}
+		
 		$headers =
 			["User-Agent: " . config::USER_AGENT,
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
 			"Accept-Encoding: gzip",
 			"Accept-Language: en-US,en;q=0.5",
 			"DNT: 1",
-			"Cookie: yp=1716337604.sp.family%3A{$nsfw}#1685406411.szm.1:1920x1080:1920x999",
+			$cookie,
 			"Referer: https://yandex.com/images/search",
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
@@ -59,6 +95,17 @@ class yandex{
 		
 		$data = curl_exec($curlproc);
 		
+		if($get_cookie === 0){
+			
+			if(isset($cookies_tmp["i"])){
+				
+				return $cookies_tmp["i"];
+			}else{
+				
+				throw new Exception("Failed to get Yandex clearance cookie");
+			}
+		}
+		
 		if(curl_errno($curlproc)){
 			
 			throw new Exception(curl_error($curlproc));
@@ -217,6 +264,23 @@ class yandex{
 		// https://yandex.com/search/site/?text=minecraft&web=1&frame=1&v=2.0&searchid=3131712
 		// &within=777&from_day=26&from_month=8&from_year=2023&to_day=26&to_month=8&to_year=2023
 		
+		// get clearance cookie
+		if(($cookie = apcu_fetch("yandexweb_cookie")) === false){
+			
+			$proxy = $this->backend->get_ip();
+			
+			$cookie =
+				$this->get(
+					$proxy,
+					"https://yandex.ru/support2/smart-captcha/ru/",
+					[],
+					false,
+					0
+				);
+			
+			apcu_store("yandexweb_cookie", $cookie);
+		}
+		
 		if($get["npt"]){
 			
 			[$npt, $proxy] = $this->backend->get($get["npt"], "web");
@@ -226,7 +290,8 @@ class yandex{
 					$proxy,
 					"https://yandex.com" . $npt,
 					[],
-					"yes"
+					"yes",
+					$cookie
 				);
 		}else{
 			
@@ -236,7 +301,7 @@ class yandex{
 				throw new Exception("Search term is empty!");
 			}
 			
-			$proxy = $this->backend->get_ip();
+			$proxy = !isset($proxy) ? $this->backend->get_ip() : $proxy;
 			$lang = $get["lang"];
 			$older = $get["older"];
 			$newer = $get["newer"];
@@ -283,7 +348,8 @@ class yandex{
 						$proxy,
 						"https://yandex.com/search/site/",
 						$params,
-						"yes"
+						"yes",
+						$cookie
 					);
 			}catch(Exception $error){
 				
@@ -314,6 +380,19 @@ class yandex{
 		
 		$this->fuckhtml->load($html);
 		
+		// Scrape page blocked error
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName("title");
+		
+		if(
+			count($title) !== 0 &&
+			$title[0]["innerHTML"] == "403"
+		){
+			
+			throw new Exception("Yandex blocked this proxy or 4get instance.");
+		}
+		
 		// get nextpage
 		$npt =
 			$this->fuckhtml
@@ -668,7 +747,6 @@ class yandex{
 		foreach($json["blocks"] as $block){
 			
 			$html .= $block["html"];
-			
 			// get next page
 			if(
 				isset($block["params"]["nextPageUrl"]) &&

settings.php

@@ -231,6 +231,10 @@ $settings = [
 						"value" => "pinterest",
 						"text" => "Pinterest"
 					],
+					[
+						"value" => "flickr",
+						"text" => "Flickr"
+					],
 					[
 						"value" => "fivehpx",
 						"text" => "500px"

static/themes/Gore's shitty theme.css

@@ -1,48 +1,45 @@
 :root{
-	--1d2021: #1d2021;
-	--282828: #282828;
-	--3c3836: #3c3836;
-	--504945: #504945;
-	
+	--1d2021:#1d2021;
+	--282828:#282828;
+	--3c3836:#3c3836;
+	--504945:#504945;
+
 	/* font */
-	--928374: #928374;
-	--a89984: #c9c5bf;
-	--bdae93: #bdae93;
-	--8ec07c: #8ec07c;
-	--ebdbb2: #ebdbb2;
+	--928374:#928374;
+	--a89984:#c9c5bf;
+	--bdae93:#bdae93;
+	--8ec07c:#8ec07c;
+	--ebdbb2:#ebdbb2;
 }
 
-
-
 body{
 	padding:15px 4% 40px;
 	margin:unset;
 }
 
-h1,h2,h3,h4,h5,h6{
+h1, h2, h3, h4, h5, h6{
 	padding:0;
 	margin:0 0 7px 0;
 	line-height:initial;
 	color:var(--bdae93);
 }
 
-h3,h4,h5,h6{
+h3, h4, h5, h6{
 	margin-bottom:14px;
 }
 
 /*
-	Web styles
+  Web styles
 */
 
 .searchbox input[type="submit"]{
 	float:right;
 	cursor:pointer;
 	padding:0 10px;
-  border-left: 1px solid var(--504945);
-  background: #723c0b;
+	border-left:1px solid var(--504945);
+	background:#723c0b;
 }
 
-
 .searchbox input{
 	all:unset;
 	line-height:36px;
@@ -97,7 +94,6 @@ h3,h4,h5,h6{
 	display:inline-block;
 }
 
-
 .tabs .tab.selected{
 	border-bottom:2px solid #fc92a5;
 }
@@ -107,7 +103,7 @@ h3,h4,h5,h6{
 	padding-bottom:12px;
 	padding-top:7px;
 	margin-bottom:7px;
-	background-color:#232525
+	background-color:#232525;
 }
 
 .filters .filter{
@@ -170,7 +166,6 @@ h3,h4,h5,h6{
 	font-size:12px;
 }
 
-
 .web .hover{
 	display:block;
 	text-decoration:none;
@@ -194,16 +189,13 @@ h3,h4,h5,h6{
 	color:#9760b1 !important;
 }
 
-
 .web .text-result .greentext{
 	font-size:14px;
 	color:var(--bdae93);
 }
 
-
 /* favicon */
 
-
 .favicon-dropdown a{
 	text-decoration:none;
 	color:#d3d0c1;
@@ -212,39 +204,33 @@ h3,h4,h5,h6{
 	font-size:13px;
 }
 
-
-.web .favicon img,
-.favicon-dropdown img{
+.web .favicon img, .favicon-dropdown img{
 	margin:3px 7px 0 0;
 	height:16px;
 	font-size:12px;
-	line-height:16px;;
+	line-height:16px;
 	display:block;
 	text-align:left;
 }
 
-
 .web .sublinks{
 	padding:17px 10px;
 	font-size:15px;
 	color:var(--#928374);
 }
 
-
 .web .text-result .sublinks:last-child{
 	padding-bottom:0;
 }
 
-
 /* Wikipedia head */
 .wiki-head{
 	padding:5px;
-	background-color: #322f2b
+	background-color:#322f2b;
 }
 
-
 /*
-	Images tab
+  Images tab
 */
 
 #images{
@@ -258,17 +244,14 @@ h3,h4,h5,h6{
 	float:left;
 }
 
-
-
 #images .image .title{
 	white-space:nowrap;
 	overflow:hidden;
 	margin-bottom:7px;
 	font-weight:bold;
-    color:var(--bdae93);
+	color:var(--bdae93);
 }
 
-
 #popup-status{
 	display:none;
 	position:fixed;
@@ -281,43 +264,59 @@ h3,h4,h5,h6{
 }
 
 /*
-	Settings page
+  Settings page
 */
 
-
 .web .settings-submit a{
 	margin-right:17px;
 	color:#bdae93;
 }
 
+/*
+  Responsive image
+*/
+@media only screen and (max-width:1454px){
+	#images .image-wrapper{
+		width:25%;
+	}
+}
+
+@media only screen and (max-width:1161px){
+	#images .image-wrapper{
+		width:25%;
+	}
+}
+
+@media only screen and (max-width:750px){
+	#images .image-wrapper{
+		width:50%;
+	}
+}
+
+@media only screen and (max-width:450px){
+	#images .image-wrapper{
+		width:100%;
+	}
+}
 
 /*
-	Responsive image
+  Responsive design
 */
-@media only screen and (max-width: 1454px){ #images .image-wrapper{ width:25%; } }
-@media only screen and (max-width: 1161px){ #images .image-wrapper{ width:25%; } }
-@media only screen and (max-width: 750px){ #images .image-wrapper{ width:50%; } }
-@media only screen and (max-width: 450px){ #images .image-wrapper{ width:100%; } }
 
-
-/*
-	Responsive design
-*/
-@media only screen and (max-width: 1550px){
-	
-	
-	.web .left,
+@media only screen and (max-width:1550px){
+.web .left,
 	.searchbox{
 		width:60%;
 	}
-
 }
 
-@media only screen and (max-width: 1000px){
-
+@media only screen and (max-width:1100px){
+.web .left,
+	.searchbox{
+		width:100%;
+	}
 }
 
-    .type{
-    color:var(--bdae93);
-    }
+.type{
+	color:var(--bdae93);
 }