pass startpage js challenge, fix video pagination

scraper/startpage.php

@@ -9,6 +9,9 @@ class startpage{
 		
 		include "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
+		
+		include "lib/anubis.php";
+		$this->anubis = new anubis();
 	}
 	
 	public function getfilters($page){
@@ -244,7 +247,7 @@ class startpage{
 		}
 	}
 	
-	private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
+	private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
 		
 		$curlproc = curl_init();
 		
@@ -259,6 +262,27 @@ class startpage{
 			$url .= "?" . $get;
 		}
 		
+		if($return_cookies){
+			
+			$cookies = [];
+			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
+				
+				$length = strlen($header);
+				
+				$header = explode(":", $header, 2);
+				
+				if(trim(strtolower($header[0])) == "set-cookie"){
+					
+					$cookie_tmp = explode("=", trim($header[1]), 2);
+					
+					$cookies[trim($cookie_tmp[0])] =
+						explode(";", $cookie_tmp[1], 2)[0];
+				}
+				
+				return $length;
+			});
+		}
+		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
 		// http2 bypass
@@ -266,6 +290,15 @@ class startpage{
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
 		
+		$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
+		
+		foreach($passed_cookies as $name => $val){
+			
+			$header_cookies[] = "$name=$val";
+		}
+		
+		$header_cookies = rtrim(implode("; ", $header_cookies));
+		
 		if($is_xhr === true){
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -279,7 +312,7 @@ class startpage{
 				"Origin: https://www.startpage.com/",
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Sec-Fetch-Dest: empty",
 				"Sec-Fetch-Mode: cors",
 				"Sec-Fetch-Site: same-origin",
@@ -298,7 +331,7 @@ class startpage{
 				"Content-Length: " . strlen($get),
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
@@ -316,7 +349,7 @@ class startpage{
 				"Accept-Encoding: gzip",
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
 				"Sec-Fetch-Site: none",
@@ -342,6 +375,15 @@ class startpage{
 		}
 		
 		curl_close($curlproc);
+		
+		if($return_cookies){
+			
+			return [
+				"cookies" => $cookies,
+				"data" => $data
+			];
+		}
+		
 		return $data;
 	}
 	
@@ -349,14 +391,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "web");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
+					$query,
-					true
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -398,7 +448,11 @@ class startpage{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$params
+					$params,
+					false,
+					false,
+					true,
+					[]
 				);
 			}catch(Exception $error){
 				
@@ -406,12 +460,41 @@ class startpage{
 			}
 			
 			//$html = file_get_contents("scraper/startpage.html");
+			
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
+			
+			// solve anubis if need be
+			$cookies = $this->solve_anubis($proxy, $html);
+			
+			if($cookies === false){
+				
+				$cookies = [];
+				$html = $html["data"];
+			}else{
+				
+				try{
+					$html = $this->get(
+						$proxy,
+						"https://www.startpage.com/sp/search",
+						$params,
+						false,
+						false,
+						false,
+						$cookies
+					);
+				}catch(Exception $error){
+					
+					throw new Exception("Failed to re-fetch search page");
+				}
+			}
 		}
 		
-		//echo htmlspecialchars($html);
+		// detect captcha even if we passed anubis challenge
-		
 		$this->detect_captcha($html);
 		
+		//echo htmlspecialchars($html);
+		
 		if(
 			preg_match(
 				'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
@@ -420,14 +503,14 @@ class startpage{
 			) === 0
 		){
 			
-			throw new Exception("Failed to grep JSON object");
+			throw new Exception("Failed to grep JSON object. Anubis error: $anubis_error");
 		}
 		
 		$json = json_decode($matches[1], true);
 		
 		if($json === null){
 			
-			throw new Exception("Failed to decode JSON");
+			throw new Exception("Failed to decode JSON. Anubis error: $anubis_error");
 		}
 		
 		//print_r($json);
@@ -449,7 +532,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "web", $proxy);
+		$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
 		
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
@@ -658,14 +741,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "images");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
+					$query,
-					true
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -680,76 +771,102 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
+			$proxy = $this->backend->get_ip();
+			
+			$params = [
+				"query" => $get["s"],
+				"cat" => "images",
+				"pl" => "opensearch"
+			];
+			
+			if($get["nsfw"] == "no"){
+				
+				$params["qadf"] = "heavy";
+			}
+			
+			if($get["size"] != "any"){
+				
+				if(
+					$get["size"] == "Small" ||
+					$get["size"] == "Medium" ||
+					$get["size"] == "Large" ||
+					$get["size"] == "Wallpaper"
+				){
+					
+					$params["flimgsize"] = $get["size"];
+				}else{
+					
+					$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
+				}
+			}
+			
+			if($get["color"] != "any"){
+				
+				if($get["color"] == "color"){
+					
+					$params["flimgcolor"] = "ic:color";
+				}elseif($get["color"] == "bnw"){
+					
+					$params["flimgcolor"] = "ic:gray";
+				}else{
+					
+					$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
+				}
+			}
+			
+			if($get["type"] != "any"){
+				
+				$params["flimgtype"] = $get["type"];
+			}
+			
+			if($get["license"] != "any"){
+				
+				$params["flimglicense"] = $get["license"];
+			}
+			
 			try{
+				$html = $this->get(
+					$proxy,
+					"https://www.startpage.com/sp/search",
+					$params,
+					false,
+					false,
+					true,
+					[]
+				);
+			}catch(Exception $error){
 				
-				$proxy = $this->backend->get_ip();
+				throw new Exception("Failed to fetch search page");
+			}
 			
-				$params = [
+			//$html = file_get_contents("scraper/startpage.html");
-					"query" => $get["s"],
-					"cat" => "images",
-					"pl" => "opensearch"
-				];
 			
-				if($get["nsfw"] == "no"){
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
 			
-					$params["qadf"] = "heavy";
+			// solve anubis if need be
-				}
+			$cookies = $this->solve_anubis($proxy, $html);
 			
-				if($get["size"] != "any"){
+			if($cookies === false){
 				
-					if(
+				$cookies = [];
-						$get["size"] == "Small" ||
+				$html = $html["data"];
-						$get["size"] == "Medium" ||
+			}else{
-						$get["size"] == "Large" ||
-						$get["size"] == "Wallpaper"
-					){
-						
-						$params["flimgsize"] = $get["size"];
-					}else{
-						
-						$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
-					}
-				}
-				
-				if($get["color"] != "any"){
-					
-					if($get["color"] == "color"){
-						
-						$params["flimgcolor"] = "ic:color";
-					}elseif($get["color"] == "bnw"){
-						
-						$params["flimgcolor"] = "ic:gray";
-					}else{
-						
-						$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
-					}
-				}
-				
-				if($get["type"] != "any"){
-					
-					$params["flimgtype"] = $get["type"];
-				}
-				
-				if($get["license"] != "any"){
-					
-					$params["flimglicense"] = $get["license"];
-				}
 				
 				try{
 					$html = $this->get(
 						$proxy,
 						"https://www.startpage.com/sp/search",
-						$params
+						$params,
+						false,
+						false,
+						false,
+						$cookies
 					);
 				}catch(Exception $error){
 					
-					throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page");
 				}
-				//$html = file_get_contents("scraper/startpage.html");
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
 			}
 		}
 		
@@ -780,7 +897,7 @@ class startpage{
 		}
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "images", $proxy);
+		$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
 		
 		// get images
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -819,14 +936,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "videos");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
+					$query,
-					true
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -841,46 +966,72 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
+			$proxy = $this->backend->get_ip();
+			
+			$params = [
+				"query" => $get["s"],
+				"cat" => "video",
+				"pl" => "opensearch"
+			];
+			
+			if($get["nsfw"] == "no"){
+				
+				$params["qadf"] = "heavy";
+			}
+			
+			if($get["sort"] != "relevance"){
+				
+				$params["sort_by"] = $get["sort"];
+			}
+			
+			if($get["duration"] != "any"){
+				
+				$params["with_duration"] = $get["duration"];
+			}
+			
 			try{
+				$html = $this->get(
+					$proxy,
+					"https://www.startpage.com/sp/search",
+					$params,
+					false,
+					false,
+					true,
+					[]
+				);
+			}catch(Exception $error){
 				
-				$proxy = $this->backend->get_ip();
+				throw new Exception("Failed to fetch search page");
+			}
 			
-				$params = [
+			//$html = file_get_contents("scraper/startpage.html");
-					"query" => $get["s"],
-					"cat" => "video",
-					"pl" => "opensearch"
-				];
 			
-				if($get["nsfw"] == "no"){
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
 			
-					$params["qadf"] = "heavy";
+			// solve anubis if need be
-				}
+			$cookies = $this->solve_anubis($proxy, $html);
 			
-				if($get["sort"] != "relevance"){
+			if($cookies === false){
 				
-					$params["sort_by"] = $get["sort"];
+				$cookies = [];
-				}
+				$html = $html["data"];
-				
+			}else{
-				if($get["duration"] != "any"){
-					
-					$params["with_duration"] = $get["duration"];
-				}
 				
 				try{
 					$html = $this->get(
 						$proxy,
 						"https://www.startpage.com/sp/search",
-						$params
+						$params,
+						false,
+						false,
+						false,
+						$cookies
 					);
 				}catch(Exception $error){
 					
-					throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page");
 				}
-				//$html = file_get_contents("scraper/startpage.html");
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
 			}
 		}
 		
@@ -915,7 +1066,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "video", $proxy);
+		$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
 		
 		// get results
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -971,14 +1122,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "news");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
+					$query,
-					true
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -993,41 +1152,67 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
+			$proxy = $this->backend->get_ip();
+			
+			$params = [
+				"query" => $get["s"],
+				"cat" => "news",
+				"pl" => "opensearch"
+			];
+							
+			if($get["nsfw"] == "no"){
+				
+				$params["qadf"] = "heavy";
+			}
+			
+			if($get["time"] != "any"){
+				
+				$params["with_date"] = $get["time"];
+			}
+			
 			try{
+				$html = $this->get(
+					$proxy,
+					"https://www.startpage.com/sp/search",
+					$params,
+					false,
+					false,
+					true,
+					[]
+				);
+			}catch(Exception $error){
 				
-				$proxy = $this->backend->get_ip();
+				throw new Exception("Failed to fetch search page");
+			}
 			
-				$params = [
+			//$html = file_get_contents("scraper/startpage.html");
-					"query" => $get["s"],
-					"cat" => "news",
-					"pl" => "opensearch"
-				];
 			
-				if($get["nsfw"] == "no"){
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
 			
-					$params["qadf"] = "heavy";
+			// solve anubis if need be
-				}
+			$cookies = $this->solve_anubis($proxy, $html);
 			
-				if($get["time"] != "any"){
+			if($cookies === false){
 				
-					$params["with_date"] = $get["time"];
+				$cookies = [];
-				}
+				$html = $html["data"];
+			}else{
 				
 				try{
 					$html = $this->get(
 						$proxy,
 						"https://www.startpage.com/sp/search",
-						$params
+						$params,
+						false,
+						false,
+						false,
+						$cookies
 					);
 				}catch(Exception $error){
 					
-					throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page");
 				}
-				//$html = file_get_contents("scraper/startpage.html");
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
 			}
 		}
 		
@@ -1058,7 +1243,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "news", $proxy);
+		$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
 		
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
@@ -1101,7 +1286,7 @@ class startpage{
 		return $out;
 	}
 	
-	private function parse_npt($json, $pagetype, $proxy){
+	private function parse_npt($json, $pagetype, $proxy, $cookies){
 		
 		foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
 			
@@ -1109,22 +1294,27 @@ class startpage{
 				
 				return
 					$this->backend->store(
-						http_build_query(
+						json_encode([
-							[
+							"query" =>
-								"lui" => "english",
+								http_build_query(
-								"language" => "english",
+									[
-								"sc" => $json["render"]["search_sc"],
+										"lui" => "english",
-								"t" => "device",
+										"language" => "english",
-								"cat" => $pagetype,
+										"sc" => $json["render"]["search_sc"],
-								"segment" => "organic",
+										"t" => "device",
-								"abd" => "0",
+										"cat" => $pagetype,
-								"abe" => "0",
+										"segment" => "startpage.udog", // "original" sometimes
-								"query" => rawurldecode($json["render"]["query"]),
+										"abd" => "0",
-								"page" => $page["number"],
+										"abe" => "0",
-								"qsr" => "all",
+										"query" => rawurldecode($json["render"]["query"]),
-								"qadf" => "none" // @ todo fix (??)
+										"page" => $page["number"],
-							]
+										"qsr" => "all",
-						),
+										"qadf" => "none", // @ todo fix (??)
+										...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
+									]
+								),
+							"cookies" => $cookies
+						]),
 						$pagetype,
 						$proxy
 					);
@@ -1270,4 +1460,57 @@ class startpage{
 			throw new Exception("Startpage redirected the scraper to an unhandled page");
 		}
 	}
+	
+	private function solve_anubis($proxy, $html){
+		
+		//
+		// solve anubis challenge, if theres one
+		//
+		$anubis_error = "all gucci";
+		$res = null;
+		$cookies = [];
+		
+		try{
+			$res = $this->anubis->scrape($html["data"]);
+			
+		}catch(Exception $error){
+			
+			// do nothang
+			$anubis_error = $error->getMessage();
+		}
+		
+		if($res !== null){
+			
+			// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
+			// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
+			// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
+			// &nonce=3098
+			// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
+			// &elapsedTime=111
+			
+			$anubis_response =
+				$this->get(
+					$proxy,
+					"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
+					[
+						"id" => $res["id"],
+						"response" => $res["response"],
+						"nonce" => $res["nonce"],
+						"redir" => "https://www.startpage.com/sp/search",
+						"elapsedTime" => $res["elapsedTime"]
+					],
+					false,
+					false,
+					true,
+					$html["cookies"]
+				);
+			
+			// re-fetch search page AGAIN with the anubis bullshit
+			$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
+			
+			return $cookies;
+		}
+		
+		return false;
+	}
 }