bypass google softcaptcha for real this time fuuuuuuuuuuuuuuuuuuuuuuuuuck

docs/configure.md

@@ -163,3 +163,7 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
 **IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
 
 If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
+
+In case it isn't clear, here is a visual representation of what you want to achieve:
+
+![Network map](https://i.imgur.com/jFjbBtG.png)

scraper/google.php

@@ -523,7 +523,7 @@ class google{
 					config::FPLAY_EXTERNAL_ENDPOINT .
 					"?url=" .				urlencode($url) .
 					"&pw=" .				urlencode(config::FPLAY_PASSWORD) .
-					"&container=" .			urlencode($container) .
+					"&container=" .//		urlencode($container) .
 					"&proxy=" .				urlencode($proxy) .
 					"&url_match=" .			urlencode("&sei=") .
 					"&fail_url_match=" .	urlencode("\/sorry\/");
@@ -733,7 +733,7 @@ class google{
 			$container = &$data["container"];
 			
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/google-jp.html");
+			//$html = file_get_contents("scraper/google-softcaptcha.html");
 		}
 		
 		$out = [
@@ -754,9 +754,26 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		// init
+		// solve softcaptcha
-		$this->detect_sorry();
+		$softcaptcha = $this->detect_sorry();
-		$this->parsestyles();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
+		
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		
@@ -1381,7 +1398,28 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get javascript images
 		$this->scrape_imagearr($html);
@@ -1563,13 +1601,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
+		
 		//
 		// Parse web video page
 		//
-		$this->detect_sorry();
-		
-		// parse all <style> tags
-		$this->parsestyles();
 		
 		// get javascript images
 		$this->scrape_dimg($html);
@@ -1887,14 +1944,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse styles
+		$this->parsestyles();		
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get images
 		$this->scrape_dimg($html);
 		
-		// parse styles
-		$this->parsestyles();
-		
 		$center_col =
 			$this->fuckhtml
 			->getElementById(
@@ -2653,6 +2728,7 @@ class google{
 	
 	private function detect_sorry(){
 		
+		// detect recaptcha
 		$captcha_form =
 			$this->fuckhtml
 			->getElementById(
@@ -2682,5 +2758,35 @@ class google{
 			
 			throw new Exception("Google returned a JS challenge");
 		}
+		
+		// bypass soft captcha
+		// "Verifying your request
+		// You'll be able to continue in a few seconds. Don't refresh this page."
+		$scripts =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"script"
+			);
+		
+		foreach($scripts as $sc){
+			
+			if(
+				preg_match(
+					'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
+					$sc["innerHTML"],
+					$matches
+				)
+			){
+				
+				sleep(10); // gotta sleep to not trigger captcha. blame google, not me
+				
+				return [
+					"sei" => $matches[1],
+					"timeout" => (int)$matches[2] // set to 10000 usually
+				];
+			}
+		}
+		
+		return false; // no need to bypass
 	}
 }