bypass google softcaptcha for real this time fuuuuuuuuuuuuuuuuuuuuuuuuuck

.gitignore

@@ -26,4 +26,3 @@ data/captcha/minecraft/
 !banner/*default*
 scraper/curlie.html
 icons/*
-node_modules/

docs/configure.md

@@ -163,3 +163,7 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
 **IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
 
 If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
+
+In case it isn't clear, here is a visual representation of what you want to achieve:
+
+![Network map](https://i.imgur.com/jFjbBtG.png)

docs/docker.md

@@ -150,7 +150,3 @@ services:
 ##### Tor
 
 You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
-
-##### 4play
-
-You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)

docs/docker_4play.md

@@ -1,48 +0,0 @@
-### Running 4get and 4play via docker guide
-
-This setup will run both 4get and the 4play sidecar 
-
-# Docker compose 
-
-```
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"
-
-```
-
-There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
-
-# Steps
-
-1. start the services with
-
-```
-docker compose up -d
-
-```
-
-2. configure firefox
-
-folllow the steps in [configure](./configure.md) for firefox setup

extra/4play/Dockerfile

@@ -1,19 +0,0 @@
-FROM alpine:3.21
-WORKDIR /app
-
-RUN apk update && apk upgrade
-RUN apk add nodejs npm
-
-COPY package*.json ./
-
-RUN npm ci --only=production && npm cache clean --force
-
-COPY . .
-
-EXPOSE 3030
-
-ENTRYPOINT  ["node"]
-CMD ["page-render.js"]
-
-
-

extra/4play/docker-compose.yaml

@@ -1,24 +0,0 @@
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"

extra/4play/package-lock.json

@@ -1,47 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "4play",
-      "version": "1.0.0",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "@lawlers/4play": "^1.2.5",
-        "ws": "^8.21.2"
-      }
-    },
-    "node_modules/@lawlers/4play": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
-      "integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "ws": "^8.21.1"
-      }
-    },
-    "node_modules/ws": {
-      "version": "8.21.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
-      "integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    }
-  }
-}

extra/4play/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "description": "",
-  "repository": {
-    "type": "git",
-    "url": "https://git.lolcat.ca/lolcat/4get/"
-  },
-  "license": "AGPL-3.0-only",
-  "author": "lolcat",
-  "type": "commonjs",
-  "main": "page-render.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "dependencies": {
-    "@lawlers/4play": "^1.2.5",
-    "ws": "^8.21.2"
-  }
-}

extra/4play/page-render.js

@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
 const http = require("http");
 
 // 4play settings
-var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
+var port = 3030;
-var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
+var timeout = 30000;
-var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
+var password = "cnc";
 
 // ==================================================
 

scraper/google.php

@@ -523,7 +523,7 @@ class google{
 					config::FPLAY_EXTERNAL_ENDPOINT .
 					"?url=" .				urlencode($url) .
 					"&pw=" .				urlencode(config::FPLAY_PASSWORD) .
-					"&container=" .			urlencode($container) .
+					"&container=" .//		urlencode($container) .
 					"&proxy=" .				urlencode($proxy) .
 					"&url_match=" .			urlencode("&sei=") .
 					"&fail_url_match=" .	urlencode("\/sorry\/");
@@ -733,7 +733,7 @@ class google{
 			$container = &$data["container"];
 			
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/google-jp.html");
+			//$html = file_get_contents("scraper/google-softcaptcha.html");
 		}
 		
 		$out = [
@@ -754,9 +754,26 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		// init
+		// solve softcaptcha
-		$this->detect_sorry();
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
 			$this->parsestyles();
+		}
+		
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		
@@ -1381,7 +1398,28 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get javascript images
 		$this->scrape_imagearr($html);
@@ -1563,13 +1601,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
+		
 		//
 		// Parse web video page
 		//
-		$this->detect_sorry();
-		
-		// parse all <style> tags
-		$this->parsestyles();
 		
 		// get javascript images
 		$this->scrape_dimg($html);
@@ -1887,14 +1944,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse styles
+		$this->parsestyles();		
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get images
 		$this->scrape_dimg($html);
 		
-		// parse styles
-		$this->parsestyles();
-		
 		$center_col =
 			$this->fuckhtml
 			->getElementById(
@@ -2653,6 +2728,7 @@ class google{
 	
 	private function detect_sorry(){
 		
+		// detect recaptcha
 		$captcha_form =
 			$this->fuckhtml
 			->getElementById(
@@ -2682,5 +2758,35 @@ class google{
 			
 			throw new Exception("Google returned a JS challenge");
 		}
+		
+		// bypass soft captcha
+		// "Verifying your request
+		// You'll be able to continue in a few seconds. Don't refresh this page."
+		$scripts =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"script"
+			);
+		
+		foreach($scripts as $sc){
+			
+			if(
+				preg_match(
+					'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
+					$sc["innerHTML"],
+					$matches
+				)
+			){
+				
+				sleep(10); // gotta sleep to not trigger captcha. blame google, not me
+				
+				return [
+					"sei" => $matches[1],
+					"timeout" => (int)$matches[2] // set to 10000 usually
+				];
+			}
+		}
+		
+		return false; // no need to bypass
 	}
 }