remove parseInt on password env

Reviewed-on: #94

.gitignore

@@ -26,3 +26,4 @@ data/captcha/minecraft/
 !banner/*default*
 scraper/curlie.html
 icons/*
+node_modules/

Dockerfile

@@ -1,8 +1,17 @@
+FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate
+
 FROM alpine:3.21
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg
+RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates 
+
+COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
+COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
+
+ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so
+ENV CURL_IMPERSONATE=ff117
+ENV CURL_IMPERSONATE_HEADERS=no
 
 COPY . .
 

README.md

@@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | libreY      | araa      | hearch.co         |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
+|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
+| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -31,25 +31,29 @@ tl;dr 4get is the best way to browse for shit.
 
 # Supported websites
 
-| Web        | Images       | Videos       | News       | Music      | Autocompleter |
-|------------|--------------|--------------|------------|------------|---------------|
-| DuckDuckGo | DuckDuckGo   | YouTube      | DuckDuckGo | Soundcloud | Brave         |
-| Brave      | Brave        | Sepia Search | Brave      |            | DuckDuckGo    |
-| Yandex     | Yandex       | DuckDuckGo   | Google     |            | Yandex        |
-| Google     | Google       | Brave        | Startpage  |            | Google        |
-| Startpage  | Startpage    | Yandex       | Qwant      |            | Startpage     |
-| Qwant      | Qwant        | Google       | Mojeek     |            | Kagi          |
-| Ghostery   | Yep          | Startpage    | Baidu      |            | Qwant         |
-| Yep        | Baidu        | Qwant        |            |            | Ghostery      |
-| Greppr     | Pinterest    | Baidu        |            |            | Yep           |
-| Crowdview  | 500px        | Coc Coc      |            |            | Marginalia    |
-| Mwmbl      | VSCO         |              |            |            | YouTube       |
-| Mojeek     | Imgur        |              |            |            | Soundcloud    |
-| Baidu      | FindThatMeme |              |            |            |               |
-| Coc Coc    |              |              |            |            |               |
-| Marginalia |              |              |            |            |               |
-| wiby       |              |              |            |            |               |
-| Curlie     |              |              |            |            |               |
+| web          | images       | videos       | news         | music      | autocomplete |
+|--------------|--------------|--------------|--------------|------------|--------------|
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
+| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
+| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
+| Qwant        | Qwant        | Startpage    |              |            | Yep          |
+| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
+| Yep          | Solofield    | Baidu        |              |            | YouTube      |
+| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
+| Mojeek       | Cara         | Solofield    |              |            |              |
+| Baidu        | Flickr       |              |              |            |              |
+| Coc Coc      | Pexels       |              |              |            |              |
+| Solofield    | Pixabay      |              |              |            |              |
+| Marginalia   | Unsplash     |              |              |            |              |
+| wiby         | 500px        |              |              |            |              |
+|              | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.
@@ -58,4 +62,4 @@ Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/"
 Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPL
+AGPLv3-only

audio/linear.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/sc.php

@@ -1,224 +0,0 @@
-<?php
-
-new sc_audio();
-
-class sc_audio{
-	
-	public function __construct(){
-		
-		include "../data/config.php";
-		include "../lib/curlproxy.php";
-		$this->proxy = new proxy();
-		
-		if(isset($_GET["u"])){
-			
-			/*
-				we're now proxying audio
-			*/
-			$viewkey = $_GET["u"];
-			
-			if(!isset($_GET["r"])){
-				
-				$this->do404("Ranges(r) are missing");
-			}
-			
-			$ranges = explode(",", $_GET["r"]);
-			
-			// sanitize ranges
-			foreach($ranges as &$range){
-				
-				if(!is_numeric($range)){
-					
-					$this->do404("Invalid range specified");
-				}
-				
-				$range = (int)$range;
-			}
-			
-			// sort ranges (just to make sure)
-			sort($ranges);
-			
-			// convert ranges to pairs
-			$last = -1;
-			foreach($ranges as &$r){
-				
-				$tmp = $r;
-				$r = [$last + 1, $r];
-				
-				$last = $tmp;
-			}
-			
-			$browser_headers = getallheaders();
-			
-			// get the requested range from client
-			$client_range = 0;
-			foreach($browser_headers as $key => $value){
-				
-				if(strtolower($key) == "range"){
-					
-					preg_match(
-						'/bytes=([0-9]+)/',
-						$value,
-						$client_regex
-					);
-					
-					if(isset($client_regex[1])){
-						
-						$client_range = (int)$client_regex[1];
-					}else{
-						
-						$client_range = 0;
-					}
-					break;
-				}
-			}
-			
-			if(
-				$client_range < 0 ||
-				$client_range > $ranges[count($ranges) - 1][1]
-			){
-				
-				// range is not satisfiable
-				http_response_code(416);
-				header("Content-Type: text/plain");
-				die();
-			}
-			
-			$rng = null;
-			for($i=0; $i<count($ranges); $i++){
-				
-				if($ranges[$i][0] <= $client_range){
-					
-					$rng = $ranges[$i];
-				}
-			}
-			
-			// proxy data!
-			http_response_code(206); // partial content
-			header("Accept-Ranges: bytes");
-			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
-			
-			$viewkey =
-				preg_replace(
-					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-					'/media/$1/' . $rng[0] . '/' . $rng[1],
-					$viewkey
-				);
-			
-			try{
-				
-				$this->proxy->stream_linear_audio(
-					$viewkey
-				);
-			}catch(Exception $error){
-				
-				$this->do404("Could not read stream");
-			}
-			
-			die();
-		}
-		
-		/*
-			redirect user to correct resource
-			we need to scrape and store the byte positions in the result URL
-		*/
-		if(!isset($_GET["s"])){
-			
-			$this->do404("The URL(s) parameter is missing");
-		}
-		
-		$viewkey = $_GET["s"];
-		
-		if(
-			preg_match(
-				'/soundcloud\.com$/',
-				parse_url($viewkey, PHP_URL_HOST)
-			) === false
-		){
-			
-			$this->do404("This endpoint can only be used for soundcloud streams");
-		}
-		
-		try{
-			
-			$json = $this->proxy->get($viewkey)["body"];
-		}catch(Exception $error){
-			
-			$this->do404("Curl error: " . $error->getMessage());
-		}
-		
-		$json = json_decode($json, true);
-		
-		if(!isset($json["url"])){
-			
-			$this->do404("Could not get URL from JSON");
-		}
-		
-		$viewkey = $json["url"];
-		
-		$m3u8 = $this->proxy->get($viewkey)["body"];
-		
-		$m3u8 = explode("\n", $m3u8);
-		
-		$lineout = null;
-		$streampos_arr = [];
-		foreach($m3u8 as $line){
-			
-			$line = trim($line);
-			if($line[0] == "#"){
-				
-				continue;
-			}
-			
-			if($lineout === null){
-				$lineout = $line;
-			}
-			
-			preg_match(
-				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
-				$line,
-				$matches
-			);
-			
-			if(isset($matches[0])){
-				
-				$streampos_arr[] = [
-					(int)$matches[1],
-					(int)$matches[2]
-				];
-			}
-		}
-		
-		if($lineout === null){
-			
-			$this->do404("Could not get stream URL");
-		}
-		
-		$lineout =
-			preg_replace(
-				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-				'/media/$1/0/0',
-				$lineout
-			);
-		
-		$streampos = [];
-		
-		foreach($streampos_arr as $pos){
-			
-			$streampos[] = $pos[1];
-		}
-		
-		$streampos = implode(",", $streampos);
-		
-		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
-		header("Accept-Ranges: bytes");
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

audio/seekable.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/spotify.php

@@ -1,214 +0,0 @@
-<?php
-
-include "../data/config.php";
-new spotify();
-
-class spotify{
-	
-	public function __construct(){
-		
-		include "../lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		if(
-			!isset($_GET["s"]) ||
-			!preg_match(
-				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
-				$_GET["s"],
-				$matches
-			)
-		){
-			
-			$this->do404("The track ID(s) parameter is missing or invalid");
-		}
-		
-		try{
-			
-			if($matches[1] == "episode"){
-				
-				$uri = "show";
-			}else{
-				
-				$uri = $matches[1];
-			}
-			
-			$embed =
-				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
-		}catch(Exception $error){
-			
-			$this->do404("Failed to fetch embed data");
-		}
-		
-		$this->fuckhtml->load($embed);
-		
-		$json =
-			$this->fuckhtml
-			->getElementById(
-				"__NEXT_DATA__",
-				"script"
-			);
-		
-		if($json === null){
-			
-			$this->do404("Failed to extract JSON");
-		}
-		
-		$json =
-			json_decode($json["innerHTML"], true);
-		
-		if($json === null){
-			
-			$this->do404("Failed to decode JSON");
-		}
-		
-		switch($matches[1]){
-			
-			case "track":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["audioPreview"]
-						["url"]
-					)
-				){
-					
-					header("Content-type: audio/mpeg");
-					header(
-						"Location: /audio/linear?s=" .
-						urlencode(
-							$json
-							["props"]
-							["pageProps"]
-							["state"]
-							["data"]
-							["entity"]
-							["audioPreview"]
-							["url"]
-						)
-					);
-				}else{
-					
-					$this->do404("Could not extract playback URL");
-				}
-				break;
-			
-			case "episode":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["id"]
-					)
-				){
-					
-					try{
-						$json =
-							$this->get(
-								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
-								$json
-								["props"]
-								["pageProps"]
-								["state"]
-								["data"]
-								["entity"]
-								["id"] .
-								"/com.widevine.alpha"
-							);
-					}catch(Exception $error){
-						
-						$this->do404("Failed to fetch audio resource");
-					}
-					
-					$json = json_decode($json, true);
-					
-					if($json === null){
-						
-						$this->do404("Failed to decode audio resource JSON");
-					}
-					
-					if(
-						isset($json["passthrough"]) &&
-						$json["passthrough"] == "ALLOWED" &&
-						isset($json["passthroughUrl"])
-					){
-						
-						header(
-							"Location:" .
-							"/audio/linear.php?s=" .
-							urlencode(
-								str_replace(
-									"http://",
-									"https://",
-									$json["passthroughUrl"]
-								)
-							)
-						);
-					}else{
-						
-						$this->do404("Failed to find passthroughUrl");
-					}
-					
-				}else{
-					
-					$this->do404("Failed to find episode ID");
-				}
-				break;
-			}
-	}
-	
-	private function get($url){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

data/api_keys/yep.txt

@@ -0,0 +1 @@
+# Paste Yep API keys here

data/config.php

@@ -23,6 +23,22 @@ class config{
 	// Enable the API?
 	const API_ENABLED = true;
 	
+	//
+	// 4play settings
+	//
+	// Enable 4get's /api/v2/provide_sesh endpoint
+	const FPLAY_ENABLE_API = false;
+	
+	// 4play password. Please set this to something secure if you use this extension.
+	// This password is used to POST sessions to /api/v2/provide_sesh
+	// It's also used as the external endpoint password.
+	const FPLAY_PASSWORD = "cnc";
+	
+	// Endpoint 4get hits to render webpages
+	// Eg: null=no endpoint provided
+	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
+	const FPLAY_EXTERNAL_ENDPOINT = null;
+	
 	//
 	// BOT PROTECTION
 	//
@@ -118,10 +134,10 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:149.0) Gecko/20100101 Firefox/149.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
 	
 	// User agent to use with 4get-friendly APIs
-	const USER_AGENT_FRIENDLY = "4get-scrapist";
+	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
@@ -131,7 +147,6 @@ class config{
 	const PROXY_YAHOO = false;
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
-	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
@@ -146,6 +161,7 @@ class config{
 	const PROXY_SWISSCOWS = false;
 	const PROXY_SPOTIFY = false;
 	const PROXY_SOLOFIELD = false;
+	const PROXY_PURILI = false;
 	const PROXY_WIBY = false;
 	const PROXY_CURLIE = false;
 	const PROXY_YT = false; // youtube
@@ -155,7 +171,6 @@ class config{
 	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
-	const PROXY_SANKAKUCOMPLEX = false;
 	const PROXY_FLICKR = false;
 	const PROXY_PIXABAY = false;
 	const PROXY_UNSPLASH = false;
@@ -164,8 +179,6 @@ class config{
 	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
-	const PROXY_GREPPR = false;
-	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
@@ -173,6 +186,11 @@ class config{
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
+	const PROXY_SAFEBOORU = false;
+	const PROXY_KONACHAN = false;
+	const PROXY_YANDERE = false;
+	const PROXY_TBIB = false;
+	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
@@ -185,4 +203,7 @@ class config{
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
+	
+	// Yep
+	const YEP_USE_API = false;
 }

docs/apache2-example.md

@@ -49,9 +49,13 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	</Directory>
 
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 

docs/apache2.md

@@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 	
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```

docs/configure.md

@@ -79,3 +79,87 @@ Done! The scraper you chose should now be using the rotating proxies. When askin
 
 ## Important!
 If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
+
+# 4play setup
+4play is a Firefox extension I wrote that lets you control a browser from 4get. When you do a search, 4get sends a query to a nodeJS http server that communicates to the browser via websockets. The browser then reports back various payloads used for scraping. It's very similar to tools like puppeteer or playwright, except it uses websockets to communicate with the browser instead of the browser's debugger. This is needed to prevent `navigator.webdriver` from being set to `true` (also avoids a fuckton of other botchecks, that's just the tip of the iceberg).
+
+## Requirements
+- A dedicated laptop or computer running a full desktop environment (like Windows, Mac, or Linux DEs)
+- Something to make the computer think it's connected to a screen. You need it, otherwise you'll fallback to software rendering and sites can detect that. You can use:
+	- Your laptop's screen
+	- A turned-on screen connected via HDMI
+	- An EDID adapter to simulate a display (no affiliate links: [HDMI](https://www.aliexpress.com/item/1005011668390064.html), [DisplayPort](https://www.aliexpress.com/item/1005011659561729.html))
+- Firefox ESR or the latest version of Firefox
+- The [4play extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) installed
+
+## Installation
+Install the dependencies on the machine you wish to run 4play on. In a sane setup, you usually want 4play to run on a dedicated user-grade machine. For the sake of simplicity, this machine will be referred as the "render server".
+
+```sh
+git clone https://git.lolcat.ca/lolcat/4get
+cd 4get/extra/4play
+
+# install nodejs 26.x or later if it's not already installed
+
+npm install @lawlers/4play
+sudo npm install -g nodemon
+
+# edit line 7 where it says 'var password = "cnc";'. Use a strong password.
+nano page-render.js
+
+# test the server
+nodemon page-render.js
+```
+
+I recommend setting up a systemd service to auto-start the 4play server:
+```sh
+sudo nano /etc/systemd/system/fplay.service
+```
+
+Paste this in (make sure to update `User` & `WorkingDirectory`)
+```sh
+[Unit]
+Description=4play
+After=network.target
+
+[Service]
+Type=simple
+User=will
+WorkingDirectory=/home/will/Desktop/4get/extra/4play/
+ExecStart=nodemon /home/will/Desktop/4get/extra/4play/page-render.js
+Restart=always
+Environment=NODE_ENV=production
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Now, enable the service
+```sh
+sudo systemctl enable fplay
+sudo service fplay start
+sudo service fplay status # check the status. You should see it running
+```
+
+This should expose a webservice located at `http://localhost:3000`. I highly recommend you put this endpoint behind apache2 or nginx to add SSL, if the render server sits on a different network.
+
+## Setting up Firefox
+- Install Firefox. You can go for the ESR version or the latest version. You'll figure it out.
+- Head over to [this page](https://addons.mozilla.org/en-US/firefox/addon/4play/) or search "4play" on the Firefox extension store and install the extension.
+- Click the settings cog next to the extension name. Enable "Run in Private Windows" and "Allow automatic updates"
+- Click the extension in the top bar. Modify the `ws://localhost:3030/cnc` URL: replace the `cnc` path with the password you set earlier for the server.
+- Use a timeout value of `30000` to avoid getting throttled by Firefox during reconnections.
+
+If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
+
+## Setting up 4get
+Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
+
+```php
+const FPLAY_PASSWORD = "cnc"; # Change this to your password
+const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.com/api/example";, with the quotes and semicolon
+```
+
+**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
+
+If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.

docs/develop.md

@@ -0,0 +1,7 @@
+# Develop
+
+Start the PHP webserver like this, assuming you have curl-impersonate installed already
+
+```sh
+LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
+```

docs/docker.md

@@ -150,3 +150,7 @@ services:
 ##### Tor
 
 You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
+
+##### 4play
+
+You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)

docs/docker_4play.md

@@ -0,0 +1,48 @@
+### Running 4get and 4play via docker guide
+
+This setup will run both 4get and the 4play sidecar 
+
+# Docker compose 
+
+```
+# docker-compose.yaml
+
+services:
+  fourget:
+    image: luuul/4get:latest
+    restart: unless-stopped
+    environment:
+      - FOURGET_PROTO=http
+      - FOURGET_SERVER_NAME=4get.ca
+      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
+      - FOURGET_FPLAY_PASSWORD=cnc
+      - FOURGET_FPLAY_TIMEOUT=30000
+      - FOURGET_FPLAY_ENABLE_API=true
+    ports:
+      - "80:80"
+  fourplay:
+    image: luuul/4get-4play:latest
+    environment:
+      - FOURPLAY_PORT=3030
+      - FOURPLAY_PASSWORD=cnc
+      - FOURPLAY_COMMAND_TIMEOUT=5000
+    ports:
+      - "3000:3000"
+      - "3030:3030"
+
+```
+
+There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
+
+# Steps
+
+1. start the services with
+
+```
+docker compose up -d
+
+```
+
+2. configure firefox
+
+folllow the steps in [configure](./configure.md) for firefox setup

extra/4play/Dockerfile

@@ -0,0 +1,19 @@
+FROM alpine:3.21
+WORKDIR /app
+
+RUN apk update && apk upgrade
+RUN apk add nodejs npm
+
+COPY package*.json ./
+
+RUN npm ci --only=production && npm cache clean --force
+
+COPY . .
+
+EXPOSE 3030
+
+ENTRYPOINT  ["node"]
+CMD ["page-render.js"]
+
+
+

extra/4play/docker-compose.yaml

@@ -0,0 +1,24 @@
+# docker-compose.yaml
+
+services:
+  fourget:
+    image: luuul/4get:latest
+    restart: unless-stopped
+    environment:
+      - FOURGET_PROTO=http
+      - FOURGET_SERVER_NAME=4get.ca
+      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
+      - FOURGET_FPLAY_PASSWORD=cnc
+      - FOURGET_FPLAY_TIMEOUT=30000
+      - FOURGET_FPLAY_ENABLE_API=true
+    ports:
+      - "80:80"
+  fourplay:
+    image: luuul/4get-4play:latest
+    environment:
+      - FOURPLAY_PORT=3030
+      - FOURPLAY_PASSWORD=cnc
+      - FOURPLAY_COMMAND_TIMEOUT=5000
+    ports:
+      - "3000:3000"
+      - "3030:3030"

extra/4play/package-lock.json

@@ -0,0 +1,47 @@
+{
+  "name": "4play",
+  "version": "1.0.0",
+  "lockfileVersion": 3,
+  "requires": true,
+  "packages": {
+    "": {
+      "name": "4play",
+      "version": "1.0.0",
+      "license": "AGPL-3.0-only",
+      "dependencies": {
+        "@lawlers/4play": "^1.2.5",
+        "ws": "^8.21.2"
+      }
+    },
+    "node_modules/@lawlers/4play": {
+      "version": "1.2.5",
+      "resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
+      "integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
+      "license": "AGPL-3.0-only",
+      "dependencies": {
+        "ws": "^8.21.1"
+      }
+    },
+    "node_modules/ws": {
+      "version": "8.21.2",
+      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
+      "integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10.0.0"
+      },
+      "peerDependencies": {
+        "bufferutil": "^4.0.1",
+        "utf-8-validate": ">=5.0.2"
+      },
+      "peerDependenciesMeta": {
+        "bufferutil": {
+          "optional": true
+        },
+        "utf-8-validate": {
+          "optional": true
+        }
+      }
+    }
+  }
+}

extra/4play/package.json

@@ -0,0 +1,20 @@
+{
+  "name": "4play",
+  "version": "1.0.0",
+  "description": "",
+  "repository": {
+    "type": "git",
+    "url": "https://git.lolcat.ca/lolcat/4get/"
+  },
+  "license": "AGPL-3.0-only",
+  "author": "lolcat",
+  "type": "commonjs",
+  "main": "page-render.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "dependencies": {
+    "@lawlers/4play": "^1.2.5",
+    "ws": "^8.21.2"
+  }
+}

extra/4play/page-render.js

@@ -0,0 +1,337 @@
+const fplay = require("@lawlers/4play");
+const http = require("http");
+
+// 4play settings
+var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
+var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
+var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
+
+// ==================================================
+
+// webserver
+
+var global_ws = null;
+const webserver_port = 3000;
+
+var callback_q = new Map();
+
+const server = http.createServer(async function(req, res){
+	
+	const parsed = new URL(req.url, "http://localhost");
+	
+	const url = parsed.searchParams.get("url");
+	const pw = parsed.searchParams.get("pw");
+	var proxy = parsed.searchParams.get("proxy");
+	const url_match = parsed.searchParams.get("url_match");
+	const content_match = parsed.searchParams.get("content_match");
+	const fail_url_match = parsed.searchParams.get("fail_url_match");
+	const fail_content_match = parsed.searchParams.get("fail_content_match");
+	
+	var container = parsed.searchParams.get("container");
+	
+	if(fplay.browser_connected === false){
+				
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("No browser available to render the motherfucking page");
+		return;
+	}
+	
+	var errors = [];
+	
+	if(
+		pw === null ||
+		pw != password
+	){
+		
+		errors.push("pw");
+	}
+	
+	if(url === null){ errors.push("url"); }
+	if(proxy === null){ errors.push("proxy"); }
+	
+	if(errors.length !== 0){
+		
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("The following parameters are missing or invalid: " + errors.join(", "));
+		return;
+	}
+	
+	console.log("Rendering " + url);
+	
+	if(
+		container === null ||
+		await fplay.container_exists(global_ws, container) === false
+	){
+		
+		container = await fplay.container_create(global_ws);
+		
+		proxy = get_proxy(proxy);
+		
+		// assign proxy
+		if(proxy.config !== false){
+			
+			await fplay.container_attach_proxy(
+				global_ws,
+				container,
+				proxy.config
+			);
+		}
+	}else{
+		
+		proxy = {raw: proxy};
+	}
+	
+	// open tab
+	const newtab = await fplay.tab_open(global_ws, url, false, container);
+	
+	if(newtab === false){
+		
+		// failed to open tab
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": proxy.raw,
+				"X-Container-ID": typeof container.id == "undefined" ? container : container.id
+			}
+		);
+		
+		res.end("Failed to open tab");
+		return;
+	}
+		
+	// callback will be handled in "web_response"
+	callback_q.set(
+		newtab.id,
+		{
+			http_res: res,
+			proxy: proxy,
+			container: container,
+			url_match: url_match,
+			content_match: content_match,
+			fail_url_match: fail_url_match,
+			fail_content_match: fail_content_match
+		}
+	);
+	
+	// close tab in 30 seconds
+	await fplay.wait_random(15000, 35000);
+	await fplay.tab_close(global_ws, newtab);
+});
+
+server.listen(webserver_port, function(){
+	
+	console.log("http server running");
+});
+
+function get_proxy(proxy_line){
+	
+	parts = proxy_line.split(":");
+	// format: type:address:port:username:password
+	// PHP explode(":", $ip, 5) splits into max 5 parts
+	var type = parts[0];
+	const address = parts[1];
+	const port = parts[2];
+	const username = parts[3];
+	const password = parts[4];
+	
+	var proxy_dns = true;
+	
+	switch(type){
+		case "raw_ip":
+		case "ip":
+			// no proxy, use direct connection
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+			break;
+		
+		case "socks5":
+			type = "socks";
+			proxy_dns = false;
+			break;
+			
+		case "socks4a":
+			type = "socks4";
+			break;
+			
+		case "socks4":
+			proxy_dns = false;
+			break;
+			
+		case "socks5_hostname":
+		case "socks5h":
+		case "socks5a":
+			type = "socks";
+			break;
+		
+		case "http":
+		case "https":
+			// do nothing
+			break;
+		
+		default:
+			// invalid proxy provided
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+	}
+	
+	return {
+		config: {
+			type: type,
+			host: address,
+			port: port,
+			username: username,
+			password: password,
+			proxyDNS: proxy_dns
+		},
+		raw: proxy_line
+	};
+}
+
+fplay.event.on("server_ready", function(){
+	
+	console.log("listening on port " + port + " (timeout=" + timeout + ")");
+});
+
+fplay.event.on("browser_connect", async function(ws){
+	
+	global_ws = ws;
+	
+	const ua = await fplay.get_ua(ws);
+	console.log("New browser instance connected: " + ua);
+	
+	await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+	
+	fplay.web_response_whitelist(ws, ["main_frame"]);
+	
+	// clean up
+	const blanktab = await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+});
+
+fplay.event.on("web_response", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	var body = page.body.toString();
+	
+	var url_match = callback_data.url_match !== null ? page.url.match(new RegExp(callback_data.url_match, "i")) : null;
+	var content_match = callback_data.content_match !== null ? body.match(new RegExp(callback_data.content_match, "i")) : null;
+	var fail_url_match = callback_data.fail_url_match !== null ? page.url.match(new RegExp(callback_data.fail_url_match, "i")) : null;
+	var fail_content_match = callback_data.fail_content_match !== null ? body.match(new RegExp(callback_data.fail_content_match, "i")) : null;
+	
+	// handle negative response
+	if(
+		(
+			callback_data.fail_url_match !== null ||
+			callback_data.fail_content_match !== null
+		) &&
+		(
+			fail_url_match !== null ||
+			fail_content_match !== null
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)");
+		
+		// cleanup browser + callback
+		await fplay.tab_close(global_ws, page.id);
+		await fplay.container_delete(global_ws, callback_data.container);
+		callback_q.delete(page.id);
+		
+		return;
+	}
+	
+	// handle positive response
+	if(
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match === null
+		) ||
+		(
+			callback_data.url_match !== null &&
+			callback_data.content_match !== null &&
+			url_match &&
+			content_match
+		) ||
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match !== null &&
+			content_match
+		) ||
+		(
+			callback_data.url_match !== null &&
+			callback_data.content_match === null &&
+			url_match
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			200,
+			{
+				"Content-Type": "text/plain;charset=UTF-8",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end(body);
+		
+		callback_q.delete(page.id); // cleanup callback only
+		return;
+	}
+});
+
+fplay.event.on("dom_load_fail", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	callback_q.delete(page.id);
+	
+	callback_data.http_res.writeHead(
+		400,
+		{
+			"Content-Type": "text/plain",
+			"X-Proxy": callback_data.proxy.raw,
+			"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+		}
+	);
+	
+	callback_data.http_res.end("Network error (" + page.error + ")");
+		
+	// cleanup browser
+	await fplay.tab_close(global_ws, page.id);
+	await fplay.container_delete(global_ws, callback_data.container);
+});
+
+fplay.init(port, password, timeout);

lib/backend.php

@@ -4,6 +4,9 @@ class backend{
 	public function __construct($scraper){
 		
 		$this->scraper = $scraper;
+		$this->db = null;
+		$this->db_path_prefix = "data/sessions/";
+		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
 	}
 	
 	/*
@@ -208,4 +211,127 @@ class backend{
 			$payload[1] // proxy
 		];
 	}
+	
+	//
+	// Sessions
+	//
+	public function session_init_db($force_init = false){
+		
+		if(
+			$force_init ||
+			$this->db === null
+		){
+			
+			$this->db = new SQLite3($this->db_path);
+			
+			// create table if it doesnt exist	
+			$this->db->exec(
+				"CREATE TABLE IF NOT EXISTS sessions ( " .
+					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
+					"created_at INTEGER NOT NULL, " .
+					"proxy TEXT NOT NULL, " .
+					"headers_json TEXT NOT NULL" .
+				")"
+			);
+			
+			return true;
+		}
+		
+		return false;
+	}
+	
+	public function session_count(){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"SELECT COUNT(*) as count FROM sessions"
+		);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return (int)($row["count"] ?? 0);
+	}
+	
+	public function session_write($proxy, $headers){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"INSERT INTO sessions ( " .
+				"created_at, " .
+				"proxy, " .
+				"headers_json " .
+			") VALUES ( " .
+				":created_at, " .
+				":proxy, " .
+				":headers_json" .
+			")"
+		);
+		
+		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
+		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
+		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
+		
+		$stmt->execute();
+		
+		return $this->db->lastInsertRowID();
+	}
+	
+	public function session_get(){
+		
+		$count = $this->session_count();
+		
+		if($count === 0){
+			
+			return false;
+		}
+		
+		$offset = apcu_inc("d." . $this->scraper) % $count;
+		
+		$stmt = $this->db->prepare(
+			"SELECT * " .
+			"FROM sessions " .
+			"ORDER BY id ASC " .
+			"LIMIT 1 OFFSET :offset"
+		);
+		
+		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return [
+			"id" => $row["id"],
+			"created_at" => $row["created_at"],
+			"proxy" => $row["proxy"],
+			"headers" => json_decode($row["headers_json"], true)
+		];
+	}
+	
+	public function session_destroy_list(){
+		
+		if(
+			!preg_match(
+				'/^[A-Za-z0-9-]+$/',
+				$this->scraper
+			)
+		){
+			
+			// path traversal
+			return false;
+		}
+		
+		if(
+			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
+			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
+		){
+			
+			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
+			return true;
+		}
+		
+		return false;
+	}
 }

lib/bot_protection.php

@@ -133,6 +133,9 @@ class bot_protection{
 			$answers[] = $regex;
 		}
 		
+		// dedup
+		$answers = array_unique($answers);
+		
 		if(
 			!$invalid &&
 			$key !== false // has captcha been gen'd?

lib/frontend.php

@@ -2,6 +2,52 @@
 
 class frontend{
 	
+	public function validateurl($url, $net_validate = false){
+		
+		$url_parts = parse_url($url);
+		
+		// check if required parts are there
+		if(
+			!isset($url_parts["scheme"]) ||
+			!(
+				$url_parts["scheme"] == "http" ||
+				$url_parts["scheme"] == "https"
+			) ||
+			!isset($url_parts["host"])
+		){
+			return false;
+		}
+		
+		if($net_validate){
+			$ip = 
+				str_replace(
+					["[", "]"], // handle ipv6
+					"",
+					$url_parts["host"]
+				);
+			
+			// if its not an IP
+			if(!filter_var($ip, FILTER_VALIDATE_IP)){
+				
+				// resolve domain's IP
+				$ip = gethostbyname($url_parts["host"] . ".");
+			}
+			
+			// check if its localhost
+			if(
+				filter_var(
+					$ip,
+					FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+				) === false
+			){
+				
+				return false;
+			}
+		}
+		
+		return true;
+	}
+	
 	public function load($template, $replacements = []){
 		
 		$replacements["server_name"] = htmlspecialchars(config::SERVER_NAME);
@@ -588,6 +634,7 @@ class frontend{
 				$filters["scraper"] = [
 					"display" => "Scraper",
 					"option" => [
+						//"fget" => "fget",
 						"ddg" => "DuckDuckGo",
 						//"yahoo" => "Yahoo!",
 						"brave" => "Brave",
@@ -598,18 +645,16 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"ghostery" => "Ghostery",
 						"yep" => "Yep",
-						"greppr" => "Greppr",
-						"crowdview" => "Crowdview",
 						"mwmbl" => "Mwmbl",
 						"mojeek" => "Mojeek",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"coccoc" => "Cốc Cốc",
 						"solofield" => "Solofield",
 						"marginalia" => "Marginalia",
-						"wiby" => "wiby",
-						"curlie" => "Curlie"
+						"purili" => "Purili",
+						"wiby" => "wiby"
 					]
 				];
 				break;
@@ -622,11 +667,12 @@ class frontend{
 						"yandex" => "Yandex",
 						"brave" => "Brave",
 						"google" => "Google",
+						"google_api" => "Google API",
 						"google_cse" => "Google CSE",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"yep" => "Yep",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"solofield" => "Solofield",
 						"pinterest" => "Pinterest",
@@ -638,8 +684,7 @@ class frontend{
 						"fivehpx" => "500px",
 						"vsco" => "VSCO",
 						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme",
-						//"sankakucomplex" => "SankakuComplex"
+						"ftm" => "FindThatMeme"
 					]
 				];
 				break;
@@ -661,8 +706,10 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"coccoc" => "Cốc Cốc",
+						"purili" => "Purili",
 						"solofield" => "Solofield"
 					]
 				];
@@ -678,7 +725,6 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"yep" => "Yep",
 						"mojeek" => "Mojeek",
 						"baidu" => "Baidu"
 					]
@@ -695,6 +741,22 @@ class frontend{
 					]
 				];
 				break;
+			
+			case "booru":
+				$filters["scraper"] = [
+					"display" => "Scraper",
+					"option" => [
+						"safebooru" => "Safebooru",
+						"konachan" => "Konachan",
+						"tbib" => "The Big Imageboard",
+						"gelbooru" => "Gelbooru",
+						"yandere" => "Yande.re",
+						"tbib" => "The Big Imageboard",
+						"sankakucomplex" => "SankakuComplex",
+						"soybooru" => "SoyBooru"
+					]
+				];
+				break;
 		}
 		
 		// get scraper name from user input, or default out to preferred scraper
@@ -871,6 +933,7 @@ class frontend{
 		
 		$html = null;
 		
+		//foreach(["web", "images", "videos", "news", "music", "booru"] as $type){
 		foreach(["web", "images", "videos", "news", "music"] as $type){
 			
 			$html .= '<a href="/' . $type . '?s=' . urlencode($query);

lib/fuckhtml.php

@@ -553,28 +553,21 @@ class fuckhtml{
 				
 				case "\"":
 				case "'":
-					if(
-						$i !== 0 && // only check if a quote could be there
-						(
-							(
-								$json[$i - 1] === "\\" &&
-								(
-									$i === 2 ||
-									$json[$i - 2] === "\\"
-								)
-							) ||
-							$json[$i - 1] !== "\\"
-						)
-					){
-						// found a non-escaped quote
+					// count preceding backslashes
+					$bsCount = 0;
+					$j = $i - 1;
 
+					while($j >= 0 && $json[$j] === "\\"){
+						$bsCount++;
+						$j--;
+					}
+
+					// quote is NOT escaped if even number of backslashes
+					if($bsCount % 2 === 0){
 						if($in_quote === null){
-							
 							// open quote
 							$in_quote = $json[$i];
-							
 						}elseif($in_quote === $json[$i]){
-							
 							// close quote
 							$in_quote = null;
 						}

scraper/baidu.php

@@ -666,14 +666,24 @@ class baidu{
 			//
 			// Don't parse as a search result if it's a card
 			//
-			$card =
+			$tpl_probe =
 				$this->fuckhtml
-				->getElementsByClassName(
-					"cosc-card",
-					$div
+				->getElementsByAttributeValue(
+					"tpl",
+					"www_index",
+					[$datafield]
 				);
 			
-			if(count($card) !== 0){
+			if(
+				count($tpl_probe) === 0 &&
+				count(
+					$this->fuckhtml
+					->getElementsByClassName(
+						"cosc-card",
+						$div
+					)
+				) !== 0
+			){
 				
 				//
 				// Parse chinese youtube shorts
@@ -870,10 +880,13 @@ class baidu{
 			// class:FYB_RD -> News garbage, IGNORE
 			
 			$result =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"result",
-					[$datafield]
+				array_merge(
+					$tpl_probe,
+					$this->fuckhtml
+					->getElementsByClassName(
+						"result",
+						[$datafield]
+					)
 				);
 			
 			if(count($result) !== 0){
@@ -885,7 +898,7 @@ class baidu{
 				$title =
 					$this->fuckhtml
 					->getElementsByClassName(
-						"sc-link",
+						"cosc-title-a",
 						"a"
 					);
 				
@@ -905,8 +918,9 @@ class baidu{
 				
 				$description =
 					$this->fuckhtml
-					->getElementsByClassName(
-						"c-color",
+					->getElementsByAttributeValue(
+						"data-sanssr-cmpt",
+						"card/www-summary-1",
 						$div
 					);
 				

scraper/brave.php

@@ -347,11 +347,8 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
-		/*
-		$handle = fopen("scraper/brave.html", "r");
-		$html = fread($handle, filesize("scraper/brave.html"));
-		fclose($handle);*/
 		
+		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(
@@ -391,14 +388,14 @@ class brave{
 		*/
 		$nextpage =
 			$this->fuckhtml
-			->getElementById(
+			->getElementsByClassName(
 				"pagination",
 				"div"
 			);
 		
-		if($nextpage){
+		if(count($nextpage) !== 0){
 			
-			$this->fuckhtml->load($nextpage);
+			$this->fuckhtml->load($nextpage[0]);
 			
 			$nextpage =
 				$this->fuckhtml
@@ -1776,14 +1773,14 @@ class brave{
 		
 		$nextpage =
 			$this->fuckhtml
-			->getElementById(
+			->getElementsByClassName(
 				"pagination",
 				"div"
 			);
 		
-		if($nextpage){
+		if(count($nextpage) !== 0){
 			
-			$this->fuckhtml->load($nextpage);
+			$this->fuckhtml->load($nextpage[0]);
 			
 			$nextpage =
 				$this->fuckhtml

scraper/crowdview.php

@@ -1,145 +0,0 @@
-<?php
-
-class crowdview{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("crowdview");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$proxy = $this->backend->get_ip();
-		
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://crowdview-next-js.onrender.com/api/search-v3",
-				[
-					"query" => $search
-				]
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$json = json_decode($json, true);
-		
-		if($json === NULL){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		foreach($json["results"] as $item){
-			
-			$description = explode("<b>", $item["snippet"], 2);
-			
-			$out["web"][] = [
-				"title" => $this->sanitize($item["title"]),
-				"description" => $this->sanitize($description[1]),
-				"url" => $item["link"],
-				"date" => strtotime($description[0]),
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function sanitize($html){
-		
-		return
-			trim(
-				$this->fuckhtml
-				->getTextContent(
-					html_entity_decode(
-						$html
-					)
-				),
-				". "
-			);
-	}
-}

scraper/curlie.php

@@ -1,309 +0,0 @@
-<?php
-
-class curlie{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("curlie");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"lang" => [
-				"display" => "Language",
-				"option" => [
-					"any" => "Any language",
-					"en" => "English",
-					"de" => "German",
-					"fr" => "French",
-					"ja" => "Japanese",
-					"it" => "Italian",
-					"es" => "Spanish",
-					"ru" => "Russian",
-					"nl" => "Dutch",
-					"pl" => "Polish",
-					"tr" => "Turkish",
-					"da" => "Danish",
-					"sv" => "Swedish",
-					"no" => "Norwegian",
-					"is" => "Icelandic",
-					"fo" => "Faroese",
-					"fi" => "Finnish",
-					"et" => "Estonian",
-					"lt" => "Lithuanian",
-					"lv" => "Latvian",
-					"cy" => "Welsh",
-					"ga" => "Irish",
-					"gd" => "Scottish Gaelic",
-					"br" => "Breton",
-					"fy" => "Frisian",
-					"frr" => "North Frisian",
-					"gem" => "Saterland Frisian",
-					"lb" => "Luxembourgish",
-					"rm" => "Romansh",
-					"pt" => "Portuguese",
-					"ca" => "Catalan",
-					"gl" => "Galician",
-					"eu" => "Basque",
-					"ast" => "Asturian",
-					"an" => "Aragonese",
-					"fur" => "Friulan",
-					"sc" => "Sardinian",
-					"scn" => "Sicilian",
-					"oc" => "Occitan",
-					"be" => "Belarusian",
-					"cs" => "Czech",
-					"hu" => "Hungarian",
-					"sk" => "Slovak",
-					"uk" => "Ukrainian",
-					"csb" => "Kashubian",
-					"tt" => "Tatar",
-					"ba" => "Bashkir",
-					"os" => "Ossetian",
-					"sl" => "Slovene",
-					"sr" => "Serbian",
-					"hr" => "Croatian",
-					"bs" => "Bosnian",
-					"bg" => "Bulgarian",
-					"sq" => "Albanian",
-					"ro" => "Romanian",
-					"mk" => "Macedonian",
-					"el" => "Greek",
-					"iw" => "Hebrew",
-					"fa" => "Persian",
-					"ar" => "Arabic",
-					"ku" => "Kurdish",
-					"az" => "Azerbaijani",
-					"hy" => "Armenian",
-					"af" => "Afrikaans",
-					"sw" => "Kiswahili",
-					"uz" => "Uzbek",
-					"kk" => "Kazakh",
-					"ky" => "Kyrgyz",
-					"tg" => "Tajik",
-					"tk" => "Turkmen",
-					"ug" => "Uyghurche",
-					"hi" => "Hindi",
-					"si" => "Sinhalese",
-					"gu" => "Gujarati",
-					"ur" => "Urdu",
-					"mr" => "Marathi",
-					"pa" => "Punjabi",
-					"bn" => "Bengali",
-					"ta" => "Tamil",
-					"te" => "Telugu",
-					"kn" => "Kannada",
-					"zh_CN" => "Chinese Simplified",
-					"zh_TW" => "Chinese Traditional",
-					"ko" => "Korean",
-					"cfr" => "Taiwanese",
-					"th" => "Thai",
-					"vi" => "Vietnamese",
-					"in" => "Indonesian",
-					"ms" => "Malay",
-					"tl" => "Tagalog",
-					"eo" => "Esperanto",
-					"ia" => "Interlingua",
-					"la" => "Latin"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/" . $query,
-					[]
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-			
-		}else{
-			$proxy = $this->backend->get_ip();
-			
-			$query = [
-				"q" => $get["s"],
-				"start" => 0,
-				"stime" => 92452189 // ?
-			];
-			
-			if($get["lang"] !== "any"){
-				
-				$query["lang"] = $get["lang"];
-			}
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/search",
-					$query
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		$nextpage =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"next-page",
-				"a"
-			);
-		
-		if(count($nextpage) !== 0){
-			
-			$nextpage =
-				$this->backend->store(
-					$nextpage[0]["attributes"]["href"],
-					"web",
-					$proxy
-				);
-		}else{
-			
-			$nextpage = null;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => $nextpage,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"site-item",
-				"div"
-			);
-		
-		foreach($items as $item){
-			
-			$this->fuckhtml->load($item);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByAttributeValue(
-					"target",
-					"_blank",
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName("site-descr");
-			
-			if(count($description) !== 0){
-				
-				$description =
-					$this->fuckhtml
-					->getTextContent(
-						$description[0]
-					);
-			}else{
-				
-				$description = null;
-			}
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-}

scraper/ddg.php

@@ -54,8 +54,8 @@ class ddg{
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
 					"Accept: */*",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip",
+					"Accept-Language: en-US,en;q=0.9",
+					"Accept-Encoding: gzip, deflate, br, zstd",
 					"Referer: https://duckduckgo.com/",
 					"DNT: 1",
 					"Sec-GPC: 1",
@@ -492,6 +492,17 @@ class ddg{
 				"form"
 			);
 		
+		foreach($forms as $form){
+			
+			if(
+				isset($form["attributes"]["id"]) &&
+				$form["attributes"]["id"] == "challenge-form"
+			){
+			
+				throw new Exception("DuckDuckGo returned an image captcha");
+			}
+		}
+		
 		foreach(array_reverse($forms) as $form){
 			
 			$this->fuckhtml->load($form);
@@ -909,112 +920,6 @@ class ddg{
 			){
 				
 				throw new Exception("DuckDuckGo returned a JSA challenge");
-				
-				// get JSA initial token
-				if(
-					!preg_match(
-						'/let jsa *= *([0-9]+)/',
-						$js,
-						$jsa
-					)
-				){
-					
-					$jsa = 0;
-				}else{
-					
-					$jsa = (int)$jsa[1];
-				}
-				
-				// get function bodies
-				preg_match_all(
-					'/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU',
-					$js,
-					$functions
-				);
-				
-				$parsed_functions = [];
-				
-				for($i=0; $i<count($functions[0]); $i++){
-					
-					$functions[2][$i] = trim($functions[2][$i]);
-					
-					if(
-						preg_match(
-							'/return num *\* *([0-9]+)/i',
-							$functions[2][$i],
-							$num
-						)
-					){
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "multiplication",
-							"num" => (int)$num[1]
-						];
-						continue;
-					}
-					
-					if(
-						preg_match(
-							'/innerHTML *= *`([^`]+)`/i',
-							$functions[2][$i],
-							$challenge
-						)
-					){
-						
-						$challenge[1] =
-							preg_replace(
-								'/<\/(br)>/',
-								'<$1>',
-								$challenge[1]
-							);
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "challenge",
-							"text" => $challenge[1]
-						];
-					}
-				}
-				
-				// get function call order
-				preg_match_all(
-					'/jsa *= *([A-Za-z0-9]+)\(jsa\)/i',
-					$js,
-					$call_order
-				);
-				
-				foreach($call_order[1] as $order){
-					
-					if(!isset($parsed_functions[$order])){
-						
-						throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function");
-					}
-					
-					if($parsed_functions[$order]["type"] == "multiplication"){
-						
-						$jsa = $jsa * $parsed_functions[$order]["num"];
-						continue;
-					}
-					
-					if($parsed_functions[$order]["type"] == "challenge"){
-						
-						// @TODO get parsed length
-						//$parsed_functions[$order]["text"]
-						
-						$jsa = $jsa + strlen($parsed_functions[$order]["text"]);
-					}
-				}
-				
-				try{
-					$js = $this->get(
-						$proxy,
-						"https://links.duckduckgo.com" . $challenge_url[1] . $jsa,
-						[],
-						ddg::req_xhr
-					);
-				}catch(Exception $error){
-					
-					throw new Exception("Failed to get challenged d.js");
-				}
 			}
 			
 			//
@@ -1858,7 +1763,7 @@ class ddg{
 					if(count($answer) !== 0){
 						
 						$out["answer"][] = [
-							"title" => ucfirst($word),
+							"title" => ucfirst(rawurldecode($word)),
 							"description" => $answer,
 							"url" => "https://www.wordnik.com/words/" . $word,
 							"thumb" => null,

scraper/ghostery.php

@@ -1,320 +0,0 @@
-<?php
-
-class ghostery{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("ghostery");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"any" => "All regions",
-					"AR" => "Argentina",
-					"AU" => "Australia",
-					"AT" => "Austria",
-					"BE" => "Belgium",
-					"BR" => "Brazil",
-					"CA" => "Canada",
-					"CL" => "Chile",
-					"DK" => "Denmark",
-					"FI" => "Finland",
-					"FR" => "France",
-					"DE" => "Germany",
-					"HK" => "Hong Kong",
-					"IN" => "India",
-					"ID" => "Indonesia",
-					"IT" => "Italy",
-					"JP" => "Japan",
-					"KR" => "Korea",
-					"MY" => "Malaysia",
-					"MX" => "Mexico",
-					"NL" => "Netherlands",
-					"NZ" => "New Zealand",
-					"NO" => "Norway",
-					"CN" => "People's Republic of China",
-					"PL" => "Poland",
-					"PT" => "Portugal",
-					"PH" => "Republic of the Philippines",
-					"RU" => "Russia",
-					"SA" => "Saudi Arabia",
-					"ZA" => "South Africa",
-					"ES" => "Spain",
-					"SE" => "Sweden",
-					"CH" => "Switzerland",
-					"TW" => "Taiwan",
-					"TR" => "Turkey",
-					"GB" => "United Kingdom",
-					"US" => "United States"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $country){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"Referer: https://ghosterysearch.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: same-origin",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"]
-		);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			parse_str($query, $query);
-			
-			// country
-			$country = $query["c"];
-			unset($query["c"]);
-			
-			$query = http_build_query($query);
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search?" . $query,
-						[],
-						$country
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search",
-						[
-							"q" => $get["s"]
-						],
-						$get["country"]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$this->fuckhtml->load($html);
-		
-		$results_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"results",
-				"section"
-			);
-		
-		if(count($results_wrapper) === 0){
-			
-			throw new Exception("Failed to grep result section");
-		}
-		
-		$this->fuckhtml->load($results_wrapper[0]);
-		
-		// get search results
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"li"
-			);
-		
-		if(count($results) === 0){
-			
-			return $out;
-		}
-		
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"url",
-					"a"
-				);
-			
-			if(count($a) === 0){
-				
-				continue;
-			}
-			
-			$a = $a[0];
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"h2"
-							)[0]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"p"
-							)[0]
-						)
-					),
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-						["attributes"]
-						["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		// get pagination token
-		$pagination_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"div"
-			);
-		
-		if(count($pagination_wrapper) !== 0){
-			
-			// found next page!
-			$this->fuckhtml->load($pagination_wrapper[0]);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				);
-			
-			if(count($a) !== 0){
-				
-				$q =
-					parse_url(
-						$this->fuckhtml
-						->getTextContent(
-							$a[count($a) - 1]
-							["attributes"]
-							["href"]
-						),
-						PHP_URL_QUERY
-					);
-				
-				$out["npt"] =
-					$this->backend
-					->store(
-						$q . "&c=" . $get["country"],
-						"web",
-						$proxy
-					);
-			}
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/google_api.php

@@ -264,6 +264,25 @@ class google_api{
 					"yes" => "Yes", // safe=active
 					"no" => "No" // safe=off
 				]
+			],
+			"sort" => [ // sort
+				"display" => "Sort by",
+				"option" => [
+					"any" => "Any order",
+					"date:d" => "Oldest",
+					"date:a" => "Newest"
+				]
+			],
+			"newer" => [
+				"display" => "Newer than",
+				"option" => "_DATE"
+			],			
+			"rm_dupes" => [ // filter
+				"display" => "Remove duplicates",
+				"option" => [
+					"yes" => "Yes", // 1
+					"no" => "No" // 0
+				]
 			]
 		];
 		
@@ -313,109 +332,29 @@ class google_api{
 								"zh-CN" => "Chinese (Simplified)",
 								"zh-TW" => "Chinese (Traditional)"
 							]
-						],
-						"sort" => [
-							"display" => "Sort by",
-							"option" => [
-								"any" => "Any order",
-								"date:d" => "Oldest",
-								"date:a" => "Newest"
-							]
-						],
-						"newer" => [
-							"display" => "Newer than",
-							"option" => "_DATE"
-						],
-						"rm_dupes" => [
-							"display" => "Remove duplicates",
-							"option" => [
-								"yes" => "Yes",
-								"no" => "No"
-							]
 						]
 					]
 				);
 				break;
-			/*
+			
 			case "images":
 				return array_merge(
 					$base,
 					[
-						"time" => [ // tbs=qdr:<time>
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"d" => "Past 24 hours",
-								"w" => "Past week",
-								"m" => "Past month",
-								"y" => "Past year"
-							]
-						],
-						"size" => [ // imgsz
+						"size" => [ // imgSize
 							"display" => "Size",
 							"option" => [
 								"any" => "Any size",
-								"l" => "Large",
-								"m" => "Medium",
-								"i" => "Icon",
-								"qsvga" => "Larger than 400x300",
-								"vga" => "Larger than 640x480",
-								"svga" => "Larger than 800x600",
-								"xga" => "Larger than 1024x768",
-								"2mp" => "Larger than 2MP",
-								"4mp" => "Larger than 4MP",
-								"6mp" => "Larger than 6MP",
-								"8mp" => "Larger than 8MP",
-								"10mp" => "Larger than 10MP",
-								"12mp" => "Larger than 12MP",
-								"15mp" => "Larger than 15MP",
-								"20mp" => "Larger than 20MP",
-								"40mp" => "Larger than 40MP",
-								"70mp" => "Larger than 70MP"
+								"icon" => "Icon",
+								"small" => "Small",
+								"medium" => "Medium",
+								"large" => "Large",
+								"xlarge" => "X-Large",
+								"xxlarge" => "XX-Large",
+								"huge" => "Huge"
 							]
 						],
-						"ratio" => [ // imgar
-							"display" => "Aspect ratio",
-							"option" => [
-								"any" => "Any ratio",
-								"t|xt" => "Tall",
-								"s" => "Square",
-								"w" => "Wide",
-								"xw" => "Panoramic"
-							]
-						],
-						"color" => [ // imgc
-							"display" => "Color",
-							"option" => [
-								"any" => "Any color",
-								"color" => "Full color",
-								"bnw" => "Black & white",
-								"trans" => "Transparent",
-								// from here, imgcolor
-								"red" => "Red",
-								"orange" => "Orange",
-								"yellow" => "Yellow",
-								"green" => "Green",
-								"teal" => "Teal",
-								"blue" => "Blue",
-								"purple" => "Purple",
-								"pink" => "Pink",
-								"white" => "White",
-								"gray" => "Gray",
-								"black" => "Black",
-								"brown" => "Brown"
-							]
-						],
-						"type" => [ // tbs=itp:<type>
-							"display" => "Type",
-							"option" => [
-								"any" => "Any type",
-								"clipart" => "Clip Art",
-								"lineart" => "Line Drawing",
-								"animated" => "Animated"
-							]
-						],
-						"format" => [ // as_filetype
+						"format" => [ // fileType
 							"display" => "Format",
 							"option" => [
 								"any" => "Any format",
@@ -429,17 +368,55 @@ class google_api{
 								"craw" => "RAW"
 							]
 						],
-						"rights" => [ // tbs=sur:<rights>
+						"color" => [
+							"display" => "Color",
+							"option" => [
+								"any" => "Any color",
+								
+								"color" => "Full color", // imgColorType
+								"mono" => "Black & White",
+								"trans" => "Transparent background",
+								
+								"red" => "Red", // imgDominantColor
+								"orange" => "Orange",
+								"yellow" => "Yellow",
+								"green" => "Green",
+								"teal" => "Teal",
+								"blue" => "Blue",
+								"purple" => "Purple",
+								"pink" => "Pink",
+								"white" => "White",
+								"gray" => "Gray",
+								"black" => "Black",
+								"brown" => "Brown"
+							]
+						],
+						"type" => [ // imgType
+							"display" => "Type",
+							"option" => [
+								"any" => "Any type",
+								"clipart" => "Clip Art",
+								"face" => "Faces",
+								"lineart" => "Line Drawing",
+								"stock" => "Stock photos",
+								"photo" => "Photos",
+								"animated" => "Animated",
+							]
+						],
+						"rights" => [ // rights
 							"display" => "Usage rights",
 							"option" => [
 								"any" => "Any license",
-								"cl" => "Creative Commons licenses",
-								"ol" => "Commercial & other licenses"
+								"cc_publicdomain" => "Public domain",
+								"cc_attribute" => "Attribution required",
+								"cc_sharealike" => "Sharealike",
+								"cc_noncommercial" => "Non-commercial use only",
+								"cc_nonderived" => "Original works"
 							]
 						]
 					]
 				);
-				break;*/
+				break;
 		}
 	}
 	
@@ -485,6 +462,7 @@ class google_api{
 		return $data;
 	}
 	
+	
 	public function web($get){
 		
 		// rotate proxy + key on EVERY request
@@ -731,6 +709,160 @@ class google_api{
 		return $out;
 	}
 	
+	
+	public function image($get){
+		
+		// rotate proxy + key on EVERY request
+		$keydata = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($keydata["increment"]);
+		
+		if($get["npt"]){
+			
+			// $p is never used
+			[$params, $p] = $this->backend->get(
+				$get["npt"],
+				"web"
+			);
+			
+			$params = json_decode($params, true);
+			
+			$params["key"] = $keydata["key"];
+			
+		}else{
+			
+			//$json = file_get_contents("scraper/google.json");
+			$params = [
+				"q" => $get["s"],
+				"cx" => config::GOOGLE_CX_ENDPOINT,
+				"num" => 10,
+				"start" => 1,
+				"searchType" => "image",
+				"key" => $keydata["key"]
+			];
+			
+			//
+			// parse filters
+			//
+			if($get["newer"] !== false){
+				
+				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
+			}
+			
+			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
+			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
+			
+			if($get["nsfw"] == "yes"){
+				
+				$params["safe"] = "off";
+			}else{
+				
+				$params["safe"] = "active";
+			}
+			
+			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
+			
+			// image filters
+			if($get["size"] != "any"){ $params["imgSize"] = $get["size"]; }
+			if($get["format"] != "any"){ $params["fileType"] = $get["format"]; }
+			
+			switch($get["color"]){
+				
+				case "any":
+					break;
+				
+				case "color":
+				case "mono":
+				case "trans":
+					$params["imgColorType"] = $get["color"];
+					break;
+				
+				default:
+					$params["imgDominantColor"] = $get["color"];
+					break;
+			}
+			
+			if($get["type"] != "any"){ $params["imgType"] = $get["type"]; }
+			if($get["rights"] != "any"){ $params["rights"] = $get["rights"]; }
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://www.googleapis.com/customsearch/v1",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(isset($json["error"]["message"])){
+			
+			throw new Exception(
+				"API returned an error: " .
+				$json["error"]["message"] .
+				" (key #" . $keydata["increment"] . ")"
+			);
+		}
+		
+		if(!isset($json["items"])){
+			
+			// google just doesnt return items when theres no results
+			return $out;
+		}
+		
+		foreach($json["items"] as $image){
+			
+			$out["image"][] = [
+				"title" => $this->titledots($image["title"]),
+				"source" => [
+					[
+						"url" => $image["link"],
+						"width" => (int)$image["image"]["width"],
+						"height" => (int)$image["image"]["height"]
+					],
+					[
+						"url" => $image["image"]["thumbnailLink"],
+						"width" => (int)$image["image"]["thumbnailWidth"],
+						"height" => (int)$image["image"]["thumbnailHeight"]
+					]
+				],
+				"url" => $image["image"]["contextLink"]
+			];
+		}
+		
+		// get npt
+		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
+			
+			unset($params["key"]);
+			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	
 	private function titledots($title){
 		
 		return trim($title, " .\t\n\r\0\x0B…");

scraper/google_cse.php

@@ -910,11 +910,10 @@ class google_cse{
 			"image" => []
 		];
 		
-		// detect next page
-		if(
-			isset($json["cursor"]["isExactTotalResults"]) || // detects last page
-			!isset($json["cursor"]["pages"]) // detects no results on page
-		){
+		// A response can contain a final page of image results while also marking
+		// the cursor as exact/finished.  Parse those results before deciding
+		// whether another page token should be offered.
+		if(!isset($json["results"]) || !is_array($json["results"])){
 			
 			return $out;
 		}
@@ -939,6 +938,16 @@ class google_cse{
 			];
 		}
 		
+		// Only decide whether a following page exists after preserving every
+		// result returned on this page.
+		if(
+			isset($json["cursor"]["isExactTotalResults"]) || // detects last page
+			!isset($json["cursor"]["pages"]) // detects no results on page
+		){
+			
+			return $out;
+		}
+		
 		// get next page
 		$out["npt"] =			
 			$this->backend->store(

scraper/greppr.php

@@ -1,452 +0,0 @@
-<?php
-// greppr dev probably monitors 4get code, lol
-// hello greppr dude, add an API you moron
-
-class greppr{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("greppr");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = [], $cookies = [], $post = false){
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		$cookie = [];
-		foreach($cookies as $k => $v){
-			
-			$cookie[] = "{$k}={$v}";
-		}
-		
-		$cookie = implode("; ", $cookie);
-		
-		if($post === false){
-						
-			if($get !== []){
-				$get = http_build_query($get);
-				$url .= "?" . $get;
-			}
-			
-			if($cookie == ""){
-				
-				curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-					["User-Agent: " . config::USER_AGENT,
-					"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip",
-					"DNT: 1",
-					"Connection: keep-alive",
-					"Upgrade-Insecure-Requests: 1",
-					"Sec-Fetch-Dest: document",
-					"Sec-Fetch-Mode: navigate",
-					"Sec-Fetch-Site: none",
-					"Sec-Fetch-User: ?1"]
-				);
-			}else{
-				
-				curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-					["User-Agent: " . config::USER_AGENT,
-					"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip, deflate, br, zstd",
-					"DNT: 1",
-					"Sec-GPC: 1",
-					"Connection: keep-alive",
-					"Referer: https://greppr.org/search",
-					"Cookie: {$cookie}",
-					"Upgrade-Insecure-Requests: 1",
-					"Sec-Fetch-Dest: document",
-					"Sec-Fetch-Mode: navigate",
-					"Sec-Fetch-Site: same-origin",
-					"Sec-Fetch-User: ?1",
-					"Priority: u=0, i"]
-				);
-			}
-		}else{
-			
-			$get = http_build_query($get);
-			
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"Content-Type: application/x-www-form-urlencoded",
-				"Content-Length: " . strlen($get),
-				"Origin: https://greppr.org",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Referer: https://greppr.org/",
-				"Cookie: {$cookie}",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i"]
-			);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$headers = [];
-		
-		curl_setopt(
-			$curlproc,
-			CURLOPT_HEADERFUNCTION,
-			function($curlproc, $header) use (&$headers){
-				
-				$len = strlen($header);
-				$header = explode(':', $header, 2);
-				
-				if(count($header) < 2){
-					
-					// ignore invalid headers
-					return $len;
-				}
-				
-				$headers[strtolower(trim($header[0]))][] = trim($header[1]);
-
-				return $len;
-			}
-		);
-				
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		
-		return [
-			"headers" => $headers,
-			"data" => $data
-		];
-	}
-	
-	public function web($get, $first_attempt = true){
-		
-		if($get["npt"]){
-			
-			[$q, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			$tokens = json_decode($q, true);
-			
-			//
-			// Get paginated page
-			//
-			try{
-			
-				$html = $this->get(
-					$proxy,
-					"https://greppr.org" . $tokens["get"],
-					[],
-					$tokens["cookies"],
-					false
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			//
-			// get token
-			//
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://greppr.org",
-						[],
-						[],
-						false
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch homepage");
-			}
-			
-			//
-			// Parse token
-			//
-			$this->fuckhtml->load($html["data"]);
-		
-			$tokens = [
-				"req" => null,
-				"data" => null,
-				"cookies" => null
-			];
-			
-			$inputs =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"input"
-				);
-				
-			foreach($inputs as $input){
-				
-				if(!isset($input["attributes"]["name"])){
-					
-					continue;
-				}
-				
-				if(
-					isset($input["attributes"]["value"]) &&
-					!empty($input["attributes"]["value"])
-				){
-					
-					$tokens
-						["data"]
-						[$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["name"]
-						)] =
-						$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["value"]
-						);
-				}else{
-					
-					$tokens["req"] =
-						$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["name"]
-						);
-				}
-			}
-			
-			if($tokens["req"] === null){
-				
-				throw new Exception("Failed to get request ID");
-			}
-			
-			if(isset($html["headers"]["set-cookie"])){
-				
-				foreach($html["headers"]["set-cookie"] as $cookie){
-					
-					if(
-						preg_match(
-							'/([^=]+)=([^;]+)/',
-							$cookie,
-							$matches
-						)
-					){
-						
-						$tokens["cookies"][$matches[1]] = $matches[2];
-					}
-				}
-			}
-			
-			//
-			// Get initial search page
-			//
-			$tokens_req = $tokens["data"];
-			$tokens_req[$tokens["req"]] = $search;
-			
-			try{
-				
-				$html = $this->get(
-					$proxy,
-					"https://greppr.org/search",
-					$tokens_req,
-					$tokens["cookies"],
-					true
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		//$html = file_get_contents("scraper/greppr.html");
-		//$this->fuckhtml->load($html);
-		$this->fuckhtml->load($html["data"]);
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		// get results for later
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"div"
-			);
-		
-		// check for next page
-		$next_elem =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"ul"
-			);
-		
-		if(count($next_elem) !== 0){
-			
-			$this->fuckhtml->load($next_elem[0]);
-			
-			$as =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"page-link",
-					"a"
-				);
-			
-			$break = false;
-			foreach($as as $a){
-				
-				if($break === true){
-					
-					$out["npt"] =
-						$this->backend->store(
-							json_encode([
-								"get" =>
-									$this->fuckhtml
-									->getTextContent(
-										$a["attributes"]["href"]
-									),
-								"cookies" => $tokens["cookies"]
-							]),
-							"web",
-							$proxy
-						);
-					break;
-				}
-				
-				if($a["attributes"]["href"] == "#"){
-					
-					$break = true;
-				}
-			}
-		}
-		
-		// scrape results
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"highlightedDesc",
-					"p"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->limitstrlen(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$date =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"p"
-				);
-			
-			$date =
-				strtotime(
-					explode(
-						":",
-						$this->fuckhtml
-						->getTextContent(
-							$date[count($date) - 1]["innerHTML"]
-						)
-					)[1]
-				);
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["innerHTML"]
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => $date,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function limitstrlen($text){
-		
-		return explode("\n", wordwrap($text, 300, "\n"))[0];
-	}
-}

scraper/marginalia.php

@@ -289,8 +289,18 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
+			if(strpos($json, "<title>Bad Gateway (502)</title>")){
+				
+				throw new Exception("Marginalia returned a 502 error");
+			}
+			
 			$json = json_decode($json, true);
 			
+			if($json === null){
+				
+				throw new Exception("Failed to decode JSON");
+			}
+			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -490,6 +500,37 @@ class marginalia{
 			return $this->web($get, $redirect);
 		}
 		
+		// detect internal error
+		$infobox =
+			$this->fuckhtml
+			->getElementsByClassName("infobox");
+		
+		if(count($infobox) !== 0){
+			
+			foreach($infobox as $i){
+				
+				$this->fuckhtml->load($i);
+				
+				$h2 =
+					$this->fuckhtml
+					->getElementsByTagName("h2");
+				
+				if(
+					count($h2) !== 0 &&
+					$this->fuckhtml
+					->getTextContent(
+						$h2[0]
+					) == "Internal error"
+				){
+					
+					throw new Exception("Marginalia returned an internal server error");
+				}
+			}
+						
+			// reset
+			$this->fuckhtml->load($html);
+		}
+		
 		$sections =
 			$this->fuckhtml
 			->getElementsByClassName(

scraper/mojeek.php

@@ -340,7 +340,7 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],
+					],/*
 					"region" => [
 						"display" => "Region",
 						"option" => [
@@ -350,7 +350,7 @@ class mojeek{
 							"fr" => "France",
 							"uk" => "United Kingdom"
 						]
-					],
+					],*/
 					"domain" => [
 						"display" => "Results per domain",
 						"option" => [
@@ -376,7 +376,7 @@ class mojeek{
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Language: en-US,en;q=0.9",
 			"Accept-Encoding: gzip",
 			"DNT: 1",
 			"Connection: keep-alive",
@@ -447,7 +447,7 @@ class mojeek{
 			$proxy = $this->backend->get_ip();
 			$lang = $get["lang"];
 			$country = $get["country"];
-			$region = $get["region"];
+			//$region = $get["region"];
 			$domain = $get["domain"];
 			$focus = $get["focus"];
 			
@@ -458,7 +458,7 @@ class mojeek{
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
 				//"dlen" => 511, // max length of description
-				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -479,11 +479,11 @@ class mojeek{
 				
 				$params["lb"] = $lang;
 			}
-			
+			/*
 			if($region != "any"){
 				
 				$params["reg"] = $region;
-			}
+			}*/
 			
 			if($domain != "1"){
 				
@@ -503,6 +503,13 @@ class mojeek{
 			}
 		}
 		
+		if(strlen($html) === 0){
+			
+			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
+		}
+		
+		$this->fuckhtml->load($html);
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -519,8 +526,6 @@ class mojeek{
 			"related" => []
 		];
 		
-		$this->fuckhtml->load($html);
-		
 		$this->detect_block();
 		
 		$results =
@@ -1168,6 +1173,18 @@ class mojeek{
 	
 	private function detect_block(){
 		
+		$captcha =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"captcha-box",
+				"div"
+			);
+		
+		if(count($captcha) !== 0){
+			
+			throw new Exception("Mojeek returned a captcha");
+		}
+		
 		$title =
 			$this->fuckhtml
 			->getElementsByTagName(

scraper/pinterest.php

@@ -297,6 +297,14 @@ class pinterest{
 			throw new Exception("Failed to decode JSON");
 		}
 		
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
@@ -426,7 +434,7 @@ class pinterest{
 							]
 						],
 						"url" =>
-							$item["link"] === null ?
+							!isset($item["link"]) ?
 							"https://ca.pinterest.com/pin/" . $item["id"] :
 							$item["link"]
 					];

scraper/purili.php

@@ -0,0 +1,350 @@
+<?php
+
+class purili{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("purili");
+	}
+	
+	public function getfilters($page){
+		
+		switch($page){
+			case "web":
+				return [
+					"nsfw" => [
+						"display" => "NSFW",
+						"option" => [
+							"yes" => "Yes",
+							"maybe" => "Maybe",
+							"no" => "No"
+						]
+					],
+					"language" => [
+						"display" => "Language",
+						"option" => [
+							"any" => "Any language",
+							"en" => "English only",
+							"nl" => "Dutch preferred",
+							"de" => "German preferred",
+							"fr" => "French preferred",
+							"es" => "Spanish preferred",
+							"it" => "Italian preferred",
+							"pt" => "Portuguese preferred"
+						]
+					]
+				];
+				break;
+			
+			case "videos":
+				return [];
+				break;
+		}
+	}
+	
+	private function get($proxy, $url, $get = [], $nsfw = null, $language = null){
+		
+		$curlproc = curl_init();
+		
+		switch($nsfw){
+			
+			case "yes":
+			case null:
+				$get["fv"] = "0";
+				$nsfw = "off";
+				break;
+				
+			case "maybe":
+				$get["fv"] = "1";
+				$nsfw = "moderate";
+				break;
+				
+			case "no":
+				$get["fv"] = "2";
+				$nsfw = "strict";
+				break;
+		}
+		
+		switch($language){
+			
+			case "any":
+			case null:
+				$language_cookie = "";
+				break;
+			
+			default:
+				$language_cookie = "; purili-language={$language}";
+				break;
+		}
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Cookie: purili-safesearch={$nsfw}{$language_cookie}",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=4"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function web($get){
+		
+		if($get["npt"]){
+			
+			[$q, $proxy] = $this->backend->get($get["npt"], "web");
+			$q = json_decode($q, true);
+			
+			$nsfw = $q["nsfw"];
+			unset($q["nsfw"]);
+			
+			$language = $q["language"];
+			unset($q["language"]);
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$q = [
+				"q" => $search,
+				"page" => 1,
+				// added at $this->get()
+				// "fv" => 0
+			];
+			
+			$nsfw = $get["nsfw"];
+			$language = $get["language"];
+		}
+		
+		// https://puri.li/api/search?q=4get&page=1&fv=0
+		try{
+			$json = $this->get(
+				$proxy,
+				"https://puri.li/api/search",
+				$q,
+				$nsfw,
+				$language
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to hit API");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Failed to access results array");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		foreach($json["results"] as $result){
+			
+			$out["web"][] = [
+				"title" => $result["title"],
+				"description" => $result["description"],
+				"url" => $result["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		// get next page
+		if($json["hasNext"]){
+			
+			$q["page"]++;
+			$q["nsfw"] = $nsfw;
+			$q["language"] = $language;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($q),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	public function video($get){
+		
+		
+		if($get["npt"]){
+			
+			[$q, $proxy] = $this->backend->get($get["npt"], "videos");
+			$q = json_decode($q, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$q = [
+				"q" => $search,
+				"offset" => 0,
+				"limit" => 12
+				// added at $this->get()
+				// "fv" => 0
+			];
+		}
+		
+		// https://puri.li/api/videos/search?q=higurashi&limit=12&fv=1
+		// hidden &offset parameter only accessible through the API
+		try{
+			$json = $this->get(
+				$proxy,
+				"https://puri.li/api/videos/search",
+				$q,
+				null,
+				null
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to hit API");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Failed to access results array");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		foreach($json["results"] as $result){
+			
+			if(isset($result["thumbnail"])){
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $result["thumbnail"]
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
+			}
+			
+			$out["video"][] = [
+				"title" => $result["title"],
+				"description" =>
+					str_replace(
+						"\n", " ",
+						mb_substr(
+							$result["description"], 0, 255, "utf-8"
+						)
+					),
+				"author" => [
+					"name" => null,
+					"url" => isset($result["channel_id"]) ? "https://www.youtube.com/channel/" . $result["channel_id"] : null,
+					"avatar" => null
+				],
+				"date" => strtotime($result["upload_date"]),
+				"duration" => $result["duration"],
+				"views" => $result["view_count"],
+				"thumb" => $thumb,
+				"url" => $result["url"]
+			];
+		}
+		
+		// get next page
+		if($json["limit"] + $json["offset"] < $json["total"]){
+			
+			$q["offset"] = $q["offset"] + $json["limit"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($q),
+					"videos",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+}

scraper/qwant.php

@@ -212,28 +212,58 @@ class qwant{
 		return $base;
 	}
 	
-	private function get($proxy, $url, $get = []){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Origin: https://www.qwant.com",
-			"Referer: https://www.qwant.com/",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"TE: trailers"
-		];
+	private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){
 		
 		$curlproc = curl_init();
 		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
+		if($is_post){
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Content-type: application/x-www-form-urlencoded",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache"
+			];
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get));
+		}else{
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: application/json, text/plain, */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				($cookie !== null ? "Cookie: {$cookie}" : ""),
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
+				"TE: trailers"
+			];
+						
+			if($get !== []){
+				$get = http_build_query($get);
+				$url .= "?" . $get;
+			}
 		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
@@ -271,6 +301,8 @@ class qwant{
 			[$params, $proxy] = $this->backend->get($get["npt"], "web");
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
 			
 		}else{
 			
@@ -297,7 +329,8 @@ class qwant{
 				"device" => "desktop",
 				"tgp" => 3,
 				"safesearch" => 0,
-				"displayed" => "true"
+				"displayed" => "true",
+				"llm" => "false"
 			];
 			
 			switch($get["nsfw"]){
@@ -306,18 +339,24 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		/*
 		$handle = fopen("scraper/qwant_web.json", "r");
 		$json = fread($handle, filesize("scraper/qwant_web.json"));
 		fclose($handle);*/
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json =
 				$this->get(
 					$proxy,
-					"https://fdn.qwant.com/v3/search/web",
-					$params
+					"https://api.qwant.com/v3/search/web",
+					$params,
+					false,
+					$cookie
 				);
 			
 		}catch(Exception $error){
@@ -353,6 +392,8 @@ class qwant{
 			"related" => []
 		];
 		
+		$this->detect_errors($json);
+		
 		if(
 			$json["status"] != "success" &&
 			$json["data"]["error_code"] === 5
@@ -362,8 +403,6 @@ class qwant{
 			return $out;
 		}
 		
-		$this->detect_errors($json);
-		
 		if(!isset($json["data"]["result"]["items"]["mainline"])){
 			
 			throw new Exception("Server did not return a result object");
@@ -436,6 +475,7 @@ class qwant{
 		if($json["data"]["result"]["lastPage"] === false){
 			
 			$params["offset"] = $params["offset"] + 10;
+			$params["cookie"] = $cookie;
 			
 			$out["npt"] =
 				$this->backend->store(
@@ -577,6 +617,9 @@ class qwant{
 				);
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
+			
 		}else{
 			
 			$search = $get["s"];
@@ -617,13 +660,19 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json = $this->get(
 				$proxy,
 				"https://api.qwant.com/v3/search/images",
 				$params,
+				false,
+				$cookie
 			);
 		}catch(Exception $err){
 			
@@ -1001,4 +1050,53 @@ class qwant{
 		
 		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
+	
+	private function fuck_datadome($proxy, $search){
+		
+		if($cookie === null){
+			// get datadome bullshit
+			// yeah go ahead and patch this you code monkey
+			
+			// ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result?
+			// i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum
+			// also, stop pretending you guys have your own index, this is clearly all bing results
+			try{
+				$datadome =
+					$this->get(
+						$proxy,
+						"https://dd.qwant.com/js/",
+						[
+							"jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA",
+							"eventCounters" => [],
+							"jsType" => "ch",
+							"cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe",
+							"ddk" => "78B13B7513D180B7AB6D6FF9EB0A51",
+							"Referer" => "https://www.qwant.com/?q={$search}&t=web",
+							"request" => "/?q={$search}&t=web",
+							"responsePage" => "origin",
+							"ddv" => "5.8.0"
+						],
+						true
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JS challenge token");
+			}
+			
+			$datadome = json_decode($datadome, true);
+			if($datadome === null){
+				
+				throw new Exception("Failed to decode JS challenge JSON");
+			}
+			
+			if(!isset($datadome["cookie"])){
+				
+				throw new Exception("Failed to get cookie from JS challenge endpoint");
+			}
+			
+			$cookie = explode(";", $datadome["cookie"])[0];
+		}
+		
+		return $cookie;
+	}
 }

scraper/startpage.php

@@ -408,6 +408,8 @@ class startpage{
 			//$html = file_get_contents("scraper/startpage.html");
 		}
 		
+		//echo htmlspecialchars($html);
+		
 		$this->detect_captcha($html);
 		
 		if(
@@ -564,12 +566,16 @@ class startpage{
 					break;
 				
 				case "spellsuggest-google":
-					$out["spelling"] =
-						[
-							"type" => "including",
-							"using" => $json["render"]["query"],
-							"correction" => $category["results"][0]["query"]
-						];
+					
+					if(isset($category["results"][0]["query"])){
+						
+						$out["spelling"] =
+							[
+								"type" => "including",
+								"using" => rawurldecode($json["render"]["query"]),
+								"correction" => html_entity_decode($category["results"][0]["query"])
+							];
+					}
 					break;
 				
 				case "dictionary-qi":
@@ -645,318 +651,6 @@ class startpage{
 			}
 		}
 		
-		// parse instant answers
-		if(
-			$get["extendedsearch"] == "yes" &&
-			$get_instant_answer === true
-		){
-			
-			// https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=BqZ3inqrAgF701&sr=1
-			try{
-				$post = [
-					"se" => "n0vze2y9dqwy",
-					"q" => $json["render"]["query"],
-					"results" => [], // populate
-					"enableKnowledgePanel" => true,
-					"enableMediaThumbBar" => false,
-					"enableSearchSuggestions" => false,
-					"enableTripadvisorProperties" => [],
-					"enableTripadvisorPlaces" => [],
-					"enableTripadvisorPlacesForLocations" => [],
-					"enableWebProducts" => false,
-					"tripadvisorPartnerId" => null,
-					"tripadvisorMapColorMode" => "light",
-					"tripadvisorDisablesKnowledgePanel" => false,
-					"instantAnswers" => [
-						"smartAnswers",
-						"youtube",
-						"tripadvisor"
-					],
-					"iaType" => null,
-					"forceEnhancedKnowledgePanel" => false,
-					"shoppingOnly" => false,
-					"allowAdultProducts" => true,
-					"lang" => "en",
-					"browserLang" => "en-US",
-					"browserTimezone" => "America/New_York",
-					"market" => null,
-					"userLocation" => null,
-					"userDate" => date("Y-m-d"),
-					"userAgentType" => "unknown"
-				];
-				
-				foreach($out["web"] as $result){
-					
-					$post["results"][] = [
-						"url" => $result["url"],
-						"title" => $result["title"]
-					];
-				}
-				
-				$post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
-				
-				$additional_data =
-					$this->get(
-						$proxy,
-						"https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
-						$post,
-						true,
-						true
-					);
-				
-				$additional_data = json_decode($additional_data, true);
-				
-				if($additional_data === null){
-					
-					throw new Exception("Failed to decode JSON"); // just break out, dont fail completely
-				}
-				
-				if(!isset($additional_data["knowledgePanel"])){
-					
-					throw new Exception("Response has missing data (knowledgePanel)");
-				}
-				
-				$additional_data = $additional_data["knowledgePanel"];
-				
-				$answer = [
-					"title" => $additional_data["meta"]["title"],
-					"description" => [
-						[
-							"type" => "quote",
-							"value" => $additional_data["meta"]["description"]
-						]
-					],
-					"url" => $additional_data["meta"]["origWikiUrl"],
-					"thumb" => $additional_data["meta"]["image"],
-					"table" => [],
-					"sublink" => []
-				];
-				
-				// parse html for instant answer
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				$div =
-					$this->fuckhtml
-					->getElementsByTagName(
-						"div"
-					);
-				
-				// get description
-				$description =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-short-extract sx-kp-short-extract-complete",
-						$div
-					);
-				
-				if(count($description) !== 0){
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							html_entity_decode(
-								$this->fuckhtml
-								->getTextContent(
-									$description[0]
-								)
-							)
-					];
-				}
-				
-				// get socials
-				$socials =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-wiki-social-link",
-						"a"
-					);
-				
-				foreach($socials as $social){
-					
-					$title =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["title"]
-						);
-					
-					$url =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["href"]
-						);
-					
-					switch($title){
-						
-						case "Official Website":
-							$title = "Website";
-							break;
-					}
-					
-					$answer["sublink"][$title] = $url;
-				}
-				
-				// get videos
-				$videos =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-video-grid-item",
-						$div
-					);
-				
-				foreach($videos as $video){
-					
-					$this->fuckhtml->load($video);
-					
-					$as =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"a"
-						);
-					
-					if(count($as) === 0){
-						
-						// ?? invalid
-						continue;
-					}
-					
-					$image =
-						$this->fuckhtml
-						->getElementsByAttributeName(
-							"data-sx-src",
-							"img"
-						);
-					
-					if(count($image) !== 0){
-						
-						$thumb = [
-							"ratio" => "16:9",
-							"url" =>
-								$this->fuckhtml
-								->getTextContent(
-									$image[0]["attributes"]["data-sx-src"]
-								)
-						];
-					}else{
-
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["video"][] = [
-						"title" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["title"]
-							),
-						"description" => null,
-						"date" => null,
-						"duration" => null,
-						"views" => null,
-						"thumb" => $thumb,
-						"url" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["href"]
-							)
-					];
-				}
-				
-				// reset
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				// get table elements
-				$table =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-infobox",
-						"table"
-					);
-				
-				if(count($table) !== 0){
-					
-					$trs =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"tr"
-						);
-					
-					foreach($trs as $tr){
-						
-						$this->fuckhtml->load($tr);
-						
-						// ok so startpage devs cant fucking code a table
-						// td = content
-						// th (AAAHH) = title
-						$tds =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"td"	
-							);
-						
-						$ths =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"th"
-							);
-						
-						if(
-							count($ths) === 1 &&
-							count($tds) === 1
-						){
-							
-							$title =
-								$this->fuckhtml
-								->getTextContent(
-									$ths[0]
-								);
-							
-							$description = [];
-							
-							$this->fuckhtml->load($tds[0]);
-							
-							$lis =
-								$this->fuckhtml
-								->getElementsByTagName(
-									"li"
-								);
-							
-							if(count($lis) !== 0){
-								
-								foreach($lis as $li){
-									
-									$description[] =
-										$this->fuckhtml
-										->getTextContent(
-											$li
-										);
-								}
-								
-								$description = implode(", ", $description);
-							}else{
-								
-								$description =
-									$this->fuckhtml
-									->getTextContent(
-										$tds[0]
-									);
-							}
-							
-							$answer["table"][$title] = $description;
-						}
-					}
-				}
-				
-				$out["answer"][] = $answer;
-				
-			}catch(Exception $error){
-				
-				// do nothing
-				//echo "error!";
-			}
-		}
-		
 		return $out;
 	}
 	
@@ -1413,27 +1107,22 @@ class startpage{
 			
 			if($page["name"] == "Next"){
 				
-				parse_str(
-					explode(
-						"?",
-						$page["url"],
-						2
-					)[1],
-					$str
-				);
-				
 				return
 					$this->backend->store(
 						http_build_query(
 							[
 								"lui" => "english",
 								"language" => "english",
-								"query" => $str["q"],
-								"cat" => $pagetype,
-								"sc" => $str["sc"],
+								"sc" => $json["render"]["search_sc"],
 								"t" => "device",
-								"segment" => "startpage.udog",
-								"page" => $str["page"]
+								"cat" => $pagetype,
+								"segment" => "organic",
+								"abd" => "0",
+								"abe" => "0",
+								"query" => rawurldecode($json["render"]["query"]),
+								"page" => $page["number"],
+								"qsr" => "all",
+								"qadf" => "none" // @ todo fix (??)
 							]
 						),
 						$pagetype,

scraper/vsco.php

@@ -30,19 +30,18 @@ class vsco{
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-Site: none",
 				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
-				"TE: trailers"]
+				"Priority: u=0, i"]
 			);
 		}else{
 			
@@ -73,8 +72,8 @@ class vsco{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// http1 bypass?!?!?!
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		$this->backend->assign_proxy($curlproc, $proxy);
 		

scraper/yandex.php

@@ -868,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
+			$get = json_decode($get, true);
 		}else{
 			
-			$search = $get["s"];
-			if(strlen($search) === 0){
+			if(strlen($get["s"]) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
-			
-			// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
-			// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
-			
-			$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-				"format" => "json",
-				"request" => json_encode([
-					"blocks" => [
-						(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
-						]
-					]
-				]),
-				"text" => $search
-			];
-			
-			if($duration != "any"){
-				
-				$params["duration"] = $duration;
-			}
-			
-			if($time != "any"){
-				
-				$params["within"] = $time;
-			}
 		}
+		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		// https://yandex.com/video/search
+		// ?text=skycamefalling
+		// &from=tabbar
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		
+		$params = [
+			"text" => $get["s"],
+			"from" => "tabbar",
+			"format" => "json",
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
+				"blocks" => [
+					(object)[
+						"block" => "video-app",
+						"params" => (object)[]
+					]
+				]
+			]),
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
+		];
+		
+		if(isset($get["p"])){
+			
+			$params["p"] = $get["p"];
+		}
+		
+		if($get["duration"] != "any"){
+			
+			$params["duration"] = $get["duration"];
+		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
+		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -996,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -1011,7 +959,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -1026,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
-		foreach($json["blocks"] as $block){
+		foreach($json["results"]["clips"]["items"] as $k => $data){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
+				}
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
+			
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
 		}
 		
-		$this->fuckhtml->load($html);
+		// get npt
+		if($json["results"]["search"]["hasNextPage"]){
 			
-		$div =
-			$this->fuckhtml
-			->getElementsByTagName("div");
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
-		
-		if(count($npt) !== 0){
-			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
-		$mod = $text[1];
+		$unit = strtolower($number[strlen($number) - 1]);
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
 		
-			case "bln.": $num = $num * 1000000000; break;
-			case "mln.": $num = $num * 1000000; break;
-			case "thsd.": $num = $num * 1000; break;
+		if(count($tmp) === 2){
+			
+			$decimal = (int)$tmp[1];
+		}else{
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

scraper/yep.php

@@ -14,234 +14,209 @@ class yep{
 	public function getfilters($page){
 		
 		return [
-			"country" => [
-				"display" => "Country",
+			"lang" => [
+				"display" => "Language",
 				"option" => [
-					"all" => "All regions",
-					"af" => "Afghanistan",
-					"al" => "Albania",
-					"dz" => "Algeria",
-					"as" => "American Samoa",
-					"ad" => "Andorra",
-					"ao" => "Angola",
-					"ai" => "Anguilla",
-					"ag" => "Antigua and Barbuda",
-					"ar" => "Argentina",
-					"am" => "Armenia",
-					"aw" => "Aruba",
-					"au" => "Australia",
-					"at" => "Austria",
-					"az" => "Azerbaijan",
-					"bs" => "Bahamas",
-					"bh" => "Bahrain",
-					"bd" => "Bangladesh",
-					"bb" => "Barbados",
-					"by" => "Belarus",
-					"be" => "Belgium",
-					"bz" => "Belize",
-					"bj" => "Benin",
-					"bt" => "Bhutan",
-					"bo" => "Bolivia",
-					"ba" => "Bosnia and Herzegovina",
-					"bw" => "Botswana",
-					"br" => "Brazil",
-					"bn" => "Brunei Darussalam",
-					"bg" => "Bulgaria",
-					"bf" => "Burkina Faso",
-					"bi" => "Burundi",
-					"cv" => "Cabo Verde",
-					"kh" => "Cambodia",
-					"cm" => "Cameroon",
-					"ca" => "Canada",
-					"ky" => "Cayman Islands",
-					"cf" => "Central African Republic",
-					"td" => "Chad",
-					"cl" => "Chile",
-					"cn" => "China",
-					"co" => "Colombia",
-					"cg" => "Congo",
-					"cd" => "Congo, Democratic Republic",
-					"ck" => "Cook Islands",
-					"cr" => "Costa Rica",
-					"hr" => "Croatia",
-					"cu" => "Cuba",
-					"cy" => "Cyprus",
-					"cz" => "Czechia",
-					"ci" => "Côte d'Ivoire",
-					"dk" => "Denmark",
-					"dj" => "Djibouti",
-					"dm" => "Dominica",
-					"do" => "Dominican Republic",
-					"ec" => "Ecuador",
-					"eg" => "Egypt",
-					"sv" => "El Salvador",
-					"gq" => "Equatorial Guinea",
-					"ee" => "Estonia",
-					"et" => "Ethiopia",
-					"fo" => "Faroe Islands",
-					"fj" => "Fiji",
-					"fi" => "Finland",
-					"fr" => "France",
-					"gf" => "French Guiana",
-					"pf" => "French Polynesia",
-					"ga" => "Gabon",
-					"gm" => "Gambia",
-					"ge" => "Georgia",
-					"de" => "Germany",
-					"gh" => "Ghana",
-					"gi" => "Gibraltar",
-					"gr" => "Greece",
-					"gl" => "Greenland",
-					"gd" => "Grenada",
-					"gp" => "Guadeloupe",
-					"gu" => "Guam",
-					"gt" => "Guatemala",
-					"gg" => "Guernsey",
-					"gn" => "Guinea",
-					"gy" => "Guyana",
-					"ht" => "Haiti",
-					"hn" => "Honduras",
-					"hk" => "Hong Kong",
-					"hu" => "Hungary",
-					"is" => "Iceland",
-					"in" => "India",
-					"id" => "Indonesia",
-					"iq" => "Iraq",
-					"ie" => "Ireland",
-					"im" => "Isle of Man",
-					"il" => "Israel",
-					"it" => "Italy",
-					"jm" => "Jamaica",
-					"jp" => "Japan",
-					"je" => "Jersey",
-					"jo" => "Jordan",
-					"kz" => "Kazakhstan",
-					"ke" => "Kenya",
-					"ki" => "Kiribati",
-					"kw" => "Kuwait",
-					"kg" => "Kyrgyzstan",
-					"la" => "Lao People's Democratic Republic",
-					"lv" => "Latvia",
-					"lb" => "Lebanon",
-					"ls" => "Lesotho",
-					"ly" => "Libya",
-					"li" => "Liechtenstein",
-					"lt" => "Lithuania",
-					"lu" => "Luxembourg",
-					"mk" => "Macedonia",
-					"mg" => "Madagascar",
-					"mw" => "Malawi",
-					"my" => "Malaysia",
-					"mv" => "Maldives",
-					"ml" => "Mali",
-					"mt" => "Malta",
-					"mq" => "Martinique",
-					"mr" => "Mauritania",
-					"mu" => "Mauritius",
-					"yt" => "Mayotte",
-					"mx" => "Mexico",
-					"fm" => "Micronesia, Federated States of",
-					"md" => "Moldova",
-					"mc" => "Monaco",
-					"mn" => "Mongolia",
-					"me" => "Montenegro",
-					"ms" => "Montserrat",
-					"ma" => "Morocco",
-					"mz" => "Mozambique",
-					"mm" => "Myanmar",
-					"na" => "Namibia",
-					"nr" => "Nauru",
-					"np" => "Nepal",
-					"nl" => "Netherlands",
-					"nc" => "New Caledonia",
-					"nz" => "New Zealand",
-					"ni" => "Nicaragua",
-					"ne" => "Niger",
-					"ng" => "Nigeria",
-					"nu" => "Niue",
-					"no" => "Norway",
-					"om" => "Oman",
-					"pk" => "Pakistan",
-					"ps" => "Palestine, State of",
-					"pa" => "Panama",
-					"pg" => "Papua New Guinea",
-					"py" => "Paraguay",
-					"pe" => "Peru",
-					"ph" => "Philippines",
-					"pn" => "Pitcairn",
-					"pl" => "Poland",
-					"pt" => "Portugal",
-					"pr" => "Puerto Rico",
-					"qa" => "Qatar",
-					"ro" => "Romania",
-					"ru" => "Russian Federation",
-					"rw" => "Rwanda",
-					"re" => "Réunion",
-					"sh" => "Saint Helena",
-					"kn" => "Saint Kitts and Nevis",
-					"lc" => "Saint Lucia",
-					"vc" => "Saint Vincent and the Grenadines",
-					"ws" => "Samoa",
-					"sm" => "San Marino",
-					"st" => "Sao Tome and Principe",
-					"sa" => "Saudi Arabia",
-					"sn" => "Senegal",
-					"rs" => "Serbia",
-					"sc" => "Seychelles",
-					"sl" => "Sierra Leone",
-					"sg" => "Singapore",
-					"sk" => "Slovakia",
-					"si" => "Slovenia",
-					"sb" => "Solomon Islands",
-					"so" => "Somalia",
-					"kr" => "Sourth Korea",
-					"za" => "South Africa",
-					"es" => "Spain",
-					"lk" => "Sri Lanka",
-					"sr" => "Suriname",
-					"se" => "Sweden",
-					"ch" => "Switzerland",
-					"tw" => "Taiwan",
-					"tj" => "Tajikistan",
-					"tz" => "Tanzania",
-					"th" => "Thailand",
-					"tl" => "Timor-Leste",
-					"tg" => "Togo",
-					"tk" => "Tokelau",
+					"any" => "Any language",
+					"aa" => "Afar",
+					"ab" => "Abkhazian",
+					"ae" => "Avestan",
+					"af" => "Afrikaans",
+					"ak" => "Akan",
+					"am" => "Amharic",
+					"an" => "Aragonese",
+					"ar" => "Arabic",
+					"as" => "Assamese",
+					"av" => "Avaric",
+					"ay" => "Aymara",
+					"az" => "Azerbaijani",
+					"ba" => "Bashkir",
+					"be" => "Belarusian",
+					"bg" => "Bulgarian",
+					"bh" => "Bihari",
+					"bi" => "Bislama",
+					"bm" => "Bambara",
+					"bn" => "Bengali",
+					"bo" => "Tibetan",
+					"br" => "Breton",
+					"bs" => "Bosnian",
+					"ca" => "Catalan",
+					"ce" => "Chechen",
+					"ch" => "Chamorro",
+					"co" => "Corsican",
+					"cr" => "Cree",
+					"cs" => "Czech",
+					"cu" => "Church Slavic",
+					"cv" => "Chuvash",
+					"cy" => "Welsh",
+					"da" => "Danish",
+					"de" => "German",
+					"dv" => "Divehi",
+					"dz" => "Dzongkha",
+					"ee" => "Ewe",
+					"el" => "Greek",
+					"en" => "English",
+					"eo" => "Esperanto",
+					"es" => "Spanish",
+					"et" => "Estonian",
+					"eu" => "Basque",
+					"fa" => "Persian",
+					"ff" => "Fulah",
+					"fi" => "Finnish",
+					"fj" => "Fijian",
+					"fo" => "Faroese",
+					"fr" => "French",
+					"fy" => "Western Frisian",
+					"ga" => "Irish",
+					"gd" => "Scottish Gaelic",
+					"gl" => "Galician",
+					"gn" => "Guarani",
+					"gu" => "Gujarati",
+					"gv" => "Manx",
+					"ha" => "Hausa",
+					"he" => "Hebrew",
+					"hi" => "Hindi",
+					"ho" => "Hiri Motu",
+					"hr" => "Croatian",
+					"ht" => "Haitian",
+					"hu" => "Hungarian",
+					"hy" => "Armenian",
+					"hz" => "Herero",
+					"ia" => "Interlingua",
+					"id" => "Indonesian",
+					"ie" => "Interlingue",
+					"ig" => "Igbo",
+					"ii" => "Sichuan Yi",
+					"ik" => "Inupiaq",
+					"io" => "Ido",
+					"is" => "Icelandic",
+					"it" => "Italian",
+					"iu" => "Inuktitut",
+					"ja" => "Japanese",
+					"jv" => "Javanese",
+					"ka" => "Georgian",
+					"kg" => "Kongo",
+					"ki" => "Kikuyu",
+					"kj" => "Kuanyama",
+					"kk" => "Kazakh",
+					"kl" => "Kalaallisut",
+					"km" => "Central Khmer",
+					"kn" => "Kannada",
+					"ko" => "Korean",
+					"kr" => "Kanuri",
+					"ks" => "Kashmiri",
+					"ku" => "Kurdish",
+					"kv" => "Komi",
+					"kw" => "Cornish",
+					"ky" => "Kyrgyz",
+					"la" => "Latin",
+					"lb" => "Luxembourgish",
+					"lg" => "Ganda",
+					"li" => "Limburgish",
+					"ln" => "Lingala",
+					"lo" => "Lao",
+					"lt" => "Lithuanian",
+					"lu" => "Luba-Katanga",
+					"lv" => "Latvian",
+					"mg" => "Malagasy",
+					"mh" => "Marshallese",
+					"mi" => "Maori",
+					"mk" => "Macedonian",
+					"ml" => "Malayalam",
+					"mn" => "Mongolian",
+					"mr" => "Marathi",
+					"ms" => "Malay",
+					"mt" => "Maltese",
+					"my" => "Burmese",
+					"na" => "Nauru",
+					"nb" => "Norwegian Bokmål",
+					"nd" => "North Ndebele",
+					"ne" => "Nepali",
+					"ng" => "Ndonga",
+					"nl" => "Dutch",
+					"nn" => "Norwegian Nynorsk",
+					"no" => "Norwegian",
+					"nr" => "South Ndebele",
+					"nv" => "Navajo",
+					"ny" => "Chichewa",
+					"oc" => "Occitan",
+					"oj" => "Ojibwa",
+					"om" => "Oromo",
+					"or" => "Oriya",
+					"os" => "Ossetian",
+					"pa" => "Punjabi",
+					"pi" => "Pali",
+					"pl" => "Polish",
+					"ps" => "Pashto",
+					"pt" => "Portuguese",
+					"qu" => "Quechua",
+					"rm" => "Romansh",
+					"rn" => "Rundi",
+					"ro" => "Romanian",
+					"ru" => "Russian",
+					"rw" => "Kinyarwanda",
+					"sa" => "Sanskrit",
+					"sc" => "Sardinian",
+					"sd" => "Sindhi",
+					"se" => "Northern Sami",
+					"sg" => "Sango",
+					"si" => "Sinhala",
+					"sk" => "Slovak",
+					"sl" => "Slovenian",
+					"sm" => "Samoan",
+					"sn" => "Shona",
+					"so" => "Somali",
+					"sq" => "Albanian",
+					"sr" => "Serbian",
+					"ss" => "Swati",
+					"st" => "Southern Sotho",
+					"su" => "Sundanese",
+					"sv" => "Swedish",
+					"sw" => "Swahili",
+					"ta" => "Tamil",
+					"te" => "Telugu",
+					"tg" => "Tajik",
+					"th" => "Thai",
+					"ti" => "Tigrinya",
+					"tk" => "Turkmen",
+					"tl" => "Tagalog",
+					"tn" => "Tswana",
 					"to" => "Tonga",
-					"tt" => "Trinidad and Tobago",
-					"tn" => "Tunisia",
-					"tr" => "Turkey",
-					"tm" => "Turkmenistan",
-					"ug" => "Uganda",
-					"ua" => "Ukraine",
-					"ae" => "United Arab Emirates",
-					"gb" => "United Kingdom",
-					"us" => "United States",
-					"uy" => "Uruguay",
-					"uz" => "Uzbekistan",
-					"vu" => "Vanuatu",
-					"ve" => "Venezuela",
-					"vn" => "Vietnam",
-					"vg" => "Virgin Islands, British",
-					"vi" => "Virgin Islands, U.S.",
-					"ye" => "Yemen",
-					"zm" => "Zambia",
-					"zw" => "Zimbabwe"
+					"tr" => "Turkish",
+					"ts" => "Tsonga",
+					"tt" => "Tatar",
+					"tw" => "Twi",
+					"ty" => "Tahitian",
+					"ug" => "Uyghur",
+					"uk" => "Ukrainian",
+					"ur" => "Urdu",
+					"uz" => "Uzbek",
+					"ve" => "Venda",
+					"vi" => "Vietnamese",
+					"vo" => "Volapük",
+					"wa" => "Walloon",
+					"wo" => "Wolof",
+					"xh" => "Xhosa",
+					"yi" => "Yiddish",
+					"yo" => "Yoruba",
+					"za" => "Zhuang",
+					"zh" => "Chinese",
+					"zh-cn" => "Chinese (Simplified)",
+					"zh-tw" => "Chinese (Traditional)",
+					"zu" => "Zulu"
 				]
 			],
 			"nsfw" => [
 				"display" => "NSFW",
 				"option" => [
 					"yes" => "Yes",
-					"maybe" => "Maybe",
 					"no" => "No"
 				]
 			]
 		];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
 		
 		$curlproc = curl_init();
 		
@@ -252,25 +227,43 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// use http1 lol
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: https://yep.com/",
-			"Origin: https://yep.com",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"Priority: u=4",
-			"TE: trailers"]
-		);
+		
+		if($use_api){
+			
+			$post_data = json_encode($post_data);
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["Content-Type: application/json",
+				"Authorization: Bearer $bearer",
+				"Content-Length: " . strlen($post_data)]
+			);
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://yep.com/",
+				"Origin: https://yep.com",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
+				"TE: trailers"]
+			);
+		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -295,22 +288,17 @@ class yep{
 	
 	public function web($get){
 		
+		if(config::YEP_USE_API){
+			
+			return $this->web_api($get);
+		}
+		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -327,22 +315,23 @@ class yep{
 			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"limit" => 100, // wwwwwwwwwwwwwww
+			"query" => $search,
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
+		if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
+		
 		try{
 			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
+			// https://api.yep.com/search?limit=20&query=asmr
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "web"
-					]
+					"https://api.yep.com/search",
+					$filters
 				);
 			
 		}catch(Exception $error){
@@ -408,7 +397,7 @@ class yep{
 									)
 								),
 							"url" => $item["url"],
-							"date" => strtotime($item["first_seen"]),
+							"date" => null,
 							"type" => "web",
 							"thumb" => [
 								"url" => null,
@@ -422,83 +411,11 @@ class yep{
 			}
 		}
 		
-		if(isset($json[1]["featured_news"])){
-			
-			foreach($json[1]["featured_news"] as $news){
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$news["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($news["first_seen"]),
-					"thumb" =>
-						isset($news["img"]) ?
-						[
-							"url" => $this->unshiturl($news["img"]),
-							"ratio" => "16:9"
-						] :
-						[
-							"url" => null,
-							"ratio" => null
-						],
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		if(isset($json[1]["featured_images"])){
-			
-			foreach($json[1]["featured_images"] as $image){
-				
-				if(
-					$image["width"] !== 0 &&
-					$image["height"] !== 0
-				){
-					
-					$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
-					$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
-					
-					$width = $image["width"];
-					$height = $image["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $image["title"],
-					"source" => [
-						[
-							"url" => $image["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $image["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $image["host_page"]
-				];
-			}
-		}
-		
 		return $out;
 	}
 	
 	
-	
-	public function image($get){
+	private function web_api($get){
 		
 		$search = $get["s"];
 		if(strlen($search) === 0){
@@ -506,142 +423,53 @@ class yep{
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
 			"npt" => null,
-			"image" => []
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"query" => $search,
+			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
+		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
+		
+		// add api key
+		$key_data = $this->backend->get_key();
+		
 		try{
 			
 			$json =
 				$this->get(
-					$this->backend->get_ip(), // no nextpage!
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "images"
-					]
+					$this->backend->get_ip($key_data["increment"]),
+					"https://platform.yep.com/api/search",
+					[],
+					true,
+					$filters,
+					$key_data["key"]
 				);
+			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
 		}
 		
-		$this->detect_cf($json);
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
-				
-				if(
-					$item["width"] !== 0 &&
-					$item["height"] !== 0
-				){
-					
-					$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
-					$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
-					
-					$width = $item["width"];
-					$height = $item["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $item["title"],
-					"source" => [
-						[
-							"url" => $item["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $item["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $item["host_page"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
-	
-	public function news($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		try{
-			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
-			$json =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "news"
-					]
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$this->detect_cf($json);
+		// should never happen
+		//$this->detect_cf($json);
 		
 		$json = json_decode($json, true);
 		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
@@ -651,34 +479,53 @@ class yep{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
+		if(isset($json["error"])){
 			
-				$out["news"][] = [
-					"title" => $item["title"],
-					"author" => null,
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$item["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($item["first_seen"]),
-					"thumb" =>
-						isset($item["img"]) ?
-						[
-							"url" => $this->unshiturl($item["img"]),
-							"ratio" => "16:9"
-						] :
-						[
-							"url" => null,
-							"ratio" => null
-						],
-					"url" => $item["url"]
-				];
+			throw new Exception("Yep API returned an error: " . $json["error"]);
+		}
+		
+		if(isset($json["errors"])){
+			
+			throw new Exception("Yep API returned the following errors: {$json["message"]}");
+		}
+		
+		if(
+			isset($json["success"]) &&
+			$json["success"] !== true
+		){
+			
+			throw new Exception("Yep API returned a false-y success value");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Yep API did not return a results object");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			if(
+				$item["url"] === null ||
+				$item["url"] == ""
+			){
+				
+				// sometimes API fucks up
+				continue;
 			}
+			
+			$out["web"][] = [
+				"title" => $item["title"],
+				"description" => $item["description"],
+				"url" => $item["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
 		}
 		
 		return $out;

settings.php

@@ -91,10 +91,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -161,14 +157,6 @@ $settings = [
 						"value" => "yep",
 						"text" => "Yep"
 					],
-					[
-						"value" => "greppr",
-						"text" => "Greppr"
-					],
-					[
-						"value" => "crowdview",
-						"text" => "Crowdview"
-					],
 					[
 						"value" => "mwmbl",
 						"text" => "Mwmbl"
@@ -177,6 +165,10 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -194,12 +186,12 @@ $settings = [
 						"text" => "Marginalia"
 					],
 					[
-						"value" => "wiby",
-						"text" => "wiby"
+						"value" => "purili",
+						"text" => "Purili"
 					],
 					[
-						"value" => "curlie",
-						"text" => "Curlie"
+						"value" => "wiby",
+						"text" => "wiby"
 					]
 				]
 			],
@@ -223,6 +215,10 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "google_api",
+						"text" => "Google API"
+					],
 					[
 						"value" => "google_cse",
 						"text" => "Google CSE"
@@ -240,8 +236,8 @@ $settings = [
 						"text" => "Qwant"
 					],
 					[
-						"value" => "yep",
-						"text" => "Yep"
+						"value" => "naver",
+						"text" => "Naver"
 					],
 					[
 						"value" => "baidu",
@@ -337,6 +333,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -345,6 +345,10 @@ $settings = [
 						"value" => "coccoc",
 						"text" => "Cốc Cốc"
 					],
+					[
+						"value" => "purili",
+						"text" => "Purili"
+					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
@@ -379,10 +383,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "yep",
-						"text" => "Yep"
-					],
 					[
 						"value" => "mojeek",
 						"text" => "Mojeek"

static/serverping.js

@@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
+
+// server popup without inline javascript
+tbody.addEventListener('click', function(event){
+	var row = event.target.closest('.show_server');
+	if(!row) return;
+
+	show_server(row.id);
+})
+
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -343,7 +352,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
+		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -406,6 +415,7 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
+
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/style.css

@@ -1013,6 +1013,23 @@ table tr a:last-child{
 	font-weight:bold;
 }
 
+/* booru */
+#images .image .i-title-wrapper{
+	margin-bottom:7px;
+	white-space:nowrap;
+	overflow:hidden;
+	text-overflow:ellipsis;
+}
+#images .image .i-title-wrapper a{
+	display:inline;
+}
+
+#images .image .i-title-wrapper .title{
+	display:inline;
+	margin-bottom:0;
+}
+/* end booru */
+
 #images .image .description{
 	overflow:hidden;
 	height:45px;
@@ -1113,6 +1130,17 @@ table tr a:last-child{
 /*
 	Settings page
 */
+.web .settings select{
+	all:unset;
+	user-select:none;
+	display:block;
+	border:1px solid var(--504945);
+	border-radius:2px;
+	font-size:14px;
+	padding:0 20px 0 4px;
+	height:24px;
+}
+
 .web .settings{
 	margin-top:17px;
 	border:1px dashed var(--504945);

static/themes/Rosé Pine.css

@@ -0,0 +1,20 @@
+:root{
+	/* background */
+	--1d2021: #191724;
+	--282828: #1f1d2e;
+	--3c3836: #26233a;
+	--504945: #524f67;
+
+	/* font */
+	--928374: #e0def4;
+	--a89984: #908caa;
+	--bdae93: #6e6a86;
+	--8ec07c: #9ccfd8;
+	--ebdbb2: #f6c177;
+
+	/* code highlighter */
+	--comment: #6e6a86;
+	--default: #c4a7e7;
+	--keyword: #31748f;
+	--string: #f6c177;
+}