yep fix. fuck fsky

Reviewed-on: lolcat/4get#94

Dockerfile

@@ -1,8 +1,17 @@
+FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate
+
 FROM alpine:3.21
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg
+RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates 
+
+COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
+COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
+
+ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so
+ENV CURL_IMPERSONATE=ff117
+ENV CURL_IMPERSONATE_HEADERS=no
 
 COPY . .
 

README.md

@@ -13,11 +13,11 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | libreY      | araa      | hearch.co         |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
+|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
+| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -31,25 +31,29 @@ tl;dr 4get is the best way to browse for shit.
 
 # Supported websites
 
-| Web        | Images       | Videos       | News       | Music      | Autocompleter |
-|------------|--------------|--------------|------------|------------|---------------|
-| DuckDuckGo | DuckDuckGo   | YouTube      | DuckDuckGo | Soundcloud | Brave         |
-| Brave      | Brave        | Sepia Search | Brave      |            | DuckDuckGo    |
-| Yandex     | Yandex       | DuckDuckGo   | Google     |            | Yandex        |
-| Google     | Google       | Brave        | Startpage  |            | Google        |
-| Startpage  | Startpage    | Yandex       | Qwant      |            | Startpage     |
-| Qwant      | Qwant        | Google       | Mojeek     |            | Kagi          |
-| Ghostery   | Yep          | Startpage    | Baidu      |            | Qwant         |
-| Yep        | Baidu        | Qwant        |            |            | Ghostery      |
-| Greppr     | Pinterest    | Baidu        |            |            | Yep           |
-| Crowdview  | 500px        | Coc Coc      |            |            | Marginalia    |
-| Mwmbl      | VSCO         |              |            |            | YouTube       |
-| Mojeek     | Imgur        |              |            |            | Soundcloud    |
-| Baidu      | FindThatMeme |              |            |            |               |
-| Coc Coc    |              |              |            |            |               |
-| Marginalia |              |              |            |            |               |
-| wiby       |              |              |            |            |               |
-| Curlie     |              |              |            |            |               |
+| web          | images       | videos       | news         | music      | autocomplete |
+|--------------|--------------|--------------|--------------|------------|--------------|
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
+| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
+| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
+| Qwant        | Qwant        | Startpage    |              |            | Yep          |
+| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
+| Yep          | Solofield    | Baidu        |              |            | YouTube      |
+| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
+| Mojeek       | Cara         | Solofield    |              |            |              |
+| Baidu        | Flickr       |              |              |            |              |
+| Coc Coc      | Pexels       |              |              |            |              |
+| Solofield    | Pixabay      |              |              |            |              |
+| Marginalia   | Unsplash     |              |              |            |              |
+| wiby         | 500px        |              |              |            |              |
+|              | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.
@@ -58,4 +62,4 @@ Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/"
 Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPL
+AGPLv3-only

audio/linear.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/sc.php

@@ -1,224 +0,0 @@
-<?php
-
-new sc_audio();
-
-class sc_audio{
-	
-	public function __construct(){
-		
-		include "../data/config.php";
-		include "../lib/curlproxy.php";
-		$this->proxy = new proxy();
-		
-		if(isset($_GET["u"])){
-			
-			/*
-				we're now proxying audio
-			*/
-			$viewkey = $_GET["u"];
-			
-			if(!isset($_GET["r"])){
-				
-				$this->do404("Ranges(r) are missing");
-			}
-			
-			$ranges = explode(",", $_GET["r"]);
-			
-			// sanitize ranges
-			foreach($ranges as &$range){
-				
-				if(!is_numeric($range)){
-					
-					$this->do404("Invalid range specified");
-				}
-				
-				$range = (int)$range;
-			}
-			
-			// sort ranges (just to make sure)
-			sort($ranges);
-			
-			// convert ranges to pairs
-			$last = -1;
-			foreach($ranges as &$r){
-				
-				$tmp = $r;
-				$r = [$last + 1, $r];
-				
-				$last = $tmp;
-			}
-			
-			$browser_headers = getallheaders();
-			
-			// get the requested range from client
-			$client_range = 0;
-			foreach($browser_headers as $key => $value){
-				
-				if(strtolower($key) == "range"){
-					
-					preg_match(
-						'/bytes=([0-9]+)/',
-						$value,
-						$client_regex
-					);
-					
-					if(isset($client_regex[1])){
-						
-						$client_range = (int)$client_regex[1];
-					}else{
-						
-						$client_range = 0;
-					}
-					break;
-				}
-			}
-			
-			if(
-				$client_range < 0 ||
-				$client_range > $ranges[count($ranges) - 1][1]
-			){
-				
-				// range is not satisfiable
-				http_response_code(416);
-				header("Content-Type: text/plain");
-				die();
-			}
-			
-			$rng = null;
-			for($i=0; $i<count($ranges); $i++){
-				
-				if($ranges[$i][0] <= $client_range){
-					
-					$rng = $ranges[$i];
-				}
-			}
-			
-			// proxy data!
-			http_response_code(206); // partial content
-			header("Accept-Ranges: bytes");
-			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
-			
-			$viewkey =
-				preg_replace(
-					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-					'/media/$1/' . $rng[0] . '/' . $rng[1],
-					$viewkey
-				);
-			
-			try{
-				
-				$this->proxy->stream_linear_audio(
-					$viewkey
-				);
-			}catch(Exception $error){
-				
-				$this->do404("Could not read stream");
-			}
-			
-			die();
-		}
-		
-		/*
-			redirect user to correct resource
-			we need to scrape and store the byte positions in the result URL
-		*/
-		if(!isset($_GET["s"])){
-			
-			$this->do404("The URL(s) parameter is missing");
-		}
-		
-		$viewkey = $_GET["s"];
-		
-		if(
-			preg_match(
-				'/soundcloud\.com$/',
-				parse_url($viewkey, PHP_URL_HOST)
-			) === false
-		){
-			
-			$this->do404("This endpoint can only be used for soundcloud streams");
-		}
-		
-		try{
-			
-			$json = $this->proxy->get($viewkey)["body"];
-		}catch(Exception $error){
-			
-			$this->do404("Curl error: " . $error->getMessage());
-		}
-		
-		$json = json_decode($json, true);
-		
-		if(!isset($json["url"])){
-			
-			$this->do404("Could not get URL from JSON");
-		}
-		
-		$viewkey = $json["url"];
-		
-		$m3u8 = $this->proxy->get($viewkey)["body"];
-		
-		$m3u8 = explode("\n", $m3u8);
-		
-		$lineout = null;
-		$streampos_arr = [];
-		foreach($m3u8 as $line){
-			
-			$line = trim($line);
-			if($line[0] == "#"){
-				
-				continue;
-			}
-			
-			if($lineout === null){
-				$lineout = $line;
-			}
-			
-			preg_match(
-				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
-				$line,
-				$matches
-			);
-			
-			if(isset($matches[0])){
-				
-				$streampos_arr[] = [
-					(int)$matches[1],
-					(int)$matches[2]
-				];
-			}
-		}
-		
-		if($lineout === null){
-			
-			$this->do404("Could not get stream URL");
-		}
-		
-		$lineout =
-			preg_replace(
-				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-				'/media/$1/0/0',
-				$lineout
-			);
-		
-		$streampos = [];
-		
-		foreach($streampos_arr as $pos){
-			
-			$streampos[] = $pos[1];
-		}
-		
-		$streampos = implode(",", $streampos);
-		
-		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
-		header("Accept-Ranges: bytes");
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

audio/seekable.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/spotify.php

@@ -1,214 +0,0 @@
-<?php
-
-include "../data/config.php";
-new spotify();
-
-class spotify{
-	
-	public function __construct(){
-		
-		include "../lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		if(
-			!isset($_GET["s"]) ||
-			!preg_match(
-				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
-				$_GET["s"],
-				$matches
-			)
-		){
-			
-			$this->do404("The track ID(s) parameter is missing or invalid");
-		}
-		
-		try{
-			
-			if($matches[1] == "episode"){
-				
-				$uri = "show";
-			}else{
-				
-				$uri = $matches[1];
-			}
-			
-			$embed =
-				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
-		}catch(Exception $error){
-			
-			$this->do404("Failed to fetch embed data");
-		}
-		
-		$this->fuckhtml->load($embed);
-		
-		$json =
-			$this->fuckhtml
-			->getElementById(
-				"__NEXT_DATA__",
-				"script"
-			);
-		
-		if($json === null){
-			
-			$this->do404("Failed to extract JSON");
-		}
-		
-		$json =
-			json_decode($json["innerHTML"], true);
-		
-		if($json === null){
-			
-			$this->do404("Failed to decode JSON");
-		}
-		
-		switch($matches[1]){
-			
-			case "track":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["audioPreview"]
-						["url"]
-					)
-				){
-					
-					header("Content-type: audio/mpeg");
-					header(
-						"Location: /audio/linear?s=" .
-						urlencode(
-							$json
-							["props"]
-							["pageProps"]
-							["state"]
-							["data"]
-							["entity"]
-							["audioPreview"]
-							["url"]
-						)
-					);
-				}else{
-					
-					$this->do404("Could not extract playback URL");
-				}
-				break;
-			
-			case "episode":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["id"]
-					)
-				){
-					
-					try{
-						$json =
-							$this->get(
-								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
-								$json
-								["props"]
-								["pageProps"]
-								["state"]
-								["data"]
-								["entity"]
-								["id"] .
-								"/com.widevine.alpha"
-							);
-					}catch(Exception $error){
-						
-						$this->do404("Failed to fetch audio resource");
-					}
-					
-					$json = json_decode($json, true);
-					
-					if($json === null){
-						
-						$this->do404("Failed to decode audio resource JSON");
-					}
-					
-					if(
-						isset($json["passthrough"]) &&
-						$json["passthrough"] == "ALLOWED" &&
-						isset($json["passthroughUrl"])
-					){
-						
-						header(
-							"Location:" .
-							"/audio/linear.php?s=" .
-							urlencode(
-								str_replace(
-									"http://",
-									"https://",
-									$json["passthroughUrl"]
-								)
-							)
-						);
-					}else{
-						
-						$this->do404("Failed to find passthroughUrl");
-					}
-					
-				}else{
-					
-					$this->do404("Failed to find episode ID");
-				}
-				break;
-			}
-	}
-	
-	private function get($url){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

data/api_keys/yep.txt

@@ -0,0 +1 @@
+# Paste Yep API keys here

data/config.php

@@ -23,6 +23,16 @@ class config{
 	// Enable the API?
 	const API_ENABLED = true;
 	
+	//
+	// 4play (session provider)
+	//
+	// Enable 4play API?
+	const FPLAY_ENABLE_API = true;
+	
+	// 4play password. Please set this to something secure if you enable the 4play API.
+	// This password is used to POST sessions to /api/v2/provide_sesh
+	const FPLAY_PASSWORD = "1234";
+	
 	//
 	// BOT PROTECTION
 	//
@@ -118,7 +128,10 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:145.0) Gecko/20100101 Firefox/145.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:151.0) Gecko/20100101 Firefox/151.0";
+	
+	// User agent to use with 4get-friendly APIs
+	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
@@ -128,12 +141,9 @@ class config{
 	const PROXY_YAHOO = false;
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
-	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
-	const PROXY_MULLVAD_GOOGLE = false;
-	const PROXY_MULLVAD_BRAVE = false;
 	const PROXY_STARTPAGE = false;
 	const PROXY_QWANT = false;
 	const PROXY_BAIDU = false;
@@ -154,14 +164,14 @@ class config{
 	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
-	const PROXY_SANKAKUCOMPLEX = false;
 	const PROXY_FLICKR = false;
+	const PROXY_PIXABAY = false;
+	const PROXY_UNSPLASH = false;
+	const PROXY_PEXELS = false;
 	const PROXY_FIVEHPX = false;
 	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
-	const PROXY_GREPPR = false;
-	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
@@ -169,6 +179,11 @@ class config{
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
+	const PROXY_SAFEBOORU = false;
+	const PROXY_KONACHAN = false;
+	const PROXY_YANDERE = false;
+	const PROXY_TBIB = false;
+	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
@@ -181,4 +196,7 @@ class config{
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
+	
+	// Yep
+	const YEP_USE_API = false;
 }

docs/apache2-example.md

@@ -49,9 +49,13 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	</Directory>
 
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 

docs/apache2.md

@@ -104,9 +104,13 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 	
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```

docs/develop.md

@@ -0,0 +1,7 @@
+# Develop
+
+Start the PHP webserver like this, assuming you have curl-impersonate installed already
+
+```sh
+LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
+```

lib/bot_protection.php

@@ -133,6 +133,9 @@ class bot_protection{
 			$answers[] = $regex;
 		}
 		
+		// dedup
+		$answers = array_unique($answers);
+		
 		if(
 			!$invalid &&
 			$key !== false // has captcha been gen'd?

lib/frontend.php

@@ -2,6 +2,52 @@
 
 class frontend{
 	
+	public function validateurl($url, $net_validate = false){
+		
+		$url_parts = parse_url($url);
+		
+		// check if required parts are there
+		if(
+			!isset($url_parts["scheme"]) ||
+			!(
+				$url_parts["scheme"] == "http" ||
+				$url_parts["scheme"] == "https"
+			) ||
+			!isset($url_parts["host"])
+		){
+			return false;
+		}
+		
+		if($net_validate){
+			$ip = 
+				str_replace(
+					["[", "]"], // handle ipv6
+					"",
+					$url_parts["host"]
+				);
+			
+			// if its not an IP
+			if(!filter_var($ip, FILTER_VALIDATE_IP)){
+				
+				// resolve domain's IP
+				$ip = gethostbyname($url_parts["host"] . ".");
+			}
+			
+			// check if its localhost
+			if(
+				filter_var(
+					$ip,
+					FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+				) === false
+			){
+				
+				return false;
+			}
+		}
+		
+		return true;
+	}
+	
 	public function load($template, $replacements = []){
 		
 		$replacements["server_name"] = htmlspecialchars(config::SERVER_NAME);
@@ -487,368 +533,6 @@ class frontend{
 					'</button>' .
 					'<div class="favicon-dropdown">';
 			
-			/*
-				Add archive links
-			*/
-			if(
-				$host["host"] == "boards.4chan.org" ||
-				$host["host"] == "boards.4channel.org"
-			){
-				
-				$archives = [];
-				$path = explode("/", $host["path"]);
-				$count = count($path);
-				// /pol/thread/417568063/post-shitty-memes-if-you-want-to
-				
-				if($count !== 0){
-					
-					$isboard = true;
-					
-					switch($path[1]){
-						
-						case "con":
-							break;
-						
-						case "q":
-							$archives[] = "desuarchive.org";
-							break;
-							
-						case "qa":
-							$archives[] = "desuarchive.org";
-							break;
-							
-						case "qb":
-							$archives[] = "arch.b4k.co";
-							break;
-							
-						case "trash":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "a":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "c":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "w":
-							break;
-						
-						case "m":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "cgl":
-							$archives[] = "desuarchive.org";
-							$archives[] = "warosu.org";
-							break;
-						
-						case "f":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "n":
-							break;
-						
-						case "jp":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "vt":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "v":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vg":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vm":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vmg":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vp":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vr":
-							$archives[] = "desuarchive.org";
-							$archives[] = "warosu.org";
-							break;
-						
-						case "vrpg":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "vst":
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "co":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "g":
-							$archives[] = "desuarchive.org";
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "tv":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "k":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "o":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "an":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "tg":
-							$archives[] = "desuarchive.org";
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "sp":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "xs":
-							$archives[] = "eientei.xyz";
-							break;
-						
-						case "pw":
-							break;
-						
-						case "sci":
-							$archives[] = "warosu.org";
-							$archives[] = "eientei.xyz";
-							break;
-						
-						case "his":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "int":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "out":
-							break;
-						
-						case "toy":
-							break;
-						
-						case "i":
-							$archives[] = "archiveofsins.com";
-							$archives[] = "eientei.xyz";
-							break;
-						
-						case "po":
-							break;
-						
-						case "p":
-							break;
-						
-						case "ck":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "ic":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "wg":
-							break;
-						
-						case "lit":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "mu":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "fa":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "3":
-							$archives[] = "warosu.org";
-							$archives[] = "eientei.xyz";
-							break;
-						
-						case "gd":
-							break;
-						
-						case "diy":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "wsg":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "qst":
-							break;
-						
-						case "biz":
-							$archives[] = "warosu.org";
-							break;
-						
-						case "trv":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "fit":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "x":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "adv":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "lgbt":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "mlp":
-							$archives[] = "desuarchive.org";
-							$archives[] = "arch.b4k.co";
-							break;
-						
-						case "news":
-							break;
-						
-						case "wsr":
-							break;
-						
-						case "vip":
-							break;
-						
-						case "b":
-							$archives[] = "thebarchive.com";
-							break;
-						
-						case "r9k":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "pol":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "bant":
-							$archives[] = "thebarchive.com";
-							break;
-						
-						case "soc":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "s4s":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "s":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "hc":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "hm":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "h":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "e":
-							break;
-						
-						case "u":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "d":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "t":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						case "hr":
-							$archives[] = "archive.4plebs.org";
-							break;
-						
-						case "gif":
-							break;
-						
-						case "aco":
-							$archives[] = "desuarchive.org";
-							break;
-						
-						case "r":
-							$archives[] = "archiveofsins.com";
-							break;
-						
-						default:
-							$isboard = false;
-							break;
-					}
-					
-					if($isboard === true){
-						
-						$archives[] = "archived.moe";
-					}
-					
-					$trail = "";
-					
-					if(
-						isset($path[2]) &&
-						isset($path[3]) &&
-						$path[2] == "thread"
-					){
-						
-						$trail .= "/" . $path[1] . "/thread/" . $path[3];
-					}elseif($isboard){
-						
-						$trail = "/" . $path[1] . "/";
-					}
-					
-					for($i=0; $i<count($archives); $i++){
-						
-						$payload .=
-							'<a href="https://' . $archives[$i] . $trail . '" class="list" target="_BLANK">' .
-								'<img src="/favicon?s=https://' . $archives[$i] . '" alt="' . $archives[$i][0] . $archives[$i][1] . '">' .
-								$archives[$i] .
-							'</a>';
-					}
-				}
-			}
-			
 			$payload .=
 					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
 					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
@@ -960,18 +644,15 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"ghostery" => "Ghostery",
 						"yep" => "Yep",
-						"greppr" => "Greppr",
-						"crowdview" => "Crowdview",
 						"mwmbl" => "Mwmbl",
 						"mojeek" => "Mojeek",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"coccoc" => "Cốc Cốc",
 						"solofield" => "Solofield",
 						"marginalia" => "Marginalia",
-						"wiby" => "wiby",
-						"curlie" => "Curlie"
+						"wiby" => "wiby"
 					]
 				];
 				break;
@@ -984,21 +665,24 @@ class frontend{
 						"yandex" => "Yandex",
 						"brave" => "Brave",
 						"google" => "Google",
+						"google_api" => "Google API",
 						"google_cse" => "Google CSE",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"yep" => "Yep",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"solofield" => "Solofield",
 						"pinterest" => "Pinterest",
 						"cara" => "Cara",
 						"flickr" => "Flickr",
+						"pexels" => "Pexels",
+						"pixabay" => "Pixabay",
+						"unsplash" => "Unsplash",
 						"fivehpx" => "500px",
 						"vsco" => "VSCO",
 						"imgur" => "Imgur",
-						"ftm" => "FindThatMeme",
-						//"sankakucomplex" => "SankakuComplex"
+						"ftm" => "FindThatMeme"
 					]
 				];
 				break;
@@ -1020,6 +704,7 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
+						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"coccoc" => "Cốc Cốc",
 						"solofield" => "Solofield"
@@ -1037,7 +722,6 @@ class frontend{
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"yep" => "Yep",
 						"mojeek" => "Mojeek",
 						"baidu" => "Baidu"
 					]
@@ -1054,6 +738,22 @@ class frontend{
 					]
 				];
 				break;
+			
+			case "booru":
+				$filters["scraper"] = [
+					"display" => "Scraper",
+					"option" => [
+						"safebooru" => "Safebooru",
+						"konachan" => "Konachan",
+						"tbib" => "The Big Imageboard",
+						"gelbooru" => "Gelbooru",
+						"yandere" => "Yande.re",
+						"tbib" => "The Big Imageboard",
+						"sankakucomplex" => "SankakuComplex",
+						"soybooru" => "SoyBooru"
+					]
+				];
+				break;
 		}
 		
 		// get scraper name from user input, or default out to preferred scraper
@@ -1230,6 +930,7 @@ class frontend{
 		
 		$html = null;
 		
+		//foreach(["web", "images", "videos", "news", "music", "booru"] as $type){
 		foreach(["web", "images", "videos", "news", "music"] as $type){
 			
 			$html .= '<a href="/' . $type . '?s=' . urlencode($query);

lib/fuckhtml.php

@@ -553,28 +553,21 @@ class fuckhtml{
 				
 				case "\"":
 				case "'":
-					if(
-						$i !== 0 && // only check if a quote could be there
-						(
-							(
-								$json[$i - 1] === "\\" &&
-								(
-									$i === 2 ||
-									$json[$i - 2] === "\\"
-								)
-							) ||
-							$json[$i - 1] !== "\\"
-						)
-					){
-						// found a non-escaped quote
+					// count preceding backslashes
+					$bsCount = 0;
+					$j = $i - 1;
 
+					while($j >= 0 && $json[$j] === "\\"){
+						$bsCount++;
+						$j--;
+					}
+
+					// quote is NOT escaped if even number of backslashes
+					if($bsCount % 2 === 0){
 						if($in_quote === null){
-							
 							// open quote
 							$in_quote = $json[$i];
-							
 						}elseif($in_quote === $json[$i]){
-							
 							// close quote
 							$in_quote = null;
 						}

scraper/baidu.php

@@ -666,14 +666,24 @@ class baidu{
 			//
 			// Don't parse as a search result if it's a card
 			//
-			$card =
+			$tpl_probe =
+				$this->fuckhtml
+				->getElementsByAttributeValue(
+					"tpl",
+					"www_index",
+					[$datafield]
+				);
+			
+			if(
+				count($tpl_probe) === 0 &&
+				count(
 					$this->fuckhtml
 					->getElementsByClassName(
 						"cosc-card",
 						$div
-				);
-			
-			if(count($card) !== 0){
+					)
+				) !== 0
+			){
 				
 				//
 				// Parse chinese youtube shorts
@@ -870,10 +880,13 @@ class baidu{
 			// class:FYB_RD -> News garbage, IGNORE
 			
 			$result =
+				array_merge(
+					$tpl_probe,
 					$this->fuckhtml
 					->getElementsByClassName(
 						"result",
 						[$datafield]
+					)
 				);
 			
 			if(count($result) !== 0){
@@ -885,7 +898,7 @@ class baidu{
 				$title =
 					$this->fuckhtml
 					->getElementsByClassName(
-						"sc-link",
+						"cosc-title-a",
 						"a"
 					);
 				
@@ -905,8 +918,9 @@ class baidu{
 				
 				$description =
 					$this->fuckhtml
-					->getElementsByClassName(
-						"c-color",
+					->getElementsByAttributeValue(
+						"data-sanssr-cmpt",
+						"card/www-summary-1",
 						$div
 					);
 				

scraper/brave.php

@@ -347,11 +347,8 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
-		/*
-		$handle = fopen("scraper/brave.html", "r");
-		$html = fread($handle, filesize("scraper/brave.html"));
-		fclose($handle);*/
 		
+		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(

scraper/crowdview.php

@@ -1,145 +0,0 @@
-<?php
-
-class crowdview{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("crowdview");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$proxy = $this->backend->get_ip();
-		
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://crowdview-next-js.onrender.com/api/search-v3",
-				[
-					"query" => $search
-				]
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$json = json_decode($json, true);
-		
-		if($json === NULL){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		foreach($json["results"] as $item){
-			
-			$description = explode("<b>", $item["snippet"], 2);
-			
-			$out["web"][] = [
-				"title" => $this->sanitize($item["title"]),
-				"description" => $this->sanitize($description[1]),
-				"url" => $item["link"],
-				"date" => strtotime($description[0]),
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function sanitize($html){
-		
-		return
-			trim(
-				$this->fuckhtml
-				->getTextContent(
-					html_entity_decode(
-						$html
-					)
-				),
-				". "
-			);
-	}
-}

scraper/curlie.php

@@ -1,309 +0,0 @@
-<?php
-
-class curlie{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("curlie");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"lang" => [
-				"display" => "Language",
-				"option" => [
-					"any" => "Any language",
-					"en" => "English",
-					"de" => "German",
-					"fr" => "French",
-					"ja" => "Japanese",
-					"it" => "Italian",
-					"es" => "Spanish",
-					"ru" => "Russian",
-					"nl" => "Dutch",
-					"pl" => "Polish",
-					"tr" => "Turkish",
-					"da" => "Danish",
-					"sv" => "Swedish",
-					"no" => "Norwegian",
-					"is" => "Icelandic",
-					"fo" => "Faroese",
-					"fi" => "Finnish",
-					"et" => "Estonian",
-					"lt" => "Lithuanian",
-					"lv" => "Latvian",
-					"cy" => "Welsh",
-					"ga" => "Irish",
-					"gd" => "Scottish Gaelic",
-					"br" => "Breton",
-					"fy" => "Frisian",
-					"frr" => "North Frisian",
-					"gem" => "Saterland Frisian",
-					"lb" => "Luxembourgish",
-					"rm" => "Romansh",
-					"pt" => "Portuguese",
-					"ca" => "Catalan",
-					"gl" => "Galician",
-					"eu" => "Basque",
-					"ast" => "Asturian",
-					"an" => "Aragonese",
-					"fur" => "Friulan",
-					"sc" => "Sardinian",
-					"scn" => "Sicilian",
-					"oc" => "Occitan",
-					"be" => "Belarusian",
-					"cs" => "Czech",
-					"hu" => "Hungarian",
-					"sk" => "Slovak",
-					"uk" => "Ukrainian",
-					"csb" => "Kashubian",
-					"tt" => "Tatar",
-					"ba" => "Bashkir",
-					"os" => "Ossetian",
-					"sl" => "Slovene",
-					"sr" => "Serbian",
-					"hr" => "Croatian",
-					"bs" => "Bosnian",
-					"bg" => "Bulgarian",
-					"sq" => "Albanian",
-					"ro" => "Romanian",
-					"mk" => "Macedonian",
-					"el" => "Greek",
-					"iw" => "Hebrew",
-					"fa" => "Persian",
-					"ar" => "Arabic",
-					"ku" => "Kurdish",
-					"az" => "Azerbaijani",
-					"hy" => "Armenian",
-					"af" => "Afrikaans",
-					"sw" => "Kiswahili",
-					"uz" => "Uzbek",
-					"kk" => "Kazakh",
-					"ky" => "Kyrgyz",
-					"tg" => "Tajik",
-					"tk" => "Turkmen",
-					"ug" => "Uyghurche",
-					"hi" => "Hindi",
-					"si" => "Sinhalese",
-					"gu" => "Gujarati",
-					"ur" => "Urdu",
-					"mr" => "Marathi",
-					"pa" => "Punjabi",
-					"bn" => "Bengali",
-					"ta" => "Tamil",
-					"te" => "Telugu",
-					"kn" => "Kannada",
-					"zh_CN" => "Chinese Simplified",
-					"zh_TW" => "Chinese Traditional",
-					"ko" => "Korean",
-					"cfr" => "Taiwanese",
-					"th" => "Thai",
-					"vi" => "Vietnamese",
-					"in" => "Indonesian",
-					"ms" => "Malay",
-					"tl" => "Tagalog",
-					"eo" => "Esperanto",
-					"ia" => "Interlingua",
-					"la" => "Latin"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/" . $query,
-					[]
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-			
-		}else{
-			$proxy = $this->backend->get_ip();
-			
-			$query = [
-				"q" => $get["s"],
-				"start" => 0,
-				"stime" => 92452189 // ?
-			];
-			
-			if($get["lang"] !== "any"){
-				
-				$query["lang"] = $get["lang"];
-			}
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/search",
-					$query
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		$nextpage =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"next-page",
-				"a"
-			);
-		
-		if(count($nextpage) !== 0){
-			
-			$nextpage =
-				$this->backend->store(
-					$nextpage[0]["attributes"]["href"],
-					"web",
-					$proxy
-				);
-		}else{
-			
-			$nextpage = null;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => $nextpage,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"site-item",
-				"div"
-			);
-		
-		foreach($items as $item){
-			
-			$this->fuckhtml->load($item);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByAttributeValue(
-					"target",
-					"_blank",
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName("site-descr");
-			
-			if(count($description) !== 0){
-				
-				$description =
-					$this->fuckhtml
-					->getTextContent(
-						$description[0]
-					);
-			}else{
-				
-				$description = null;
-			}
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-}

scraper/ddg.php

@@ -54,8 +54,8 @@ class ddg{
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
 					"Accept: */*",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip",
+					"Accept-Language: en-US,en;q=0.9",
+					"Accept-Encoding: gzip, deflate, br, zstd",
 					"Referer: https://duckduckgo.com/",
 					"DNT: 1",
 					"Sec-GPC: 1",
@@ -492,6 +492,17 @@ class ddg{
 				"form"
 			);
 		
+		foreach($forms as $form){
+			
+			if(
+				isset($form["attributes"]["id"]) &&
+				$form["attributes"]["id"] == "challenge-form"
+			){
+			
+				throw new Exception("DuckDuckGo returned an image captcha");
+			}
+		}
+		
 		foreach(array_reverse($forms) as $form){
 			
 			$this->fuckhtml->load($form);
@@ -909,112 +920,6 @@ class ddg{
 			){
 				
 				throw new Exception("DuckDuckGo returned a JSA challenge");
-				
-				// get JSA initial token
-				if(
-					!preg_match(
-						'/let jsa *= *([0-9]+)/',
-						$js,
-						$jsa
-					)
-				){
-					
-					$jsa = 0;
-				}else{
-					
-					$jsa = (int)$jsa[1];
-				}
-				
-				// get function bodies
-				preg_match_all(
-					'/let *([A-Za-z0-9]+) *= *function\(.*\) *{(.*)};/sU',
-					$js,
-					$functions
-				);
-				
-				$parsed_functions = [];
-				
-				for($i=0; $i<count($functions[0]); $i++){
-					
-					$functions[2][$i] = trim($functions[2][$i]);
-					
-					if(
-						preg_match(
-							'/return num *\* *([0-9]+)/i',
-							$functions[2][$i],
-							$num
-						)
-					){
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "multiplication",
-							"num" => (int)$num[1]
-						];
-						continue;
-					}
-					
-					if(
-						preg_match(
-							'/innerHTML *= *`([^`]+)`/i',
-							$functions[2][$i],
-							$challenge
-						)
-					){
-						
-						$challenge[1] =
-							preg_replace(
-								'/<\/(br)>/',
-								'<$1>',
-								$challenge[1]
-							);
-						
-						$parsed_functions[$functions[1][$i]] = [
-							"type" => "challenge",
-							"text" => $challenge[1]
-						];
-					}
-				}
-				
-				// get function call order
-				preg_match_all(
-					'/jsa *= *([A-Za-z0-9]+)\(jsa\)/i',
-					$js,
-					$call_order
-				);
-				
-				foreach($call_order[1] as $order){
-					
-					if(!isset($parsed_functions[$order])){
-						
-						throw new Exception("JS challenge solve failure: DuckDuckGo called an unknown function");
-					}
-					
-					if($parsed_functions[$order]["type"] == "multiplication"){
-						
-						$jsa = $jsa * $parsed_functions[$order]["num"];
-						continue;
-					}
-					
-					if($parsed_functions[$order]["type"] == "challenge"){
-						
-						// @TODO get parsed length
-						//$parsed_functions[$order]["text"]
-						
-						$jsa = $jsa + strlen($parsed_functions[$order]["text"]);
-					}
-				}
-				
-				try{
-					$js = $this->get(
-						$proxy,
-						"https://links.duckduckgo.com" . $challenge_url[1] . $jsa,
-						[],
-						ddg::req_xhr
-					);
-				}catch(Exception $error){
-					
-					throw new Exception("Failed to get challenged d.js");
-				}
 			}
 			
 			//
@@ -1858,7 +1763,7 @@ class ddg{
 					if(count($answer) !== 0){
 						
 						$out["answer"][] = [
-							"title" => ucfirst($word),
+							"title" => ucfirst(rawurldecode($word)),
 							"description" => $answer,
 							"url" => "https://www.wordnik.com/words/" . $word,
 							"thumb" => null,

scraper/ghostery.php

@@ -1,320 +0,0 @@
-<?php
-
-class ghostery{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("ghostery");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"any" => "All regions",
-					"AR" => "Argentina",
-					"AU" => "Australia",
-					"AT" => "Austria",
-					"BE" => "Belgium",
-					"BR" => "Brazil",
-					"CA" => "Canada",
-					"CL" => "Chile",
-					"DK" => "Denmark",
-					"FI" => "Finland",
-					"FR" => "France",
-					"DE" => "Germany",
-					"HK" => "Hong Kong",
-					"IN" => "India",
-					"ID" => "Indonesia",
-					"IT" => "Italy",
-					"JP" => "Japan",
-					"KR" => "Korea",
-					"MY" => "Malaysia",
-					"MX" => "Mexico",
-					"NL" => "Netherlands",
-					"NZ" => "New Zealand",
-					"NO" => "Norway",
-					"CN" => "People's Republic of China",
-					"PL" => "Poland",
-					"PT" => "Portugal",
-					"PH" => "Republic of the Philippines",
-					"RU" => "Russia",
-					"SA" => "Saudi Arabia",
-					"ZA" => "South Africa",
-					"ES" => "Spain",
-					"SE" => "Sweden",
-					"CH" => "Switzerland",
-					"TW" => "Taiwan",
-					"TR" => "Turkey",
-					"GB" => "United Kingdom",
-					"US" => "United States"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $country){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"Referer: https://ghosterysearch.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: same-origin",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"]
-		);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			parse_str($query, $query);
-			
-			// country
-			$country = $query["c"];
-			unset($query["c"]);
-			
-			$query = http_build_query($query);
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search?" . $query,
-						[],
-						$country
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search",
-						[
-							"q" => $get["s"]
-						],
-						$get["country"]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$this->fuckhtml->load($html);
-		
-		$results_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"results",
-				"section"
-			);
-		
-		if(count($results_wrapper) === 0){
-			
-			throw new Exception("Failed to grep result section");
-		}
-		
-		$this->fuckhtml->load($results_wrapper[0]);
-		
-		// get search results
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"li"
-			);
-		
-		if(count($results) === 0){
-			
-			return $out;
-		}
-		
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"url",
-					"a"
-				);
-			
-			if(count($a) === 0){
-				
-				continue;
-			}
-			
-			$a = $a[0];
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"h2"
-							)[0]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"p"
-							)[0]
-						)
-					),
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-						["attributes"]
-						["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		// get pagination token
-		$pagination_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"div"
-			);
-		
-		if(count($pagination_wrapper) !== 0){
-			
-			// found next page!
-			$this->fuckhtml->load($pagination_wrapper[0]);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				);
-			
-			if(count($a) !== 0){
-				
-				$q =
-					parse_url(
-						$this->fuckhtml
-						->getTextContent(
-							$a[count($a) - 1]
-							["attributes"]
-							["href"]
-						),
-						PHP_URL_QUERY
-					);
-				
-				$out["npt"] =
-					$this->backend
-					->store(
-						$q . "&c=" . $get["country"],
-						"web",
-						$proxy
-					);
-			}
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/google.php

@@ -12,6 +12,8 @@ class google{
 		
 		include "lib/backend.php";
 		$this->backend = new backend("google");
+		
+		$this->message = "Still working on a Google scraper that uses a headful browser. It will require Firefox + a webExtension running on a dedicated server. Waiting for my EDID adapter and we can get the show going. In the meantime, use the Google CSE/API or Yahoo JP/Startpage scrapers. They're all crippled in their own special ways but they're serviceable I guess.";
 	}
 	
 	public function getfilters($page){
@@ -505,7 +507,7 @@ class google{
 		}
 	}
 	
-	private function get($proxy, $url, $get = [], $alt_ua = false){
+	private function get($proxy, $url, $get = []){
 		
 		$curlproc = curl_init();
 		
@@ -518,18 +520,6 @@ class google{
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
 		
-		if($alt_ua === true){
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
-				"User-Agent: Mozilla/5.0 (iPhone; CPU iPhone OS 26_0_0 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) GSA/406.0.862495628 Mobile/15E148 Safari/604.1",
-				"Accept: text/html, application/xml;q=0.9, */*;q=0.8",
-				"Accept-Language: en-US,en;q=0.8",
-				"Accept-Encoding: gzip, deflate",
-				"Connection: Keep-Alive",
-				"Cache-Control: no-cache"
-			]);
-		}else{
-			
 		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 		curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
 			"User-Agent: " . config::USER_AGENT,
@@ -546,7 +536,6 @@ class google{
 			"Priority: u=1",
 			"TE: trailers"
 		]);
-		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -574,228 +563,22 @@ class google{
 	
 	public function web($get){
 		
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
-	}
-	
-	
-	public function video($get){
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
-	}
-	
-	
-	public function news($get){
-		throw new Exception("There are no known ways to scrape Google's /search endpoint without JS at this time. I'm working on a method that extracts cookies from browsers. Use Google API/CSE/Yahoo JP/Startpage for google results for now.");
+		throw new Exception($this->message);
 	}
 	
 	
 	public function image($get){
-		
-		// generate parameters
-		if($get["npt"]){
-			
-			[$params, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$params = json_decode($params, true);
-			
-			$page = $params["page"] + 1;
-			$params = $params["params"];
-			$params["async"] = "_fmt:json,p:1,ijn:{$page}";
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-			
-				throw new Exception("Search term is empty!");
+		throw new Exception($this->message);
 	}
 	
-			$proxy = $this->backend->get_ip();
-			$country = $get["country"];
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$size = $get["size"];
-			$ratio = $get["ratio"];
-			$color = $get["color"];
-			$type = $get["type"];
-			$format = $get["format"];
-			$rights = $get["rights"];
 	
-			$page = 0;
-			
-			$params = [
-				"q" => $search,
-				"tbm" => "isch",
-				"asearch" => "isch",
-				"async" => "_fmt:json,p:0,ijn:{$page}", // ijn:0 = page 1
-			];
-			
-			// country (image search uses cr instead of gl)
-			if($country != "any"){
-				
-				$params["cr"] = "country" . strtoupper($country);
+	public function video($get){
+		throw new Exception($this->message);
 	}
 	
-			// nsfw
-			$params["safe"] = $nsfw == "yes" ? "off" : "active";
 	
-			// generate tbs
-			$tbs = [];
-			
-			// time
-			if($time != "any"){
-				
-				$tbs["qdr"] = $time;
-			}
-			
-			// size
-			if($size != "any"){
-				
-				$params["imgsz"] = $size;
-			}
-			
-			// ratio
-			if($ratio != "any"){
-				
-				$params["imgar"] = $ratio;
-			}
-			
-			// color
-			if($color != "any"){
-				
-				if(
-					$color == "color" ||
-					$color == "trans"
-				){
-					
-					$params["imgc"] = $color;
-				}elseif($color == "bnw"){
-					
-					$params["imgc"] = "gray";
-				}else{
-					
-					$tbs["ic"] = "specific";
-					$tbs["isc"] = $color;
-				}
-			}
-			
-			// type
-			if($type != "any"){
-				
-				$tbs["itp"] = $type;
-			}
-			
-			// format
-			if($format != "any"){
-				
-				$params["as_filetype"] = $format;
-			}
-			
-			// rights (tbs)
-			if($rights != "any"){
-				
-				$tbs["sur"] = $rights;
-			}
-			
-			// append tbs
-			if(count($tbs) !== 0){
-				
-				$params["tbs"] = "";
-				
-				foreach($tbs as $key => $value){
-					
-					$params["tbs"] .= $key . ":" . $value . ",";
-				}
-				
-				$params["tbs"] = rtrim($params["tbs"], ",");
-			}
-		}
-		
-		try{
-			$json = 
-				$this->get(
-					$proxy,
-					"https://www.google.com/search",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to get search page");
-		}
-		
-		unset($params["async"]);
-		
-		//$json = file_get_contents("scraper/google.json");
-		
-		// detect captcha
-		$this->fuckhtml->load($json);
-		$this->detect_sorry();
-		
-		// remove xssi
-		$json =
-			preg_replace(
-				'/^[^{]*/',
-				"",
-				$json
-			);
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if(!isset($json["ischj"]["metadata"])){
-			
-			throw new Exception("Google did not return an image array");
-		}
-		
-		foreach($json["ischj"]["metadata"] as $image){
-			
-			$out["image"][] = [
-				"title" => $this->titledots($image["result"]["page_title"]),
-				"source" => [
-					[
-						"url" => $image["original_image"]["url"],
-						"width" => (int)$image["original_image"]["width"],
-						"height" => (int)$image["original_image"]["height"]
-					],
-					[
-						"url" => $image["thumbnail"]["url"],
-						"width" => (int)$image["thumbnail"]["width"],
-						"height" => (int)$image["thumbnail"]["height"]
-					]
-				],
-				"url" => $image["result"]["referrer_url"]
-			];
-		}
-		
-		$page++;
-		
-		if(count($out["image"]) === 10){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode([
-						"params" => $params,
-						"page" => $page
-					]),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
+	public function news($get){
+		throw new Exception($this->message);
 	}
 	
 	

scraper/google_api.php

@@ -264,6 +264,25 @@ class google_api{
 					"yes" => "Yes", // safe=active
 					"no" => "No" // safe=off
 				]
+			],
+			"sort" => [ // sort
+				"display" => "Sort by",
+				"option" => [
+					"any" => "Any order",
+					"date:d" => "Oldest",
+					"date:a" => "Newest"
+				]
+			],
+			"newer" => [
+				"display" => "Newer than",
+				"option" => "_DATE"
+			],			
+			"rm_dupes" => [ // filter
+				"display" => "Remove duplicates",
+				"option" => [
+					"yes" => "Yes", // 1
+					"no" => "No" // 0
+				]
 			]
 		];
 		
@@ -313,109 +332,29 @@ class google_api{
 								"zh-CN" => "Chinese (Simplified)",
 								"zh-TW" => "Chinese (Traditional)"
 							]
-						],
-						"sort" => [
-							"display" => "Sort by",
-							"option" => [
-								"any" => "Any order",
-								"date:d" => "Oldest",
-								"date:a" => "Newest"
-							]
-						],
-						"newer" => [
-							"display" => "Newer than",
-							"option" => "_DATE"
-						],
-						"rm_dupes" => [
-							"display" => "Remove duplicates",
-							"option" => [
-								"yes" => "Yes",
-								"no" => "No"
-							]
 						]
 					]
 				);
 				break;
-			/*
+			
 			case "images":
 				return array_merge(
 					$base,
 					[
-						"time" => [ // tbs=qdr:<time>
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"d" => "Past 24 hours",
-								"w" => "Past week",
-								"m" => "Past month",
-								"y" => "Past year"
-							]
-						],
-						"size" => [ // imgsz
+						"size" => [ // imgSize
 							"display" => "Size",
 							"option" => [
 								"any" => "Any size",
-								"l" => "Large",
-								"m" => "Medium",
-								"i" => "Icon",
-								"qsvga" => "Larger than 400x300",
-								"vga" => "Larger than 640x480",
-								"svga" => "Larger than 800x600",
-								"xga" => "Larger than 1024x768",
-								"2mp" => "Larger than 2MP",
-								"4mp" => "Larger than 4MP",
-								"6mp" => "Larger than 6MP",
-								"8mp" => "Larger than 8MP",
-								"10mp" => "Larger than 10MP",
-								"12mp" => "Larger than 12MP",
-								"15mp" => "Larger than 15MP",
-								"20mp" => "Larger than 20MP",
-								"40mp" => "Larger than 40MP",
-								"70mp" => "Larger than 70MP"
+								"icon" => "Icon",
+								"small" => "Small",
+								"medium" => "Medium",
+								"large" => "Large",
+								"xlarge" => "X-Large",
+								"xxlarge" => "XX-Large",
+								"huge" => "Huge"
 							]
 						],
-						"ratio" => [ // imgar
-							"display" => "Aspect ratio",
-							"option" => [
-								"any" => "Any ratio",
-								"t|xt" => "Tall",
-								"s" => "Square",
-								"w" => "Wide",
-								"xw" => "Panoramic"
-							]
-						],
-						"color" => [ // imgc
-							"display" => "Color",
-							"option" => [
-								"any" => "Any color",
-								"color" => "Full color",
-								"bnw" => "Black & white",
-								"trans" => "Transparent",
-								// from here, imgcolor
-								"red" => "Red",
-								"orange" => "Orange",
-								"yellow" => "Yellow",
-								"green" => "Green",
-								"teal" => "Teal",
-								"blue" => "Blue",
-								"purple" => "Purple",
-								"pink" => "Pink",
-								"white" => "White",
-								"gray" => "Gray",
-								"black" => "Black",
-								"brown" => "Brown"
-							]
-						],
-						"type" => [ // tbs=itp:<type>
-							"display" => "Type",
-							"option" => [
-								"any" => "Any type",
-								"clipart" => "Clip Art",
-								"lineart" => "Line Drawing",
-								"animated" => "Animated"
-							]
-						],
-						"format" => [ // as_filetype
+						"format" => [ // fileType
 							"display" => "Format",
 							"option" => [
 								"any" => "Any format",
@@ -429,17 +368,55 @@ class google_api{
 								"craw" => "RAW"
 							]
 						],
-						"rights" => [ // tbs=sur:<rights>
+						"color" => [
+							"display" => "Color",
+							"option" => [
+								"any" => "Any color",
+								
+								"color" => "Full color", // imgColorType
+								"mono" => "Black & White",
+								"trans" => "Transparent background",
+								
+								"red" => "Red", // imgDominantColor
+								"orange" => "Orange",
+								"yellow" => "Yellow",
+								"green" => "Green",
+								"teal" => "Teal",
+								"blue" => "Blue",
+								"purple" => "Purple",
+								"pink" => "Pink",
+								"white" => "White",
+								"gray" => "Gray",
+								"black" => "Black",
+								"brown" => "Brown"
+							]
+						],
+						"type" => [ // imgType
+							"display" => "Type",
+							"option" => [
+								"any" => "Any type",
+								"clipart" => "Clip Art",
+								"face" => "Faces",
+								"lineart" => "Line Drawing",
+								"stock" => "Stock photos",
+								"photo" => "Photos",
+								"animated" => "Animated",
+							]
+						],
+						"rights" => [ // rights
 							"display" => "Usage rights",
 							"option" => [
 								"any" => "Any license",
-								"cl" => "Creative Commons licenses",
-								"ol" => "Commercial & other licenses"
+								"cc_publicdomain" => "Public domain",
+								"cc_attribute" => "Attribution required",
+								"cc_sharealike" => "Sharealike",
+								"cc_noncommercial" => "Non-commercial use only",
+								"cc_nonderived" => "Original works"
 							]
 						]
 					]
 				);
-				break;*/
+				break;
 		}
 	}
 	
@@ -485,6 +462,7 @@ class google_api{
 		return $data;
 	}
 	
+	
 	public function web($get){
 		
 		// rotate proxy + key on EVERY request
@@ -731,6 +709,160 @@ class google_api{
 		return $out;
 	}
 	
+	
+	public function image($get){
+		
+		// rotate proxy + key on EVERY request
+		$keydata = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($keydata["increment"]);
+		
+		if($get["npt"]){
+			
+			// $p is never used
+			[$params, $p] = $this->backend->get(
+				$get["npt"],
+				"web"
+			);
+			
+			$params = json_decode($params, true);
+			
+			$params["key"] = $keydata["key"];
+			
+		}else{
+			
+			//$json = file_get_contents("scraper/google.json");
+			$params = [
+				"q" => $get["s"],
+				"cx" => config::GOOGLE_CX_ENDPOINT,
+				"num" => 10,
+				"start" => 1,
+				"searchType" => "image",
+				"key" => $keydata["key"]
+			];
+			
+			//
+			// parse filters
+			//
+			if($get["newer"] !== false){
+				
+				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
+			}
+			
+			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
+			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
+			
+			if($get["nsfw"] == "yes"){
+				
+				$params["safe"] = "off";
+			}else{
+				
+				$params["safe"] = "active";
+			}
+			
+			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
+			
+			// image filters
+			if($get["size"] != "any"){ $params["imgSize"] = $get["size"]; }
+			if($get["format"] != "any"){ $params["fileType"] = $get["format"]; }
+			
+			switch($get["color"]){
+				
+				case "any":
+					break;
+				
+				case "color":
+				case "mono":
+				case "trans":
+					$params["imgColorType"] = $get["color"];
+					break;
+				
+				default:
+					$params["imgDominantColor"] = $get["color"];
+					break;
+			}
+			
+			if($get["type"] != "any"){ $params["imgType"] = $get["type"]; }
+			if($get["rights"] != "any"){ $params["rights"] = $get["rights"]; }
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://www.googleapis.com/customsearch/v1",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(isset($json["error"]["message"])){
+			
+			throw new Exception(
+				"API returned an error: " .
+				$json["error"]["message"] .
+				" (key #" . $keydata["increment"] . ")"
+			);
+		}
+		
+		if(!isset($json["items"])){
+			
+			// google just doesnt return items when theres no results
+			return $out;
+		}
+		
+		foreach($json["items"] as $image){
+			
+			$out["image"][] = [
+				"title" => $this->titledots($image["title"]),
+				"source" => [
+					[
+						"url" => $image["link"],
+						"width" => (int)$image["image"]["width"],
+						"height" => (int)$image["image"]["height"]
+					],
+					[
+						"url" => $image["image"]["thumbnailLink"],
+						"width" => (int)$image["image"]["thumbnailWidth"],
+						"height" => (int)$image["image"]["thumbnailHeight"]
+					]
+				],
+				"url" => $image["image"]["contextLink"]
+			];
+		}
+		
+		// get npt
+		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
+			
+			unset($params["key"]);
+			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	
 	private function titledots($title){
 		
 		return trim($title, " .\t\n\r\0\x0B…");

scraper/greppr.php

@@ -1,452 +0,0 @@
-<?php
-// greppr dev probably monitors 4get code, lol
-// hello greppr dude, add an API you moron
-
-class greppr{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("greppr");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = [], $cookies = [], $post = false){
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		$cookie = [];
-		foreach($cookies as $k => $v){
-			
-			$cookie[] = "{$k}={$v}";
-		}
-		
-		$cookie = implode("; ", $cookie);
-		
-		if($post === false){
-						
-			if($get !== []){
-				$get = http_build_query($get);
-				$url .= "?" . $get;
-			}
-			
-			if($cookie == ""){
-				
-				curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-					["User-Agent: " . config::USER_AGENT,
-					"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip",
-					"DNT: 1",
-					"Connection: keep-alive",
-					"Upgrade-Insecure-Requests: 1",
-					"Sec-Fetch-Dest: document",
-					"Sec-Fetch-Mode: navigate",
-					"Sec-Fetch-Site: none",
-					"Sec-Fetch-User: ?1"]
-				);
-			}else{
-				
-				curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-					["User-Agent: " . config::USER_AGENT,
-					"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-					"Accept-Language: en-US,en;q=0.5",
-					"Accept-Encoding: gzip, deflate, br, zstd",
-					"DNT: 1",
-					"Sec-GPC: 1",
-					"Connection: keep-alive",
-					"Referer: https://greppr.org/search",
-					"Cookie: {$cookie}",
-					"Upgrade-Insecure-Requests: 1",
-					"Sec-Fetch-Dest: document",
-					"Sec-Fetch-Mode: navigate",
-					"Sec-Fetch-Site: same-origin",
-					"Sec-Fetch-User: ?1",
-					"Priority: u=0, i"]
-				);
-			}
-		}else{
-			
-			$get = http_build_query($get);
-			
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"Content-Type: application/x-www-form-urlencoded",
-				"Content-Length: " . strlen($get),
-				"Origin: https://greppr.org",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Referer: https://greppr.org/",
-				"Cookie: {$cookie}",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i"]
-			);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$headers = [];
-		
-		curl_setopt(
-			$curlproc,
-			CURLOPT_HEADERFUNCTION,
-			function($curlproc, $header) use (&$headers){
-				
-				$len = strlen($header);
-				$header = explode(':', $header, 2);
-				
-				if(count($header) < 2){
-					
-					// ignore invalid headers
-					return $len;
-				}
-				
-				$headers[strtolower(trim($header[0]))][] = trim($header[1]);
-
-				return $len;
-			}
-		);
-				
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		
-		return [
-			"headers" => $headers,
-			"data" => $data
-		];
-	}
-	
-	public function web($get, $first_attempt = true){
-		
-		if($get["npt"]){
-			
-			[$q, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			$tokens = json_decode($q, true);
-			
-			//
-			// Get paginated page
-			//
-			try{
-			
-				$html = $this->get(
-					$proxy,
-					"https://greppr.org" . $tokens["get"],
-					[],
-					$tokens["cookies"],
-					false
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			//
-			// get token
-			//
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://greppr.org",
-						[],
-						[],
-						false
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch homepage");
-			}
-			
-			//
-			// Parse token
-			//
-			$this->fuckhtml->load($html["data"]);
-		
-			$tokens = [
-				"req" => null,
-				"data" => null,
-				"cookies" => null
-			];
-			
-			$inputs =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"input"
-				);
-				
-			foreach($inputs as $input){
-				
-				if(!isset($input["attributes"]["name"])){
-					
-					continue;
-				}
-				
-				if(
-					isset($input["attributes"]["value"]) &&
-					!empty($input["attributes"]["value"])
-				){
-					
-					$tokens
-						["data"]
-						[$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["name"]
-						)] =
-						$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["value"]
-						);
-				}else{
-					
-					$tokens["req"] =
-						$this->fuckhtml
-						->getTextContent(
-							$input["attributes"]["name"]
-						);
-				}
-			}
-			
-			if($tokens["req"] === null){
-				
-				throw new Exception("Failed to get request ID");
-			}
-			
-			if(isset($html["headers"]["set-cookie"])){
-				
-				foreach($html["headers"]["set-cookie"] as $cookie){
-					
-					if(
-						preg_match(
-							'/([^=]+)=([^;]+)/',
-							$cookie,
-							$matches
-						)
-					){
-						
-						$tokens["cookies"][$matches[1]] = $matches[2];
-					}
-				}
-			}
-			
-			//
-			// Get initial search page
-			//
-			$tokens_req = $tokens["data"];
-			$tokens_req[$tokens["req"]] = $search;
-			
-			try{
-				
-				$html = $this->get(
-					$proxy,
-					"https://greppr.org/search",
-					$tokens_req,
-					$tokens["cookies"],
-					true
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		//$html = file_get_contents("scraper/greppr.html");
-		//$this->fuckhtml->load($html);
-		$this->fuckhtml->load($html["data"]);
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		// get results for later
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"div"
-			);
-		
-		// check for next page
-		$next_elem =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"ul"
-			);
-		
-		if(count($next_elem) !== 0){
-			
-			$this->fuckhtml->load($next_elem[0]);
-			
-			$as =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"page-link",
-					"a"
-				);
-			
-			$break = false;
-			foreach($as as $a){
-				
-				if($break === true){
-					
-					$out["npt"] =
-						$this->backend->store(
-							json_encode([
-								"get" =>
-									$this->fuckhtml
-									->getTextContent(
-										$a["attributes"]["href"]
-									),
-								"cookies" => $tokens["cookies"]
-							]),
-							"web",
-							$proxy
-						);
-					break;
-				}
-				
-				if($a["attributes"]["href"] == "#"){
-					
-					$break = true;
-				}
-			}
-		}
-		
-		// scrape results
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"highlightedDesc",
-					"p"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->limitstrlen(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$date =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"p"
-				);
-			
-			$date =
-				strtotime(
-					explode(
-						":",
-						$this->fuckhtml
-						->getTextContent(
-							$date[count($date) - 1]["innerHTML"]
-						)
-					)[1]
-				);
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["innerHTML"]
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => $date,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function limitstrlen($text){
-		
-		return explode("\n", wordwrap($text, 300, "\n"))[0];
-	}
-}

scraper/marginalia.php

@@ -289,8 +289,18 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
+			if(strpos($json, "<title>Bad Gateway (502)</title>")){
+				
+				throw new Exception("Marginalia returned a 502 error");
+			}
+			
 			$json = json_decode($json, true);
 			
+			if($json === null){
+				
+				throw new Exception("Failed to decode JSON");
+			}
+			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -490,6 +500,37 @@ class marginalia{
 			return $this->web($get, $redirect);
 		}
 		
+		// detect internal error
+		$infobox =
+			$this->fuckhtml
+			->getElementsByClassName("infobox");
+		
+		if(count($infobox) !== 0){
+			
+			foreach($infobox as $i){
+				
+				$this->fuckhtml->load($i);
+				
+				$h2 =
+					$this->fuckhtml
+					->getElementsByTagName("h2");
+				
+				if(
+					count($h2) !== 0 &&
+					$this->fuckhtml
+					->getTextContent(
+						$h2[0]
+					) == "Internal error"
+				){
+					
+					throw new Exception("Marginalia returned an internal server error");
+				}
+			}
+						
+			// reset
+			$this->fuckhtml->load($html);
+		}
+		
 		$sections =
 			$this->fuckhtml
 			->getElementsByClassName(

scraper/mojeek.php

@@ -340,7 +340,7 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],
+					],/*
 					"region" => [
 						"display" => "Region",
 						"option" => [
@@ -350,7 +350,7 @@ class mojeek{
 							"fr" => "France",
 							"uk" => "United Kingdom"
 						]
-					],
+					],*/
 					"domain" => [
 						"display" => "Results per domain",
 						"option" => [
@@ -458,7 +458,7 @@ class mojeek{
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
 				//"dlen" => 511, // max length of description
-				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -503,6 +503,11 @@ class mojeek{
 			}
 		}
 		
+		if(strlen($html) === 0){
+			
+			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [

scraper/pexels.php

@@ -0,0 +1,285 @@
+<?php
+
+class pexels{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("pexels");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"time" => [ // date_from
+				"display" => "Time posted",
+				"option" => [
+					"any" => "Any time",
+					"last_24_hours" => "Last 24 hours",
+					"last_week" => "Last week",
+					"last_month" => "Last month",
+					"last_year" => "Last year"
+				]
+			],
+			"orientation" => [
+				"display" => "Orientation",
+				"option" => [
+					"any" => "Any orientation",
+					"landscape" => "Horizontal",
+					"portrait" => "Vertical",
+					"square" => "Square"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					"795548" => "Brown",
+					"F44336" => "Red",
+					"E91E63" => "Hot pink",
+					"9C27B0" => "Magenta",
+					"673AB7" => "Purple",
+					"3F51B5" => "Indigo",
+					"2196F3" => "Blue",
+					"03A9F4" => "Light blue",
+					"00BCD4" => "Cyan",
+					"009688" => "Forest green",
+					"4CAF50" => "Green",
+					"8BC34A" => "Lime",
+					"CDDC39" => "Pear",
+					"FFEB3B" => "Yellow",
+					"FFC107" => "Gold",
+					"FF9800" => "Orange",
+					"FF5722" => "Tomato",
+					"9E9E9E" => "Gray",
+					"607D8B" => "Teal",
+					"000000" => "Black",
+					"FFFFFF" => "White"
+				]
+			],
+			"people_count" => [
+				"display" => "Head count",
+				"option" => [
+					"any" => "Any number",
+					"0" => "0",
+					"1" => "1",
+					"2" => "2",
+					"3_plus" => "3+",
+				]
+			],
+			"people_age" => [
+				"display" => "People's age",
+				"option" => [
+					"any" => "Any age",
+					"baby" => "Baby",
+					"child" => "Child",
+					"teenager" => "Teenager",
+					"adult" => "Adult",
+					"senior_adult" => "Senior adult"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		$search = $get["query"];
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: https://www.pexels.com/search/" . rawurlencode($search) . "/",
+			"Content-Type: application/json",
+			"secret-key: H2jk9uKnhRmL6WPwh89zBezWvr", // hardcoded but like, people on github have been using this shit since 23'
+			"X-Client-Type: react",
+			"X-Next-Forwarded-CF-Connecting-IP: ",
+			"X-Next-Forwarded-CF-IPCountry: ",
+			"X-Next-Forwarded-CF-IPRegionCode: ",
+			"X-Active-Experiment: ",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Alt-Used: www.pexels.com",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=4",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filter, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$filter = json_decode($filter, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			// ?query=blue%20footed%20booby&page=1&per_page=24&seo_tags=true
+			$filter = [
+				"query" => $search,
+				"page" => 1,
+				"per_page" => 24,
+				"seo_tags" => "true"
+			];
+			
+			// add filters
+			if($get["time"] != "any"){
+				
+				$filter["date_from"] = $get["time"];
+			}
+			
+			if($get["orientation"] != "any"){
+				
+				$filter["orientation"] = $get["orientation"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				$filter["color"] = $get["color"];
+			}
+			
+			if($get["people_count"] != "any"){
+				
+				$filter["people_count"] = $get["people_count"];
+			}
+			
+			if($get["people_age"] != "any"){
+				
+				$filter["people_age"] = $get["people_age"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		try{
+			$html =
+				$this->get(
+					$proxy,
+					"https://www.pexels.com/en-us/api/v3/search/photos",
+					$filter
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($html, true);
+		
+		if(!isset($json["data"])){
+			
+			throw new Exception("Pexels did not return a data object");
+		}
+		
+		foreach($json["data"] as $image){
+			
+			$thumb_size =
+				$this->imgratio(
+					(int)$image["attributes"]["width"],
+					(int)$image["attributes"]["height"],
+					350
+				);
+			
+			$out["image"][] = [
+				"title" => $image["attributes"]["title"] . ": " . $image["attributes"]["description"],
+				"source" => [
+					[
+						"url" => $image["attributes"]["image"]["download_link"],
+						"width" => (int)$image["attributes"]["width"],
+						"height" => (int)$image["attributes"]["height"]
+					],
+					[
+						"url" =>
+							preg_replace(
+								'/(?:w|h)=[0-9]+$/',
+								"w=350",
+								$image["attributes"]["image"]["small"]
+							),
+						"width" => $thumb_size[0],
+						"height" => $thumb_size[1]
+					]
+				],
+				"url" =>
+					"https://pexels.com/photo/" .
+					$image["attributes"]["slug"] . "-" .
+					$image["attributes"]["id"] . "/"
+			];
+		}
+		
+		// get next page
+		if((int)$json["pagination"]["current_page"] < (int)$json["pagination"]["total_pages"]){
+			
+			$filter["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filter),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/pinterest.php

@@ -297,6 +297,14 @@ class pinterest{
 			throw new Exception("Failed to decode JSON");
 		}
 		
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
@@ -426,7 +434,7 @@ class pinterest{
 							]
 						],
 						"url" =>
-							$item["link"] === null ?
+							!isset($item["link"]) ?
 							"https://ca.pinterest.com/pin/" . $item["id"] :
 							$item["link"]
 					];

scraper/pixabay.php

@@ -0,0 +1,396 @@
+<?php
+
+class pixabay{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("pixabay");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes",
+					"no" => "No"
+				]
+			],
+			"time" => [
+				"display" => "Time posted",
+				"option" => [
+					"any" => "Any time",
+					"1d" => "< 24 hours",
+					"3d" => "< 72 hours",
+					"1w" => "< 7 days",
+					"6m" => "< 6 months",
+					"1y" => "< 12 months"
+				]
+			],
+			"category" => [
+				"display" => "Category",
+				"option" => [
+					"images" => "All images",
+					"photos" => "Photos",
+					"illustrations" => "Illustrations",
+					"vectors" => "Vectors",
+					"gifs" => "GIFs"
+				]
+			],
+			"content_type" => [
+				"display" => "Authenticity",
+				"option" => [
+					"authentic" => "Authentic only",
+					"show_all" => "Authentic + AI slop",
+					"ai" => "AI slop only"
+				]
+			],
+			"order" => [
+				"display" => "Sort by",
+				"option" => [
+					"relevance" => "Most relevant",
+					"latest" => "Latest",
+					"ec" => "Editor's choice",
+					"trending" => "Trending"
+				]
+			],
+			"orientation" => [
+				"display" => "Orientation",
+				"option" => [
+					"any" => "Any orientation",
+					"vertical" => "Vertical",
+					"horizontal" => "Horizontal"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					"transparent" => "Transparent",
+					"grayscale" => "Grayscale",
+					"red" => "Red",
+					"orange" => "Orange",
+					"yellow" => "Yellow",
+					"green" => "Green",
+					"turquoise" => "Turquoise",
+					"blue" => "Blue",
+					"brown" => "Brown",
+					"black" => "Black",
+					"gray" => "Gray",
+					"white" => "White",
+					"pink" => "Pink",
+					"lilac" => "Lilac"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $nsfw = true){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT . " Pixabay",
+			"Accept: application/json",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: {$url}",
+			"Cookie: " . ($nsfw ? "g_rated=off" : "g_rated=1"),
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: no-cors",
+			"Sec-Fetch-Site: same-origin",
+			"x-fetch-bootstrap: 1",
+			"Alt-Used: pixabay.com",
+			"Priority: u=0",
+			"Pragma: no-cache",
+			"Cache-Control: no-cache"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$npt, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$npt = json_decode($npt, true);
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$npt = [
+				"s" => $search,
+				"pagi" => 1,
+				"category" => $get["category"]
+			];
+			
+			// filters
+			if($get["content_type"] != "show_all"){
+				
+				$npt["content_type"] = $get["content_type"];
+			}
+			
+			if($get["order"] != "relevance"){
+				
+				$npt["order"] = $get["order"];
+			}
+			
+			$npt["nsfw"] = $get["nsfw"] == "yes" ? true : false;
+			
+			if($get["orientation"] != "any"){
+				
+				$npt["orientation"] = $get["orientation"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				$npt["colors"] = $get["color"];
+			}
+			
+			if($get["time"] != "any"){
+				
+				$npt["date"] = $get["time"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		// https://pixabay.com/images/search/japan/?pagi=1
+		
+		$npt_pass = $npt;
+		unset($npt_pass["s"]);
+		unset($npt_pass["category"]);
+		unset($npt_pass["nsfw"]);
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://pixabay.com/{$npt["category"]}/search/" . rawurlencode($npt["s"]) . "/",
+					$npt_pass,
+					$npt["nsfw"]
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["page"]["results"])){
+			
+			throw new Exception("Pixabay API did not return a results object");
+		}
+		
+		//print_r($json);
+		
+		foreach($json["page"]["results"] as $image){
+			
+			if(
+				!(
+					$image["mediaType"] == "photo" ||
+					$image["mediaType"] == "animation"
+				)
+			){
+				
+				continue;
+			}
+			
+			// handle images that supply canvaRetouchUrl
+			if(
+				isset($image["canvaRetouchUrl"]) &&
+				$image["canvaRetouchUrl"] !== null
+			){
+				
+				parse_str(
+					parse_url(
+						$image["canvaRetouchUrl"],
+						PHP_URL_QUERY
+					),
+					$base
+				);
+				
+				if(!isset($base["image-url"])){
+					
+					// should not happen
+					continue;
+				}
+				
+				// get extension
+				$base = $base["image-url"];
+				
+				$parsed_base = parse_url($base);
+				
+				$ext =
+					explode(
+						".",
+						$parsed_base["path"]
+					);
+				
+				$ext = $ext[count($ext) - 1];
+				
+				// restructure url
+				$base =
+					explode(
+						"_",
+						$base
+					);
+				
+				unset($base[count($base) - 1]);
+				$base = implode("_", $base);
+				
+				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
+				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
+				
+				$source = [
+					[
+						"url" => "{$base}_1920.{$ext}",
+						"width" => $ratio_2x[0],
+						"height" => $ratio_2x[1]
+					],
+					[
+						"url" => "{$base}_180.{$ext}",
+						"width" => $ratio_1x[0],
+						"height" => $ratio_1x[1]
+					]
+				];
+			}else{
+				
+				// get intended sizes if canva is not set
+				// get 1x
+				$source_1x = null;
+				if(isset($image["sources"]["image_1x"])){ $source_1x = $image["sources"]["image_1x"]; }
+				elseif(isset($image["sources"]["1x"])){ $source_1x = $image["sources"]["1x"]; }
+				
+				if($source_1x === null){ continue; } // should not happen
+				
+				preg_match(
+					'/_([0-9]+)\./',
+					$image["sources"]["1x"],
+					$size_1x
+				);
+				
+				$size_1x = (int)$size_1x[1];
+				
+				// get 2x
+				$source_2x = null;
+				if(isset($image["sources"]["gif_2x"])){ $source_2x = $image["sources"]["gif_2x"]; }
+				elseif(isset($image["sources"]["2x"])){ $source_2x = $image["sources"]["2x"]; }
+				
+				if($source_2x === null){ continue; } // should not happen
+				
+				preg_match(
+					'/_([0-9]+)\./',
+					$source_2x,
+					$size_2x
+				);
+				
+				$size_2x = (int)$size_2x[1];
+				
+				// compute ratios
+				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_1x);
+				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_2x);
+				
+				// handle images that only give normal thumbnails
+				$source = [
+					[
+						"url" => $source_2x,
+						"width" => $ratio_2x[0],
+						"height" => $ratio_2x[1]
+					],
+					[
+						"url" => $source_1x,
+						"width" => $ratio_1x[0],
+						"height" => $ratio_1x[1]
+					]
+				];
+			}
+			
+			$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
+			$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
+			
+			$out["image"][] = [
+				"title" => $image["name"],
+				"source" => $source,
+				"url" => "https://pixabay.com" . $image["href"]
+			];
+		}
+		
+		// add next page
+		if((int)$json["page"]["page"] < (int)$json["page"]["pages"]){
+			
+			$npt["pagi"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($npt),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/qwant.php

@@ -212,29 +212,59 @@ class qwant{
 		return $base;
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){
+		
+		$curlproc = curl_init();
+		
+		if($is_post){
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://www.qwant.com/",
+				"Content-type: application/x-www-form-urlencoded",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache"
+			];
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get));
+		}else{
 			
 			$headers = [
 				"User-Agent: " . config::USER_AGENT,
 				"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Origin: https://www.qwant.com",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"Referer: https://www.qwant.com/",
+				"Origin: https://www.qwant.com",
+				"DNT: 1",
+				($cookie !== null ? "Cookie: {$cookie}" : ""),
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
 				"Sec-Fetch-Dest: empty",
 				"Sec-Fetch-Mode: cors",
 				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
 				"TE: trailers"
 			];
 						
-		$curlproc = curl_init();
-		
 			if($get !== []){
 				$get = http_build_query($get);
 				$url .= "?" . $get;
 			}
+		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
@@ -271,6 +301,8 @@ class qwant{
 			[$params, $proxy] = $this->backend->get($get["npt"], "web");
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
 			
 		}else{
 			
@@ -297,7 +329,8 @@ class qwant{
 				"device" => "desktop",
 				"tgp" => 3,
 				"safesearch" => 0,
-				"displayed" => "true"
+				"displayed" => "true",
+				"llm" => "false"
 			];
 			
 			switch($get["nsfw"]){
@@ -306,18 +339,24 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		/*
 		$handle = fopen("scraper/qwant_web.json", "r");
 		$json = fread($handle, filesize("scraper/qwant_web.json"));
 		fclose($handle);*/
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json =
 				$this->get(
 					$proxy,
-					"https://fdn.qwant.com/v3/search/web",
-					$params
+					"https://api.qwant.com/v3/search/web",
+					$params,
+					false,
+					$cookie
 				);
 			
 		}catch(Exception $error){
@@ -353,6 +392,8 @@ class qwant{
 			"related" => []
 		];
 		
+		$this->detect_errors($json);
+		
 		if(
 			$json["status"] != "success" &&
 			$json["data"]["error_code"] === 5
@@ -362,8 +403,6 @@ class qwant{
 			return $out;
 		}
 		
-		$this->detect_errors($json);
-		
 		if(!isset($json["data"]["result"]["items"]["mainline"])){
 			
 			throw new Exception("Server did not return a result object");
@@ -436,6 +475,7 @@ class qwant{
 		if($json["data"]["result"]["lastPage"] === false){
 			
 			$params["offset"] = $params["offset"] + 10;
+			$params["cookie"] = $cookie;
 			
 			$out["npt"] =
 				$this->backend->store(
@@ -577,6 +617,9 @@ class qwant{
 				);
 			
 			$params = json_decode($params, true);
+			$cookie = $params["cookie"];
+			unset($params["cookie"]);
+			
 		}else{
 			
 			$search = $get["s"];
@@ -617,13 +660,19 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
+			
+			$cookie = null;
 		}
 		
+		$cookie = $this->fuck_datadome($proxy, $search);
+		
 		try{
 			$json = $this->get(
 				$proxy,
 				"https://api.qwant.com/v3/search/images",
 				$params,
+				false,
+				$cookie
 			);
 		}catch(Exception $err){
 			
@@ -1001,4 +1050,53 @@ class qwant{
 		
 		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
 	}
+	
+	private function fuck_datadome($proxy, $search){
+		
+		if($cookie === null){
+			// get datadome bullshit
+			// yeah go ahead and patch this you code monkey
+			
+			// ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result?
+			// i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum
+			// also, stop pretending you guys have your own index, this is clearly all bing results
+			try{
+				$datadome =
+					$this->get(
+						$proxy,
+						"https://dd.qwant.com/js/",
+						[
+							"jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA",
+							"eventCounters" => [],
+							"jsType" => "ch",
+							"cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe",
+							"ddk" => "78B13B7513D180B7AB6D6FF9EB0A51",
+							"Referer" => "https://www.qwant.com/?q={$search}&t=web",
+							"request" => "/?q={$search}&t=web",
+							"responsePage" => "origin",
+							"ddv" => "5.8.0"
+						],
+						true
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JS challenge token");
+			}
+			
+			$datadome = json_decode($datadome, true);
+			if($datadome === null){
+				
+				throw new Exception("Failed to decode JS challenge JSON");
+			}
+			
+			if(!isset($datadome["cookie"])){
+				
+				throw new Exception("Failed to get cookie from JS challenge endpoint");
+			}
+			
+			$cookie = explode(";", $datadome["cookie"])[0];
+		}
+		
+		return $cookie;
+	}
 }

scraper/startpage.php

@@ -408,6 +408,8 @@ class startpage{
 			//$html = file_get_contents("scraper/startpage.html");
 		}
 		
+		//echo htmlspecialchars($html);
+		
 		$this->detect_captcha($html);
 		
 		if(
@@ -564,12 +566,16 @@ class startpage{
 					break;
 				
 				case "spellsuggest-google":
+					
+					if(isset($category["results"][0]["query"])){
+						
 						$out["spelling"] =
 							[
 								"type" => "including",
-							"using" => $json["render"]["query"],
-							"correction" => $category["results"][0]["query"]
+								"using" => rawurldecode($json["render"]["query"]),
+								"correction" => html_entity_decode($category["results"][0]["query"])
 							];
+					}
 					break;
 				
 				case "dictionary-qi":
@@ -645,318 +651,6 @@ class startpage{
 			}
 		}
 		
-		// parse instant answers
-		if(
-			$get["extendedsearch"] == "yes" &&
-			$get_instant_answer === true
-		){
-			
-			// https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=BqZ3inqrAgF701&sr=1
-			try{
-				$post = [
-					"se" => "n0vze2y9dqwy",
-					"q" => $json["render"]["query"],
-					"results" => [], // populate
-					"enableKnowledgePanel" => true,
-					"enableMediaThumbBar" => false,
-					"enableSearchSuggestions" => false,
-					"enableTripadvisorProperties" => [],
-					"enableTripadvisorPlaces" => [],
-					"enableTripadvisorPlacesForLocations" => [],
-					"enableWebProducts" => false,
-					"tripadvisorPartnerId" => null,
-					"tripadvisorMapColorMode" => "light",
-					"tripadvisorDisablesKnowledgePanel" => false,
-					"instantAnswers" => [
-						"smartAnswers",
-						"youtube",
-						"tripadvisor"
-					],
-					"iaType" => null,
-					"forceEnhancedKnowledgePanel" => false,
-					"shoppingOnly" => false,
-					"allowAdultProducts" => true,
-					"lang" => "en",
-					"browserLang" => "en-US",
-					"browserTimezone" => "America/New_York",
-					"market" => null,
-					"userLocation" => null,
-					"userDate" => date("Y-m-d"),
-					"userAgentType" => "unknown"
-				];
-				
-				foreach($out["web"] as $result){
-					
-					$post["results"][] = [
-						"url" => $result["url"],
-						"title" => $result["title"]
-					];
-				}
-				
-				$post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
-				
-				$additional_data =
-					$this->get(
-						$proxy,
-						"https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
-						$post,
-						true,
-						true
-					);
-				
-				$additional_data = json_decode($additional_data, true);
-				
-				if($additional_data === null){
-					
-					throw new Exception("Failed to decode JSON"); // just break out, dont fail completely
-				}
-				
-				if(!isset($additional_data["knowledgePanel"])){
-					
-					throw new Exception("Response has missing data (knowledgePanel)");
-				}
-				
-				$additional_data = $additional_data["knowledgePanel"];
-				
-				$answer = [
-					"title" => $additional_data["meta"]["title"],
-					"description" => [
-						[
-							"type" => "quote",
-							"value" => $additional_data["meta"]["description"]
-						]
-					],
-					"url" => $additional_data["meta"]["origWikiUrl"],
-					"thumb" => $additional_data["meta"]["image"],
-					"table" => [],
-					"sublink" => []
-				];
-				
-				// parse html for instant answer
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				$div =
-					$this->fuckhtml
-					->getElementsByTagName(
-						"div"
-					);
-				
-				// get description
-				$description =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-short-extract sx-kp-short-extract-complete",
-						$div
-					);
-				
-				if(count($description) !== 0){
-					
-					$answer["description"][] = [
-						"type" => "text",
-						"value" =>
-							html_entity_decode(
-								$this->fuckhtml
-								->getTextContent(
-									$description[0]
-								)
-							)
-					];
-				}
-				
-				// get socials
-				$socials =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-wiki-social-link",
-						"a"
-					);
-				
-				foreach($socials as $social){
-					
-					$title =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["title"]
-						);
-					
-					$url =
-						$this->fuckhtml
-						->getTextContent(
-							$social["attributes"]["href"]
-						);
-					
-					switch($title){
-						
-						case "Official Website":
-							$title = "Website";
-							break;
-					}
-					
-					$answer["sublink"][$title] = $url;
-				}
-				
-				// get videos
-				$videos =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-kp-video-grid-item",
-						$div
-					);
-				
-				foreach($videos as $video){
-					
-					$this->fuckhtml->load($video);
-					
-					$as =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"a"
-						);
-					
-					if(count($as) === 0){
-						
-						// ?? invalid
-						continue;
-					}
-					
-					$image =
-						$this->fuckhtml
-						->getElementsByAttributeName(
-							"data-sx-src",
-							"img"
-						);
-					
-					if(count($image) !== 0){
-						
-						$thumb = [
-							"ratio" => "16:9",
-							"url" =>
-								$this->fuckhtml
-								->getTextContent(
-									$image[0]["attributes"]["data-sx-src"]
-								)
-						];
-					}else{
-
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["video"][] = [
-						"title" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["title"]
-							),
-						"description" => null,
-						"date" => null,
-						"duration" => null,
-						"views" => null,
-						"thumb" => $thumb,
-						"url" =>
-							$this->fuckhtml
-							->getTextContent(
-								$as[0]["attributes"]["href"]
-							)
-					];
-				}
-				
-				// reset
-				$this->fuckhtml->load($additional_data["html"]);
-				
-				// get table elements
-				$table =
-					$this->fuckhtml
-					->getElementsByClassName(
-						"sx-infobox",
-						"table"
-					);
-				
-				if(count($table) !== 0){
-					
-					$trs =
-						$this->fuckhtml
-						->getElementsByTagName(
-							"tr"
-						);
-					
-					foreach($trs as $tr){
-						
-						$this->fuckhtml->load($tr);
-						
-						// ok so startpage devs cant fucking code a table
-						// td = content
-						// th (AAAHH) = title
-						$tds =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"td"	
-							);
-						
-						$ths =
-							$this->fuckhtml
-							->getElementsByTagName(
-								"th"
-							);
-						
-						if(
-							count($ths) === 1 &&
-							count($tds) === 1
-						){
-							
-							$title =
-								$this->fuckhtml
-								->getTextContent(
-									$ths[0]
-								);
-							
-							$description = [];
-							
-							$this->fuckhtml->load($tds[0]);
-							
-							$lis =
-								$this->fuckhtml
-								->getElementsByTagName(
-									"li"
-								);
-							
-							if(count($lis) !== 0){
-								
-								foreach($lis as $li){
-									
-									$description[] =
-										$this->fuckhtml
-										->getTextContent(
-											$li
-										);
-								}
-								
-								$description = implode(", ", $description);
-							}else{
-								
-								$description =
-									$this->fuckhtml
-									->getTextContent(
-										$tds[0]
-									);
-							}
-							
-							$answer["table"][$title] = $description;
-						}
-					}
-				}
-				
-				$out["answer"][] = $answer;
-				
-			}catch(Exception $error){
-				
-				// do nothing
-				//echo "error!";
-			}
-		}
-		
 		return $out;
 	}
 	
@@ -1413,27 +1107,22 @@ class startpage{
 			
 			if($page["name"] == "Next"){
 				
-				parse_str(
-					explode(
-						"?",
-						$page["url"],
-						2
-					)[1],
-					$str
-				);
-				
 				return
 					$this->backend->store(
 						http_build_query(
 							[
 								"lui" => "english",
 								"language" => "english",
-								"query" => $str["q"],
-								"cat" => $pagetype,
-								"sc" => $str["sc"],
+								"sc" => $json["render"]["search_sc"],
 								"t" => "device",
-								"segment" => "startpage.udog",
-								"page" => $str["page"]
+								"cat" => $pagetype,
+								"segment" => "organic",
+								"abd" => "0",
+								"abe" => "0",
+								"query" => rawurldecode($json["render"]["query"]),
+								"page" => $page["number"],
+								"qsr" => "all",
+								"qadf" => "none" // @ todo fix (??)
 							]
 						),
 						$pagetype,

scraper/unsplash.php

@@ -0,0 +1,267 @@
+<?php
+
+class unsplash{
+	
+	public function __construct(){
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+		
+		include "lib/backend.php";
+		$this->backend = new backend("unsplash");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"order_by" => [
+				"display" => "Order by",
+				"option" => [
+					"relevance" => "Relevance",
+					"latest" => "Newest",
+					"editorial" => "Curated"
+				]
+			],
+			"orientation" => [
+				"display" => "Order by",
+				"option" => [
+					"any" => "Any orientation",
+					"landscape" => "Landscape",
+					"portrait" => "Portrait",
+					"squarish" => "Square"
+				]
+			],
+			"license" => [
+				"display" => "License",
+				"option" => [
+					"any" => "Any license",
+					"only" => "Unsplash+",
+					"none" => "Free"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $referer){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: {$referer}",
+			"client-geo-region: global",
+			"x-client-version: 8999df28be3f138bf2c646df5d656e4dc6970ba0",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=0",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filter, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$filter = json_decode($filter, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$filter = [
+				"page" => 1,
+				"per_page" => 20,
+				"query" => $search
+			];
+			
+			// add filters
+			if($get["order_by"] != "relevance"){
+				
+				$filter["order_by"] = $get["order_by"];
+			}
+			
+			if($get["orientation"] != "any"){
+				
+				$filter["orientation"] = $get["orientation"];
+			}
+			
+			if($get["license"] != "any"){
+				
+				$filter["plus"] = $get["license"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+				
+		// https://unsplash.com/s/photos/shibuya-night?order_by=latest&orientation=landscape&license=free
+		// https://unsplash.com/s/photos/%3Ctest-%3F!-haha == <test ?! haha
+		
+		// ?page=1&per_page=20&query=japan
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://unsplash.com/napi/search/photos",
+					$filter,
+					"https://unsplash.com/s/photos/" . rawurlencode(str_replace(" ", "-", $filter["query"])),
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Could not decode the JSON payload");
+		}
+		
+		foreach($json["results"] as $image){
+			
+			$base = explode("?", $image["urls"]["raw"])[0];
+			
+			if(
+				(bool)$image["premium"] ||
+				(bool)$image["plus"]
+			){
+				
+				// when we get "premium" images, give
+				// 1. highest resolution with watermarks
+				// 2. highest resolution without watermarks
+				// (if width of image is above 900px, it has watermarks)
+				// 3. thumbnail without watermark
+				$x900 = $this->imgratio((int)$image["width"], (int)$image["height"], 900);
+				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
+				
+				$source = [
+					[
+						"url" => $base,
+						"width" => (int)$image["width"],
+						"height" => (int)$image["height"]
+					],
+					[
+						"url" => $base . "?w=900",
+						"width" => $x900[0],
+						"height" => $x900[1]
+					],
+					[
+						"url" => $base . "?w=500",
+						"width" => $x500[0],
+						"height" => $x500[1]
+					]
+				];
+			}else{
+				
+				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
+				
+				// image is free as in freedom(tm)
+				$source = [
+					[
+						"url" => $base,
+						"width" => (int)$image["width"],
+						"height" => (int)$image["height"]
+					],
+					[
+						"url" => $base . "?w=500",
+						"width" => $x500[0],
+						"height" => $x500[1]
+					]
+				];
+			}
+			
+			$title = [];
+			
+			$image["description"] = trim($image["description"]);
+			$image["alt_description"] = trim($image["alt_description"]);
+			
+			if(!empty($image["description"])){ $title[] = $image["description"]; }
+			if(!empty($image["alt_description"])){ $title[] = $image["alt_description"]; }
+			
+			$title = implode(": ", $title);
+			
+			$out["image"][] = [
+				"title" => $title,
+				"source" => $source,
+				"url" => "https://unsplash.com/photos/" . $image["slug"]
+			];
+		}
+		
+		// next page stuff
+		if($filter["page"] < (int)$json["total_pages"]){
+			
+			$filter["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filter),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/vsco.php

@@ -30,19 +30,18 @@ class vsco{
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
 				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-Site: none",
 				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
-				"TE: trailers"]
+				"Priority: u=0, i"]
 			);
 		}else{
 			
@@ -73,8 +72,8 @@ class vsco{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// http1 bypass?!?!?!
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		$this->backend->assign_proxy($curlproc, $proxy);
 		

scraper/yandex.php

@@ -868,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
+			$get = json_decode($get, true);
 		}else{
 			
-			$search = $get["s"];
-			if(strlen($search) === 0){
+			if(strlen($get["s"]) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
+		}
 		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
 		// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
+		// ?text=skycamefalling
 		// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
 		
 		$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
+			"text" => $get["s"],
+			"from" => "tabbar",
 			"format" => "json",
-				"request" => json_encode([
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
 				"blocks" => [
 					(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
+						"block" => "video-app",
+						"params" => (object)[]
 					]
 				]
 			]),
-				"text" => $search
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
 		];
 		
-			if($duration != "any"){
+		if(isset($get["p"])){
 			
-				$params["duration"] = $duration;
+			$params["p"] = $get["p"];
 		}
 		
-			if($time != "any"){
+		if($get["duration"] != "any"){
 			
-				$params["within"] = $time;
+			$params["duration"] = $get["duration"];
 		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
 		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -996,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -1011,7 +959,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -1026,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
-		foreach($json["blocks"] as $block){
+		foreach($json["results"]["clips"]["items"] as $k => $data){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
-			}
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
 				}
 				
-		$this->fuckhtml->load($html);
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
 				
-		$div =
-			$this->fuckhtml
-			->getElementsByTagName("div");
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
+			}
 			
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
+		}
 		
-		if(count($npt) !== 0){
+		// get npt
+		if($json["results"]["search"]["hasNextPage"]){
+			
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
-		$mod = $text[1];
+		$unit = strtolower($number[strlen($number) - 1]);
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
 		
-			case "bln.": $num = $num * 1000000000; break;
-			case "mln.": $num = $num * 1000000; break;
-			case "thsd.": $num = $num * 1000; break;
+		if(count($tmp) === 2){
+			
+			$decimal = (int)$tmp[1];
+		}else{
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

scraper/yep.php

@@ -14,234 +14,209 @@ class yep{
 	public function getfilters($page){
 		
 		return [
-			"country" => [
-				"display" => "Country",
+			"lang" => [
+				"display" => "Language",
 				"option" => [
-					"all" => "All regions",
-					"af" => "Afghanistan",
-					"al" => "Albania",
-					"dz" => "Algeria",
-					"as" => "American Samoa",
-					"ad" => "Andorra",
-					"ao" => "Angola",
-					"ai" => "Anguilla",
-					"ag" => "Antigua and Barbuda",
-					"ar" => "Argentina",
-					"am" => "Armenia",
-					"aw" => "Aruba",
-					"au" => "Australia",
-					"at" => "Austria",
-					"az" => "Azerbaijan",
-					"bs" => "Bahamas",
-					"bh" => "Bahrain",
-					"bd" => "Bangladesh",
-					"bb" => "Barbados",
-					"by" => "Belarus",
-					"be" => "Belgium",
-					"bz" => "Belize",
-					"bj" => "Benin",
-					"bt" => "Bhutan",
-					"bo" => "Bolivia",
-					"ba" => "Bosnia and Herzegovina",
-					"bw" => "Botswana",
-					"br" => "Brazil",
-					"bn" => "Brunei Darussalam",
-					"bg" => "Bulgaria",
-					"bf" => "Burkina Faso",
-					"bi" => "Burundi",
-					"cv" => "Cabo Verde",
-					"kh" => "Cambodia",
-					"cm" => "Cameroon",
-					"ca" => "Canada",
-					"ky" => "Cayman Islands",
-					"cf" => "Central African Republic",
-					"td" => "Chad",
-					"cl" => "Chile",
-					"cn" => "China",
-					"co" => "Colombia",
-					"cg" => "Congo",
-					"cd" => "Congo, Democratic Republic",
-					"ck" => "Cook Islands",
-					"cr" => "Costa Rica",
-					"hr" => "Croatia",
-					"cu" => "Cuba",
-					"cy" => "Cyprus",
-					"cz" => "Czechia",
-					"ci" => "Côte d'Ivoire",
-					"dk" => "Denmark",
-					"dj" => "Djibouti",
-					"dm" => "Dominica",
-					"do" => "Dominican Republic",
-					"ec" => "Ecuador",
-					"eg" => "Egypt",
-					"sv" => "El Salvador",
-					"gq" => "Equatorial Guinea",
-					"ee" => "Estonia",
-					"et" => "Ethiopia",
-					"fo" => "Faroe Islands",
-					"fj" => "Fiji",
-					"fi" => "Finland",
-					"fr" => "France",
-					"gf" => "French Guiana",
-					"pf" => "French Polynesia",
-					"ga" => "Gabon",
-					"gm" => "Gambia",
-					"ge" => "Georgia",
-					"de" => "Germany",
-					"gh" => "Ghana",
-					"gi" => "Gibraltar",
-					"gr" => "Greece",
-					"gl" => "Greenland",
-					"gd" => "Grenada",
-					"gp" => "Guadeloupe",
-					"gu" => "Guam",
-					"gt" => "Guatemala",
-					"gg" => "Guernsey",
-					"gn" => "Guinea",
-					"gy" => "Guyana",
-					"ht" => "Haiti",
-					"hn" => "Honduras",
-					"hk" => "Hong Kong",
-					"hu" => "Hungary",
-					"is" => "Iceland",
-					"in" => "India",
-					"id" => "Indonesia",
-					"iq" => "Iraq",
-					"ie" => "Ireland",
-					"im" => "Isle of Man",
-					"il" => "Israel",
-					"it" => "Italy",
-					"jm" => "Jamaica",
-					"jp" => "Japan",
-					"je" => "Jersey",
-					"jo" => "Jordan",
-					"kz" => "Kazakhstan",
-					"ke" => "Kenya",
-					"ki" => "Kiribati",
-					"kw" => "Kuwait",
-					"kg" => "Kyrgyzstan",
-					"la" => "Lao People's Democratic Republic",
-					"lv" => "Latvia",
-					"lb" => "Lebanon",
-					"ls" => "Lesotho",
-					"ly" => "Libya",
-					"li" => "Liechtenstein",
-					"lt" => "Lithuania",
-					"lu" => "Luxembourg",
-					"mk" => "Macedonia",
-					"mg" => "Madagascar",
-					"mw" => "Malawi",
-					"my" => "Malaysia",
-					"mv" => "Maldives",
-					"ml" => "Mali",
-					"mt" => "Malta",
-					"mq" => "Martinique",
-					"mr" => "Mauritania",
-					"mu" => "Mauritius",
-					"yt" => "Mayotte",
-					"mx" => "Mexico",
-					"fm" => "Micronesia, Federated States of",
-					"md" => "Moldova",
-					"mc" => "Monaco",
-					"mn" => "Mongolia",
-					"me" => "Montenegro",
-					"ms" => "Montserrat",
-					"ma" => "Morocco",
-					"mz" => "Mozambique",
-					"mm" => "Myanmar",
-					"na" => "Namibia",
-					"nr" => "Nauru",
-					"np" => "Nepal",
-					"nl" => "Netherlands",
-					"nc" => "New Caledonia",
-					"nz" => "New Zealand",
-					"ni" => "Nicaragua",
-					"ne" => "Niger",
-					"ng" => "Nigeria",
-					"nu" => "Niue",
-					"no" => "Norway",
-					"om" => "Oman",
-					"pk" => "Pakistan",
-					"ps" => "Palestine, State of",
-					"pa" => "Panama",
-					"pg" => "Papua New Guinea",
-					"py" => "Paraguay",
-					"pe" => "Peru",
-					"ph" => "Philippines",
-					"pn" => "Pitcairn",
-					"pl" => "Poland",
-					"pt" => "Portugal",
-					"pr" => "Puerto Rico",
-					"qa" => "Qatar",
-					"ro" => "Romania",
-					"ru" => "Russian Federation",
-					"rw" => "Rwanda",
-					"re" => "Réunion",
-					"sh" => "Saint Helena",
-					"kn" => "Saint Kitts and Nevis",
-					"lc" => "Saint Lucia",
-					"vc" => "Saint Vincent and the Grenadines",
-					"ws" => "Samoa",
-					"sm" => "San Marino",
-					"st" => "Sao Tome and Principe",
-					"sa" => "Saudi Arabia",
-					"sn" => "Senegal",
-					"rs" => "Serbia",
-					"sc" => "Seychelles",
-					"sl" => "Sierra Leone",
-					"sg" => "Singapore",
-					"sk" => "Slovakia",
-					"si" => "Slovenia",
-					"sb" => "Solomon Islands",
-					"so" => "Somalia",
-					"kr" => "Sourth Korea",
-					"za" => "South Africa",
-					"es" => "Spain",
-					"lk" => "Sri Lanka",
-					"sr" => "Suriname",
-					"se" => "Sweden",
-					"ch" => "Switzerland",
-					"tw" => "Taiwan",
-					"tj" => "Tajikistan",
-					"tz" => "Tanzania",
-					"th" => "Thailand",
-					"tl" => "Timor-Leste",
-					"tg" => "Togo",
-					"tk" => "Tokelau",
+					"any" => "Any language",
+					"aa" => "Afar",
+					"ab" => "Abkhazian",
+					"ae" => "Avestan",
+					"af" => "Afrikaans",
+					"ak" => "Akan",
+					"am" => "Amharic",
+					"an" => "Aragonese",
+					"ar" => "Arabic",
+					"as" => "Assamese",
+					"av" => "Avaric",
+					"ay" => "Aymara",
+					"az" => "Azerbaijani",
+					"ba" => "Bashkir",
+					"be" => "Belarusian",
+					"bg" => "Bulgarian",
+					"bh" => "Bihari",
+					"bi" => "Bislama",
+					"bm" => "Bambara",
+					"bn" => "Bengali",
+					"bo" => "Tibetan",
+					"br" => "Breton",
+					"bs" => "Bosnian",
+					"ca" => "Catalan",
+					"ce" => "Chechen",
+					"ch" => "Chamorro",
+					"co" => "Corsican",
+					"cr" => "Cree",
+					"cs" => "Czech",
+					"cu" => "Church Slavic",
+					"cv" => "Chuvash",
+					"cy" => "Welsh",
+					"da" => "Danish",
+					"de" => "German",
+					"dv" => "Divehi",
+					"dz" => "Dzongkha",
+					"ee" => "Ewe",
+					"el" => "Greek",
+					"en" => "English",
+					"eo" => "Esperanto",
+					"es" => "Spanish",
+					"et" => "Estonian",
+					"eu" => "Basque",
+					"fa" => "Persian",
+					"ff" => "Fulah",
+					"fi" => "Finnish",
+					"fj" => "Fijian",
+					"fo" => "Faroese",
+					"fr" => "French",
+					"fy" => "Western Frisian",
+					"ga" => "Irish",
+					"gd" => "Scottish Gaelic",
+					"gl" => "Galician",
+					"gn" => "Guarani",
+					"gu" => "Gujarati",
+					"gv" => "Manx",
+					"ha" => "Hausa",
+					"he" => "Hebrew",
+					"hi" => "Hindi",
+					"ho" => "Hiri Motu",
+					"hr" => "Croatian",
+					"ht" => "Haitian",
+					"hu" => "Hungarian",
+					"hy" => "Armenian",
+					"hz" => "Herero",
+					"ia" => "Interlingua",
+					"id" => "Indonesian",
+					"ie" => "Interlingue",
+					"ig" => "Igbo",
+					"ii" => "Sichuan Yi",
+					"ik" => "Inupiaq",
+					"io" => "Ido",
+					"is" => "Icelandic",
+					"it" => "Italian",
+					"iu" => "Inuktitut",
+					"ja" => "Japanese",
+					"jv" => "Javanese",
+					"ka" => "Georgian",
+					"kg" => "Kongo",
+					"ki" => "Kikuyu",
+					"kj" => "Kuanyama",
+					"kk" => "Kazakh",
+					"kl" => "Kalaallisut",
+					"km" => "Central Khmer",
+					"kn" => "Kannada",
+					"ko" => "Korean",
+					"kr" => "Kanuri",
+					"ks" => "Kashmiri",
+					"ku" => "Kurdish",
+					"kv" => "Komi",
+					"kw" => "Cornish",
+					"ky" => "Kyrgyz",
+					"la" => "Latin",
+					"lb" => "Luxembourgish",
+					"lg" => "Ganda",
+					"li" => "Limburgish",
+					"ln" => "Lingala",
+					"lo" => "Lao",
+					"lt" => "Lithuanian",
+					"lu" => "Luba-Katanga",
+					"lv" => "Latvian",
+					"mg" => "Malagasy",
+					"mh" => "Marshallese",
+					"mi" => "Maori",
+					"mk" => "Macedonian",
+					"ml" => "Malayalam",
+					"mn" => "Mongolian",
+					"mr" => "Marathi",
+					"ms" => "Malay",
+					"mt" => "Maltese",
+					"my" => "Burmese",
+					"na" => "Nauru",
+					"nb" => "Norwegian Bokmål",
+					"nd" => "North Ndebele",
+					"ne" => "Nepali",
+					"ng" => "Ndonga",
+					"nl" => "Dutch",
+					"nn" => "Norwegian Nynorsk",
+					"no" => "Norwegian",
+					"nr" => "South Ndebele",
+					"nv" => "Navajo",
+					"ny" => "Chichewa",
+					"oc" => "Occitan",
+					"oj" => "Ojibwa",
+					"om" => "Oromo",
+					"or" => "Oriya",
+					"os" => "Ossetian",
+					"pa" => "Punjabi",
+					"pi" => "Pali",
+					"pl" => "Polish",
+					"ps" => "Pashto",
+					"pt" => "Portuguese",
+					"qu" => "Quechua",
+					"rm" => "Romansh",
+					"rn" => "Rundi",
+					"ro" => "Romanian",
+					"ru" => "Russian",
+					"rw" => "Kinyarwanda",
+					"sa" => "Sanskrit",
+					"sc" => "Sardinian",
+					"sd" => "Sindhi",
+					"se" => "Northern Sami",
+					"sg" => "Sango",
+					"si" => "Sinhala",
+					"sk" => "Slovak",
+					"sl" => "Slovenian",
+					"sm" => "Samoan",
+					"sn" => "Shona",
+					"so" => "Somali",
+					"sq" => "Albanian",
+					"sr" => "Serbian",
+					"ss" => "Swati",
+					"st" => "Southern Sotho",
+					"su" => "Sundanese",
+					"sv" => "Swedish",
+					"sw" => "Swahili",
+					"ta" => "Tamil",
+					"te" => "Telugu",
+					"tg" => "Tajik",
+					"th" => "Thai",
+					"ti" => "Tigrinya",
+					"tk" => "Turkmen",
+					"tl" => "Tagalog",
+					"tn" => "Tswana",
 					"to" => "Tonga",
-					"tt" => "Trinidad and Tobago",
-					"tn" => "Tunisia",
-					"tr" => "Turkey",
-					"tm" => "Turkmenistan",
-					"ug" => "Uganda",
-					"ua" => "Ukraine",
-					"ae" => "United Arab Emirates",
-					"gb" => "United Kingdom",
-					"us" => "United States",
-					"uy" => "Uruguay",
-					"uz" => "Uzbekistan",
-					"vu" => "Vanuatu",
-					"ve" => "Venezuela",
-					"vn" => "Vietnam",
-					"vg" => "Virgin Islands, British",
-					"vi" => "Virgin Islands, U.S.",
-					"ye" => "Yemen",
-					"zm" => "Zambia",
-					"zw" => "Zimbabwe"
+					"tr" => "Turkish",
+					"ts" => "Tsonga",
+					"tt" => "Tatar",
+					"tw" => "Twi",
+					"ty" => "Tahitian",
+					"ug" => "Uyghur",
+					"uk" => "Ukrainian",
+					"ur" => "Urdu",
+					"uz" => "Uzbek",
+					"ve" => "Venda",
+					"vi" => "Vietnamese",
+					"vo" => "Volapük",
+					"wa" => "Walloon",
+					"wo" => "Wolof",
+					"xh" => "Xhosa",
+					"yi" => "Yiddish",
+					"yo" => "Yoruba",
+					"za" => "Zhuang",
+					"zh" => "Chinese",
+					"zh-cn" => "Chinese (Simplified)",
+					"zh-tw" => "Chinese (Traditional)",
+					"zu" => "Zulu"
 				]
 			],
 			"nsfw" => [
 				"display" => "NSFW",
 				"option" => [
 					"yes" => "Yes",
-					"maybe" => "Maybe",
 					"no" => "No"
 				]
 			]
 		];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
 		
 		$curlproc = curl_init();
 		
@@ -252,14 +227,29 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		// use http1 lol
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($use_api){
+			
+			$post_data = json_encode($post_data);
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["Content-Type: application/json",
+				"Authorization: Bearer $bearer",
+				"Content-Length: " . strlen($post_data)]
+			);
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
+		}else{
+			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
 				"Accept: */*",
-			"Accept-Language: en-US,en;q=0.5",
+				"Accept-Language: en-US,en;q=0.9",
 				"Accept-Encoding: gzip, deflate, br, zstd",
 				"Referer: https://yep.com/",
 				"Origin: https://yep.com",
@@ -269,8 +259,11 @@ class yep{
 				"Sec-Fetch-Mode: cors",
 				"Sec-Fetch-Site: same-site",
 				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
 				"TE: trailers"]
 			);
+		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -295,22 +288,17 @@ class yep{
 	
 	public function web($get){
 		
+		if(config::YEP_USE_API){
+			
+			return $this->web_api($get);
+		}
+		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -327,22 +315,23 @@ class yep{
 			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"limit" => 100, // wwwwwwwwwwwwwww
+			"query" => $search,
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
+		if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
+		
 		try{
 			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
+			// https://api.yep.com/search?limit=20&query=asmr
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "web"
-					]
+					"https://api.yep.com/search",
+					$filters
 				);
 			
 		}catch(Exception $error){
@@ -408,7 +397,7 @@ class yep{
 									)
 								),
 							"url" => $item["url"],
-							"date" => strtotime($item["first_seen"]),
+							"date" => null,
 							"type" => "web",
 							"thumb" => [
 								"url" => null,
@@ -422,83 +411,11 @@ class yep{
 			}
 		}
 		
-		if(isset($json[1]["featured_news"])){
-			
-			foreach($json[1]["featured_news"] as $news){
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$news["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($news["first_seen"]),
-					"thumb" =>
-						isset($news["img"]) ?
-						[
-							"url" => $this->unshiturl($news["img"]),
-							"ratio" => "16:9"
-						] :
-						[
-							"url" => null,
-							"ratio" => null
-						],
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		if(isset($json[1]["featured_images"])){
-			
-			foreach($json[1]["featured_images"] as $image){
-				
-				if(
-					$image["width"] !== 0 &&
-					$image["height"] !== 0
-				){
-					
-					$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
-					$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
-					
-					$width = $image["width"];
-					$height = $image["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $image["title"],
-					"source" => [
-						[
-							"url" => $image["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $image["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $image["host_page"]
-				];
-			}
-		}
-		
 		return $out;
 	}
 	
 	
-	
-	public function image($get){
+	private function web_api($get){
 		
 		$search = $get["s"];
 		if(strlen($search) === 0){
@@ -506,142 +423,53 @@ class yep{
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
 			"npt" => null,
-			"image" => []
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"query" => $search,
+			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
+		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
+		
+		// add api key
+		$key_data = $this->backend->get_key();
+		
 		try{
 			
 			$json =
 				$this->get(
-					$this->backend->get_ip(), // no nextpage!
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "images"
-					]
+					$this->backend->get_ip($key_data["increment"]),
+					"https://platform.yep.com/api/search",
+					[],
+					true,
+					$filters,
+					$key_data["key"]
 				);
+			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
 		}
 		
-		$this->detect_cf($json);
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
-				
-				if(
-					$item["width"] !== 0 &&
-					$item["height"] !== 0
-				){
-					
-					$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
-					$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
-					
-					$width = $item["width"];
-					$height = $item["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $item["title"],
-					"source" => [
-						[
-							"url" => $item["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $item["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $item["host_page"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
-	
-	public function news($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		try{
-			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
-			$json =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "news"
-					]
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$this->detect_cf($json);
+		// should never happen
+		//$this->detect_cf($json);
 		
 		$json = json_decode($json, true);
 		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
@@ -651,35 +479,54 @@ class yep{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
+		if(isset($json["error"])){
 			
-				$out["news"][] = [
+			throw new Exception("Yep API returned an error: " . $json["error"]);
+		}
+		
+		if(isset($json["errors"])){
+			
+			throw new Exception("Yep API returned the following errors: {$json["message"]}");
+		}
+		
+		if(
+			isset($json["success"]) &&
+			$json["success"] !== true
+		){
+			
+			throw new Exception("Yep API returned a false-y success value");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Yep API did not return a results object");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			if(
+				$item["url"] === null ||
+				$item["url"] == ""
+			){
+				
+				// sometimes API fucks up
+				continue;
+			}
+			
+			$out["web"][] = [
 				"title" => $item["title"],
-					"author" => null,
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$item["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($item["first_seen"]),
-					"thumb" =>
-						isset($item["img"]) ?
-						[
-							"url" => $this->unshiturl($item["img"]),
-							"ratio" => "16:9"
-						] :
-						[
+				"description" => $item["description"],
+				"url" => $item["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
 					"url" => null,
 					"ratio" => null
 				],
-					"url" => $item["url"]
+				"sublink" => [],
+				"table" => []
 			];
 		}
-		}
 		
 		return $out;
 	}

settings.php

@@ -91,10 +91,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -161,14 +157,6 @@ $settings = [
 						"value" => "yep",
 						"text" => "Yep"
 					],
-					[
-						"value" => "greppr",
-						"text" => "Greppr"
-					],
-					[
-						"value" => "crowdview",
-						"text" => "Crowdview"
-					],
 					[
 						"value" => "mwmbl",
 						"text" => "Mwmbl"
@@ -177,6 +165,10 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -196,10 +188,6 @@ $settings = [
 					[
 						"value" => "wiby",
 						"text" => "wiby"
-					],
-					[
-						"value" => "curlie",
-						"text" => "Curlie"
 					]
 				]
 			],
@@ -223,6 +211,10 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "google_api",
+						"text" => "Google API"
+					],
 					[
 						"value" => "google_cse",
 						"text" => "Google CSE"
@@ -240,8 +232,8 @@ $settings = [
 						"text" => "Qwant"
 					],
 					[
-						"value" => "yep",
-						"text" => "Yep"
+						"value" => "naver",
+						"text" => "Naver"
 					],
 					[
 						"value" => "baidu",
@@ -263,6 +255,18 @@ $settings = [
 						"value" => "flickr",
 						"text" => "Flickr"
 					],
+					[
+						"value" => "pexels",
+						"text" => "Pexels"
+					],
+					[
+						"value" => "pixabay",
+						"text" => "Pixabay"
+					],
+					[
+						"value" => "unsplash",
+						"text" => "Unsplash"
+					],
 					[
 						"value" => "fivehpx",
 						"text" => "500px"
@@ -325,6 +329,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
 					[
 						"value" => "baidu",
 						"text" => "Baidu"
@@ -367,10 +375,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "yep",
-						"text" => "Yep"
-					],
 					[
 						"value" => "mojeek",
 						"text" => "Mojeek"

static/serverping.js

@@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
+
+// server popup without inline javascript
+tbody.addEventListener('click', function(event){
+	var row = event.target.closest('.show_server');
+	if(!row) return;
+
+	show_server(row.id);
+})
+
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -343,7 +352,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
+		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -406,6 +415,7 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
+
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/themes/Rosé Pine.css

@@ -0,0 +1,20 @@
+:root{
+	/* background */
+	--1d2021: #191724;
+	--282828: #1f1d2e;
+	--3c3836: #26233a;
+	--504945: #524f67;
+
+	/* font */
+	--928374: #e0def4;
+	--a89984: #908caa;
+	--bdae93: #6e6a86;
+	--8ec07c: #9ccfd8;
+	--ebdbb2: #f6c177;
+
+	/* code highlighter */
+	--comment: #6e6a86;
+	--default: #c4a7e7;
+	--keyword: #31748f;
+	--string: #f6c177;
+}

static/themes/evergarden.css

@@ -0,0 +1,25 @@
+:root {
+	/* background */
+	--1d2021: #232a2e;
+	--282828: #2b3337;
+	--3c3836: #374145;
+	--504945: #4a585c;
+
+	/* font */
+	--928374: #839e9a;
+	--a89984: #adc9bc;
+	--bdae93: #f8f9e8;
+	--8ec07c: #cbe3b3;
+	--ebdbb2: #adc9bc;
+
+	/* code highlighter */
+	--comment: #839e9a;
+	--default: #f8f9e8;
+	--keyword: #6f8788;
+	--string: #cbe3b3;
+
+	/* color codes for instance list */
+	--green: #cbe3b3;
+	--yellow: #f5d098;
+	--red: #f57f82;
+}