change variable

Reviewed-on: #98

.dockerignore

@@ -0,0 +1 @@
+.git

Dockerfile

@@ -1,10 +1,18 @@
-FROM alpine:latest
+FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate
+
+FROM alpine:3.24
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php83-fileinfo php83-openssl php83-iconv php83-common php83-dom php83-sodium php83-curl curl php83-pecl-apcu php83-apache2 imagemagick php83-pecl-imagick php-mbstring imagemagick-webp imagemagick-jpeg
+RUN apk add php apache2-ssl php85-fileinfo php85-openssl php85-iconv php85-common php85-dom php85-sodium php85-curl curl php85-pecl-apcu php85-apache2 imagemagick php85-pecl-imagick php85-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates ghostscript-fonts
+
+COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
+COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
+
+ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so
+ENV CURL_IMPERSONATE=ff117
+ENV CURL_IMPERSONATE_HEADERS=no
 
-COPY ./docker/apache/ /etc/apache2/
 COPY . .
 
 RUN chmod 777 /var/www/html/4get/icons
@@ -14,4 +22,5 @@ EXPOSE 443
 
 ENV FOURGET_PROTO=http
 
-CMD  ["./docker/docker-entrypoint.sh"]
+ENTRYPOINT  ["./docker/docker-entrypoint.sh"]
+CMD ["start"]

README.md

@@ -9,13 +9,15 @@ https://4get.ca/about
 ## Official instance
 https://4get.ca , or visit the official instance list: https://4get.ca/instances
 
+_NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
+
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | libreY      | araa      | hearch            |
-|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
+|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
+|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
+| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
+| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
+| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -23,36 +25,40 @@ https://4get.ca , or visit the official instance list: https://4get.ca/instances
 3. Bot protection that *actually* filters out the bots (when configured)
 4. Interface doesn't require javascript
 5. Favicon fetcher with caching support & image proxy
-6. Bunch of other shit
+6. Bunch of other shits
 
-tl;dr the best way to actually browse for shit.
+tl;dr 4get is the best way to browse for shit.
 
 # Supported websites
 
-| Web        | Images       | Videos     | News       | Music      | Autocompleter |
-|------------|--------------|------------|------------|------------|---------------|
-| DuckDuckGo | DuckDuckGo   | YouTube    | DuckDuckGo | Soundcloud | Brave         |
-| Brave      | Brave        | DuckDuckGo | Brave      |            | DuckDuckGo    |
-| Yandex     | Yandex       | Brave      | Google     |            | Yandex        |
-| Google     | Google       | Yandex     | Startpage  |            | Google        |
-| Startpage  | Startpage    | Google     | Qwant      |            | Startpage     |
-| Qwant      | Qwant        | Startpage  | Mojeek     |            | Kagi          |
-| Ghostery   | Yep          | Qwant      |            |            | Qwant         |
-| Yep        | Solofield    | Solofield  |            |            | Ghostery      |
-| Greppr     | Imgur        |            |            |            | Yep           |
-| Crowdview  | FindThatMeme |            |            |            | Marginalia    |
-| Mwmbl      |              |            |            |            | YouTube       |
-| Mojeek     |              |            |            |            | Soundcloud    |
-| Solofield  |              |            |            |            |               |
-| Marginalia |              |            |            |            |               |
-| wiby       |              |            |            |            |               |
-| Curlie     |              |            |            |            |               |
+| web          | images       | videos       | news         | music      | autocomplete |
+|--------------|--------------|--------------|--------------|------------|--------------|
+| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google API   | Google API   | Brave        | Startpage    |            | Qwant        |
+| Google CSE   | Google CSE   | Yandex       | Baidu        |            | Marginalia   |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       |              |            | YouTube      |
+| Startpage    | Startpage    | Yahoo! JAPAN |              |            | SoundCloud   |
+| Yep          | Naver        | Startpage    |              |            | Startpage    |
+| Mwmbl        | Baidu        | Naver        |              |            | Kagi         |
+| Mojeek       | Solofield    | Baidu        |              |            |              |
+| Naver        | Pinterest    | Cốc Cốc      |              |            |              |
+| Baidu        | Flickr       | Purili       |              |            |              |
+| Cốc Cốc      | Pexels       | Solofield    |              |            |              |
+| Solofield    | Pixabay      |              |              |            |              |
+| Marginalia   | Unsplash     |              |              |            |              |
+| Purili       | 500px        |              |              |            |              |
+| wiby         | VSCO         |              |              |            |              |
+|              | Imgur        |              |              |            |              |
+|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.
 
 ## Contact
-Shit breaks all the time but I repair it all the time too... Email me here: <b>will (at) lolcat.ca</b> or create an issue.
+Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPL
+AGPLv3-only

ami4get.php

@@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *");
 
 include "data/config.php";
 
-$real_requests = apcu_fetch("real_requests");
-$bot_requests = apcu_fetch("captcha_gen");
+// get request count in last 24 hours
+$bot_requests = 0;
+$real_requests = 0;
+$time = time();
+
+for($i=0; $i<24; $i++){
+	
+	$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
+}
+
+for($i=0; $i<24; $i++){
+	
+	$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
+}
 
 echo json_encode(
 	[
@@ -16,10 +28,10 @@ echo json_encode(
 			"name" => config::SERVER_NAME,
 			"description" => config::SERVER_LONG_DESCRIPTION,
 			"bot_protection" => config::BOT_PROTECTION,
-			"real_requests" => $real_requests === false ? 0 : $real_requests,
-			"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
+			"real_requests" => $real_requests,
+			"bot_requests" => $bot_requests,
 			"api_enabled" => config::API_ENABLED,
-			"alt_addresses" => config::ALT_ADDRESSES,
+			"alt_addresses" => config::ADDRESSES,
 			"version" => config::VERSION
 		],
 		"instances" => config::INSTANCES

api.txt

@@ -1,9 +1,16 @@
-                        __ __             __
-                       / // / ____ ____  / /_
-                      / // /_/ __ `/ _ \/ __/
-                     /__  __/ /_/ /  __/ /_ 
-                       /_/  \__, /\___/\__/
-                           /____/         
+                   44
+                 4444444      44   
+                 44444444   44444       444
+                 44444444  444444    444444444
+                  44444   44444444  444444444
+                         444444444   4444444
+                        4444444444    444444
+                      4444444444444
+                    444444444444444444
+                       444444444444444
+                          44444444
+                          4444
+                           44
                  
            + Welcome to the 4get API documentation +
 

api/v1/ac.php

@@ -15,13 +15,11 @@ class autocomplete{
 			"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
 			"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
 			"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
-			"yep" => "https://api.yep.com/ac/?query={searchTerms}",
 			"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
 			"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
 			"sc" => "",
 			"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
-			"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
-			"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
+			"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
 		];
 		
 		/*
@@ -165,6 +163,8 @@ class autocomplete{
 				// if it respects the openSearch protocol
 				$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
 				
+				if(!isset($json[1])){ $json[1] = []; }
+				
 				echo json_encode(
 					[
 						$_GET["s"],

api/v1/images.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->image($get),

api/v1/music.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->music($get),

api/v1/news.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->news($get),

api/v1/videos.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->video($get),

api/v1/web.php

@@ -38,6 +38,9 @@ if(
 	$get["extendedsearch"] = "no";
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	
 	echo

api/v2/graph.php

@@ -0,0 +1,100 @@
+<?php
+
+header("Content-Type: image/png");
+date_default_timezone_set('America/Toronto');
+include "../../data/config.php";
+
+function get_requests($slug){
+	
+	$time = time();
+	$requests = [];
+	
+	for($i=0; $i<73; $i++){
+		
+		$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
+		
+		$requests[] = [$i, ($data === false ? 0 : $data)];
+	}
+	
+	// rtrim() out all entries that are 0
+	$out = [];
+	$found = false;
+	for($i=count($requests) - 1; $i>=0; $i--){
+		
+		if($requests[$i][1] !== 0){
+			
+			$found = $i;
+			break;
+		}
+	}
+	
+	if($found === false){
+		
+		return [];
+	}
+	
+	$requests = array_slice($requests, 0, $found + 1);
+	
+	return $requests;
+}
+
+$bot_requests = get_requests("bot_requests");
+$real_requests = get_requests("real_requests");
+
+$real_reqs_8h =
+	array_sum(
+		array_column(
+			array_slice(
+				$real_requests,
+				0,
+				8
+			),
+			1
+		)
+	);
+
+$real_reqs_24h =
+	array_sum(
+		array_column(
+			array_slice(
+				$real_requests,
+				0,
+				24
+			),
+			1
+		)
+	);
+
+$real_reqs_72h =
+	array_sum(
+		array_column(
+			$real_requests,
+			1
+		)
+	);
+
+include "../../lib/graph_gen.php";
+$graph = new graph();
+
+$c = count($real_requests);
+
+if($c > 25){
+	
+	$c = 25;
+}elseif($c <= 2){
+	
+	$c = 2;
+}
+
+echo $graph->render([
+	"title" => "Number of requests within the last 72 hours (non-cummulative)",
+	"subtitle" => date("jS M y @ g:ia", time()) . " || 8h=" . number_format($real_reqs_8h) . " - 24h=" . number_format($real_reqs_24h) . " - 72h=" . number_format($real_reqs_72h) . " || " . config::SERVER_NAME,
+	"x_label" => "<= Hours passed =>",
+	"y_label" => "<= Request count/hour =>",
+	"grad_x" => $c,
+	"grad_y" => 20,
+	"data" => [
+		"Bot requests" => $bot_requests,
+		"Real requests" => $real_requests,
+	]
+]);

audio/linear.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/sc.php

@@ -1,224 +0,0 @@
-<?php
-
-new sc_audio();
-
-class sc_audio{
-	
-	public function __construct(){
-		
-		include "../data/config.php";
-		include "../lib/curlproxy.php";
-		$this->proxy = new proxy();
-		
-		if(isset($_GET["u"])){
-			
-			/*
-				we're now proxying audio
-			*/
-			$viewkey = $_GET["u"];
-			
-			if(!isset($_GET["r"])){
-				
-				$this->do404("Ranges(r) are missing");
-			}
-			
-			$ranges = explode(",", $_GET["r"]);
-			
-			// sanitize ranges
-			foreach($ranges as &$range){
-				
-				if(!is_numeric($range)){
-					
-					$this->do404("Invalid range specified");
-				}
-				
-				$range = (int)$range;
-			}
-			
-			// sort ranges (just to make sure)
-			sort($ranges);
-			
-			// convert ranges to pairs
-			$last = -1;
-			foreach($ranges as &$r){
-				
-				$tmp = $r;
-				$r = [$last + 1, $r];
-				
-				$last = $tmp;
-			}
-			
-			$browser_headers = getallheaders();
-			
-			// get the requested range from client
-			$client_range = 0;
-			foreach($browser_headers as $key => $value){
-				
-				if(strtolower($key) == "range"){
-					
-					preg_match(
-						'/bytes=([0-9]+)/',
-						$value,
-						$client_regex
-					);
-					
-					if(isset($client_regex[1])){
-						
-						$client_range = (int)$client_regex[1];
-					}else{
-						
-						$client_range = 0;
-					}
-					break;
-				}
-			}
-			
-			if(
-				$client_range < 0 ||
-				$client_range > $ranges[count($ranges) - 1][1]
-			){
-				
-				// range is not satisfiable
-				http_response_code(416);
-				header("Content-Type: text/plain");
-				die();
-			}
-			
-			$rng = null;
-			for($i=0; $i<count($ranges); $i++){
-				
-				if($ranges[$i][0] <= $client_range){
-					
-					$rng = $ranges[$i];
-				}
-			}
-			
-			// proxy data!
-			http_response_code(206); // partial content
-			header("Accept-Ranges: bytes");
-			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
-			
-			$viewkey =
-				preg_replace(
-					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-					'/media/$1/' . $rng[0] . '/' . $rng[1],
-					$viewkey
-				);
-			
-			try{
-				
-				$this->proxy->stream_linear_audio(
-					$viewkey
-				);
-			}catch(Exception $error){
-				
-				$this->do404("Could not read stream");
-			}
-			
-			die();
-		}
-		
-		/*
-			redirect user to correct resource
-			we need to scrape and store the byte positions in the result URL
-		*/
-		if(!isset($_GET["s"])){
-			
-			$this->do404("The URL(s) parameter is missing");
-		}
-		
-		$viewkey = $_GET["s"];
-		
-		if(
-			preg_match(
-				'/soundcloud\.com$/',
-				parse_url($viewkey, PHP_URL_HOST)
-			) === false
-		){
-			
-			$this->do404("This endpoint can only be used for soundcloud streams");
-		}
-		
-		try{
-			
-			$json = $this->proxy->get($viewkey)["body"];
-		}catch(Exception $error){
-			
-			$this->do404("Curl error: " . $error->getMessage());
-		}
-		
-		$json = json_decode($json, true);
-		
-		if(!isset($json["url"])){
-			
-			$this->do404("Could not get URL from JSON");
-		}
-		
-		$viewkey = $json["url"];
-		
-		$m3u8 = $this->proxy->get($viewkey)["body"];
-		
-		$m3u8 = explode("\n", $m3u8);
-		
-		$lineout = null;
-		$streampos_arr = [];
-		foreach($m3u8 as $line){
-			
-			$line = trim($line);
-			if($line[0] == "#"){
-				
-				continue;
-			}
-			
-			if($lineout === null){
-				$lineout = $line;
-			}
-			
-			preg_match(
-				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
-				$line,
-				$matches
-			);
-			
-			if(isset($matches[0])){
-				
-				$streampos_arr[] = [
-					(int)$matches[1],
-					(int)$matches[2]
-				];
-			}
-		}
-		
-		if($lineout === null){
-			
-			$this->do404("Could not get stream URL");
-		}
-		
-		$lineout =
-			preg_replace(
-				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
-				'/media/$1/0/0',
-				$lineout
-			);
-		
-		$streampos = [];
-		
-		foreach($streampos_arr as $pos){
-			
-			$streampos[] = $pos[1];
-		}
-		
-		$streampos = implode(",", $streampos);
-		
-		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
-		header("Accept-Ranges: bytes");
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

audio/seekable.php

@@ -1,20 +0,0 @@
-<?php
-
-if(!isset($_GET["s"])){
-	
-	http_response_code(404);
-	header("X-Error: No SOUND(s) provided!");
-	die();
-}
-
-include "../data/config.php";
-include "../lib/curlproxy.php";
-$proxy = new proxy();
-
-try{
-	
-	$proxy->stream_linear_audio($_GET["s"]);
-}catch(Exception $error){
-	
-	header("X-Error: " . $error->getMessage());
-}

audio/spotify.php

@@ -1,214 +0,0 @@
-<?php
-
-include "../data/config.php";
-new spotify();
-
-class spotify{
-	
-	public function __construct(){
-		
-		include "../lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		if(
-			!isset($_GET["s"]) ||
-			!preg_match(
-				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
-				$_GET["s"],
-				$matches
-			)
-		){
-			
-			$this->do404("The track ID(s) parameter is missing or invalid");
-		}
-		
-		try{
-			
-			if($matches[1] == "episode"){
-				
-				$uri = "show";
-			}else{
-				
-				$uri = $matches[1];
-			}
-			
-			$embed =
-				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
-		}catch(Exception $error){
-			
-			$this->do404("Failed to fetch embed data");
-		}
-		
-		$this->fuckhtml->load($embed);
-		
-		$json =
-			$this->fuckhtml
-			->getElementById(
-				"__NEXT_DATA__",
-				"script"
-			);
-		
-		if($json === null){
-			
-			$this->do404("Failed to extract JSON");
-		}
-		
-		$json =
-			json_decode($json["innerHTML"], true);
-		
-		if($json === null){
-			
-			$this->do404("Failed to decode JSON");
-		}
-		
-		switch($matches[1]){
-			
-			case "track":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["audioPreview"]
-						["url"]
-					)
-				){
-					
-					header("Content-type: audio/mpeg");
-					header(
-						"Location: /audio/linear?s=" .
-						urlencode(
-							$json
-							["props"]
-							["pageProps"]
-							["state"]
-							["data"]
-							["entity"]
-							["audioPreview"]
-							["url"]
-						)
-					);
-				}else{
-					
-					$this->do404("Could not extract playback URL");
-				}
-				break;
-			
-			case "episode":
-				if(
-					isset(
-						$json
-						["props"]
-						["pageProps"]
-						["state"]
-						["data"]
-						["entity"]
-						["id"]
-					)
-				){
-					
-					try{
-						$json =
-							$this->get(
-								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
-								$json
-								["props"]
-								["pageProps"]
-								["state"]
-								["data"]
-								["entity"]
-								["id"] .
-								"/com.widevine.alpha"
-							);
-					}catch(Exception $error){
-						
-						$this->do404("Failed to fetch audio resource");
-					}
-					
-					$json = json_decode($json, true);
-					
-					if($json === null){
-						
-						$this->do404("Failed to decode audio resource JSON");
-					}
-					
-					if(
-						isset($json["passthrough"]) &&
-						$json["passthrough"] == "ALLOWED" &&
-						isset($json["passthroughUrl"])
-					){
-						
-						header(
-							"Location:" .
-							"/audio/linear.php?s=" .
-							urlencode(
-								str_replace(
-									"http://",
-									"https://",
-									$json["passthroughUrl"]
-								)
-							)
-						);
-					}else{
-						
-						$this->do404("Failed to find passthroughUrl");
-					}
-					
-				}else{
-					
-					$this->do404("Failed to find episode ID");
-				}
-				break;
-			}
-	}
-	
-	private function get($url){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	private function do404($error){
-		
-		http_response_code(404);
-		header("Content-Type: text/plain");
-		header("X-Error: $error");
-		die();
-	}
-}

data/api_keys/google_api.txt

@@ -0,0 +1,8 @@
+# Specify API keys for the Google API in the following format:
+# <key>
+#
+# Generate keys here:
+# https://developers.google.com/custom-search/v1/overview
+# Make sure to use a different Google account for each key, cause I'm
+# pretty sure the ratelimit is on a per-account basis :P
+#

data/api_keys/yep.txt

@@ -0,0 +1 @@
+# Paste Yep API keys here

data/config.php

@@ -5,7 +5,7 @@ class config{
 	// any parameters.
 	
 	// 4get version. Please keep this updated
-	const VERSION = 8;
+	const VERSION = 9;
 	
 	// Will be shown pretty much everywhere.
 	const SERVER_NAME = "4get";
@@ -23,6 +23,48 @@ class config{
 	// Enable the API?
 	const API_ENABLED = true;
 	
+	// Display errors. This should ONLY be enabled for debugging and constitutes a security risk.
+	const DISPLAY_ERRORS = false;
+	
+	//
+	// 4play settings
+	//
+	// Enable 4get's /api/v2/provide_sesh endpoint
+	const FPLAY_ENABLE_API = false;
+	
+	// 4play password. Please set this to something secure if you use this extension.
+	// This password is used to POST sessions to /api/v2/provide_sesh
+	// It's also used as the external endpoint password.
+	const FPLAY_PASSWORD = "cnc";
+	
+	// Endpoint 4get hits to render webpages
+	// Eg: null=no endpoint provided
+	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
+	const FPLAY_EXTERNAL_ENDPOINT = null;
+	
+	//
+	// RATELIMITING
+	// Limits also applies to the captcha page
+	//
+	
+	// Amount of requests allowed before we block
+	// Set to 0 to disable ratelimiting
+	const RATELIMIT_BURST = 0;
+	
+	// IP source for ratelimiting. Choices: false for getting the IP directly,
+	// or "X-Header-Name" to get the IP from a forwarded header
+	const RATELIMIT_IP_SOURCE = false;
+	
+	// How long till the counter resets? Note: counter resets everytime
+	// a new request occurs while being blocked.
+	const RATELIMIT_TIME = 2;
+	
+	// How many seconds to add every time a new request occurs while being blocked?
+	const RATELIMIT_PUNISH = 1;
+	
+	// Add seconds of punishment time to users of the API?
+	const RATELIMIT_PUNISH_API = false;
+	
 	//
 	// BOT PROTECTION
 	//
@@ -43,7 +85,7 @@ class config{
 	
 	// If this regex expression matches on the user agent, it blocks the request
 	// Not useful at all against a targetted attack
-	const HEADER_REGEX = '/bot|wget|curl|python-requests|scrapy|go-http-client|ruby|yahoo|spider|qwant/i';
+	const HEADER_REGEX = '/bot|wget|curl|python-requests|scrapy|go-http-client|ruby|yahoo|spider|qwant|meta/i';
 	
 	// Block clients who present any of the following headers in their request (SPECIFY IN !!lowercase!!)
 	// Eg: ["x-forwarded-for", "x-via", "forwarded-for", "via"];
@@ -77,11 +119,9 @@ class config{
 	const MAX_SEARCHES = 100;
 	
 	// List of domains that point to your servers. Include your tor/i2p
-	// addresses here! Must be a valid URL. Won't affect links placed on
-	// the homepage.
-	const ALT_ADDRESSES = [
-		//"https://4get.alt-tld",
-		//"http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion"
+	// addresses here! Must be a valid URL.
+	const ADDRESSES = [
+		//"https://4get.ca",
 	];
 	
 	// Known 4get instances. MUST use the https protocol if your instance uses
@@ -90,75 +130,102 @@ class config{
 	// eachother your serber should appear everywhere.
 	const INSTANCES = [
 		"https://4get.ca",
-		"https://4get.zzls.xyz",
-		"https://4getus.zzls.xyz",
+		"https://search.fischbytes.de",
+		"https://4get.nadeko.net",
 		"https://4get.silly.computer",
-		"https://4get.konakona.moe",
-		"https://4get.lvkaszus.pl",
-		"https://4g.ggtyler.dev",
-		"https://4get.perennialte.ch",
 		"https://4get.sijh.net",
 		"https://4get.hbubli.cc",
-		"https://4get.plunked.party",
-		"https://4get.seitan-ayoub.lol",
-		"https://4get.etenie.pl",
 		"https://4get.lunar.icu",
 		"https://4get.dcs0.hu",
-		"https://4get.kizuki.lol",
-		"https://4get.psily.garden",
-		"https://search.milivojevic.in.rs",
-		"https://4get.snine.nl",
-		"https://4get.datura.network",
 		"https://4get.neco.lol",
-		"https://4get.lol",
-		"https://4get.ch",
 		"https://4get.edmateo.site",
 		"https://4get.sudovanilla.org",
-		"https://search.mint.lgbt"
+		"https://search.mint.lgbt",
+		"https://4get.aishiteiru.moe",
+		"https://4.nboeck.de",
+		"https://search.yonderly.org",
+		"https://4get.lurx.net",
+		"https://4get.canine.tools",
+		"https://4get.sny.sh",
+		"https://4g.opnxng.com",
+		"https://s.307200.xyz",
+		"https://4get.privadency.com/"
 	];
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0";
+	
+	// User agent to use with 4get-friendly APIs
+	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
 	// string = will load a proxy list from data/proxies
 	// Eg. "onion" will load data/proxies/onion.txt
 	const PROXY_DDG = false; // duckduckgo
+	const PROXY_YAHOO = false;
+	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
-	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
+	const PROXY_GOOGLE_ALT = false; // alt proxies for url resolution
+	const PROXY_GOOGLE_API = false;
+	const PROXY_GOOGLE_CSE = false;
 	const PROXY_STARTPAGE = false;
 	const PROXY_QWANT = false;
+	const PROXY_BAIDU = false;
+	const PROXY_COCCOC = false;
 	const PROXY_GHOSTERY = false;
 	const PROXY_MARGINALIA = false;
 	const PROXY_MOJEEK = false;
 	const PROXY_SC = false; // soundcloud
+	const PROXY_SWISSCOWS = false;
 	const PROXY_SPOTIFY = false;
 	const PROXY_SOLOFIELD = false;
+	const PROXY_PURILI = false;
 	const PROXY_WIBY = false;
 	const PROXY_CURLIE = false;
 	const PROXY_YT = false; // youtube
+	const PROXY_ARCHIVEORG = false;
+	const PROXY_SEPIASEARCH = false;
+	const PROXY_ODYSEE = false;
+	const PROXY_DAILYMOTION = false;
+	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
+	const PROXY_FLICKR = false;
+	const PROXY_PIXABAY = false;
+	const PROXY_UNSPLASH = false;
+	const PROXY_PEXELS = false;
+	const PROXY_FIVEHPX = false;
+	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
-	const PROXY_GREPPR = false;
-	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
+	const PROXY_CARA = false;
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
+	const PROXY_SAFEBOORU = false;
+	const PROXY_KONACHAN = false;
+	const PROXY_YANDERE = false;
+	const PROXY_TBIB = false;
+	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
 	//
 	
+	// GOOGLE CSE & GOOGLE API
+	const GOOGLE_CX_ENDPOINT = "d4e68b99b876541f0";
+	
 	// MARGINALIA
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
+	
+	// Yep
+	const YEP_USE_API = false;
 }

docker-compose.yaml

@@ -6,14 +6,15 @@ services:
     image: luuul/4get:latest
     restart: unless-stopped
     environment:
+      - FOURGET_PROTO=http
       - FOURGET_SERVER_NAME=4get.ca
+      - FOURGET_INSTANCES=https://4get.ca
 
     ports:
       - "80:80"
       - "443:443"
 
-    volumes:
-      - /etc/letsencrypt/live/domain.tld:/etc/4get/certs
-      # mount custom banners and captcha
-      - ./banners:/var/www/html/4get/banner
-      - ./captcha:/var/www/html/4get/data/captcha
+    # volumes:
+    # - /etc/letsencrypt/live/domain.tld:/etc/4get/certs # mount ssl
+    # - ./banners:/var/www/html/4get/banner # mount custom banners
+    # - ./captcha:/var/www/html/4get/data/captcha # mount captcha images

docker/apache/http/conf.d/ssl.conf

@@ -0,0 +1 @@
+# intentionally blank

docker/docker-entrypoint.sh

@@ -8,18 +8,27 @@ FOURGET_PROTO="${FOURGET_PROTO#\"}"
 # make lowercase
 FOURGET_PROTO=`echo $FOURGET_PROTO | awk '{print tolower($0)}'`
 
+FOURGET_SRC='/var/www/html/4get'
+
+mkdir -p /etc/apache2
 
 if [ "$FOURGET_PROTO" = "https" ]; then
         echo "Using https configuration"
-        cp /etc/apache2/https.conf /etc/apache2/httpd.conf
+        cp -r ${FOURGET_SRC}/docker/apache/https/httpd.conf /etc/apache2
+        cp -r ${FOURGET_SRC}/docker/apache/https/conf.d/* /etc/apache2/conf.d
+
 else
         echo "Using http configuration"
-        cp /etc/apache2/http.conf /etc/apache2/httpd.conf
+        cp -r ${FOURGET_SRC}/docker/apache/http/httpd.conf /etc/apache2
+        cp -r ${FOURGET_SRC}/docker/apache/http/conf.d/* /etc/apache2/conf.d
 fi
 
 php ./docker/gen_config.php
 
-
-echo "4get is running"
-exec httpd -DFOREGROUND
+if [ "$@" = "start" ]; then
+        echo "4get is running"
+        exec httpd -DFOREGROUND
+else 
+        exec "$@"
+fi
 

docker/gen_config.php

@@ -13,7 +13,8 @@ $fourget_env = array_filter($env, function($v, $k) {
 
 foreach($fourget_env as $key => $val)  {
         $target_key = preg_replace('/^FOURGET_/', '', $key);
-        $from_env[$target_key] = trim($val, '\'"');
+        $val = trim($val, '\'"');
+        $from_env[$target_key] = match(strtolower($val)) { "true" => true, "false" => false, default => $val };
 };
 
 $merged_config = array_merge($from_config, $from_env);

docs/apache2-example.md

@@ -1,31 +1,16 @@
 # Sample Apache2 configuration
 This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
 
+Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
+
 Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
 
 ```xml
-<VirtualHost *:443>
-	ServerName www.4get.ca
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://4get.ca$1
-</VirtualHost>
-
-<VirtualHost *:443>
+<VirtualHost *:8080>
 	ServerName 4get.ca
+	ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
 
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/4get
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
+	# compress
 	AddOutputFilterByType DEFLATE application/json
 	AddOutputFilterByType DEFLATE application/javascript
 	AddOutputFilterByType DEFLATE application/x-javascript
@@ -33,163 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	AddOutputFilterByType DEFLATE text/plain
 	AddOutputFilterByType DEFLATE text/css
 	
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	DocumentRoot /var/www/4get
+	ErrorLog ${APACHE_LOG_DIR}/error.log
+	CustomLog /dev/null combined
+	#CustomLog /var/log/apache2/access.log combined
+	
+	<FilesMatch \.php$>
+		SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
+	</FilesMatch>
 
 	<Directory /var/www/4get>
 		Options -MultiViews
 		AllowOverride All
 		Require all granted
 		
+		# prevent 404s from hitting php-fpm
 		RewriteEngine On
+		RewriteCond %{REQUEST_URI} \.php$ [NC]
+		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteRule ^ - [R=404,L]
+		
+		# prevent /index hits
+		RewriteRule ^index$ - [R=404,L]
+		
+		# 404 direct .php file access
+		RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
+		RewriteRule ^ - [R=404,L]
+		
+		# internally redirect to non-php extension
 		RewriteCond %{REQUEST_FILENAME} !-d
 		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteCond %{REQUEST_FILENAME}.php -f
 		RewriteRule ^([^\.]+)$ $1.php [NC,L]
 	</Directory>
 
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
+		Require all denied
+	</DirectoryMatch>
+	
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
-
-<VirtualHost *:443>
-	ServerName www.lolcat.ca
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName lolcat.ca
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/lolcat
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-
-	<Directory /var/www/lolcat>
-		Options -MultiViews
-		AllowOverride All
-		Require all granted
-		
-		RewriteEngine On
-		RewriteCond %{REQUEST_FILENAME} !-d
-		RewriteCond %{REQUEST_FILENAME} !-f
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
-	</Directory>
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName www.nyym.co
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://nyym.co$1
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName nyym.co
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/nyym
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
-	
-	<Directory /var/www/nyym>
-		Options -MultiViews
-		AllowOverride All
-		Require all granted
-		
-		RewriteEngine On
-		RewriteCond %{REQUEST_FILENAME} !-d
-		RewriteCond %{REQUEST_FILENAME} !-f
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
-	</Directory>
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName git.lolcat.ca
-
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-
-	ProxyPreserveHost On
-	ProxyRequests off
-	AllowEncodedSlashes NoDecode
-	ProxyPass / http://localhost:3000/ nocanon
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName live.lolcat.ca
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/live
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-</VirtualHost>
+```
+
+# Haproxy configuration
+
+```cfg
+global
+	#log /dev/log	local0
+	log /dev/log	local0 warning
+	log /dev/log	local1 notice
+	chroot /var/lib/haproxy
+	stats socket /run/haproxy/admin.sock mode 660 level admin
+	stats timeout 30s
+	user haproxy
+	group haproxy
+	daemon
+
+	# Default SSL material locations
+	ca-base /etc/ssl/certs
+	crt-base /etc/ssl/private
+
+	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
+	#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+	ssl-default-bind-ciphers HIGH:!aNULL
+	ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+	ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
+	
+	# ja4 settings
+	tune.lua.bool-sample-conversion normal
+	tune.ssl.capture-buffer-size 190
+	lua-load /etc/haproxy/lua/ja4.lua
+defaults
+	log	global
+	mode	http
+	#option	httplog
+	option	dontlognull
+	timeout connect 5000
+	timeout client  50000
+	timeout server  50000
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 408 /etc/haproxy/errors/408.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
+	errorfile 504 /etc/haproxy/errors/504.http
+
+frontend http_80_in
+	# bind tcp ip4/6
+	bind *:80
+	bind [::]:80
+	
+	mode http
+	
+	# handle ip.lolcat.ca+onion addy on plain HTTP
+	acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
+		
+	# redirect to https
+	http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
+	
+	# redirect www.lolcat.ca to lolcat.ca
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
+	
+	# everything else redirects to 4get.ca
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
+	
+	# ... unless its an ip host, or the onion domain
+	use_backend apache if is_ip_host
+
+frontend https_443_in
+	# bind tcp ip4/6
+	bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
+	bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
+	
+	# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
+	# bind udp ip4/6
+	#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
+	#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
+	
+	# tell client we support http3 now
+	#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
+	
+	mode http
+	
+	# redirect all unknown domains to 4get.ca
+	acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
+	
+	# add onion domain to 4get.ca
+	acl is_4get hdr(host) -i 4get.ca
+	http-request set-var(txn.is_4get) bool(true) if is_4get
+	http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
+
+	# git.lolcat.ca -> gitea service
+	acl acl_gitea hdr(host) -i git.lolcat.ca
+	
+	# redirects
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
+	
+	# add security headers
+	http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+	http-response set-header X-Content-Type-Options "nosniff"
+	http-response set-header X-Frame-Options "SAMEORIGIN"
+	
+	# route
+	use_backend gitea if acl_gitea
+	default_backend apache
+
+backend apache
+	# pass ja4
+	http-request lua.fingerprint_ja4 if { ssl_fc }
+	http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
+	
+	# delete server header
+	http-response del-header Server
+	
+	# pass remote IP
+	option forwardfor
+	server apache_server 127.0.0.1:8080 check
+
+backend gitea
+	server gitea_server 127.0.0.1:3000 check
 ```

docs/apache2.md

@@ -7,27 +7,35 @@ Then, install the following dependencies:
 ```sh
 apt update
 apt upgrade
-apt install php-mbstring apache2 certbot php-imagick imagemagick php-curl curl php-apcu git libapache2-mod-php
+apt install php-mbstring apache2 certbot php-imagick imagemagick php-curl curl php-apcu git libapache2-mod-fcgid php-fpm
 ```
 
 Enable the required modules:
 ```sh
+a2dismod mpm_prefork
+a2enmod mpm_event
 a2enmod ssl
 a2enmod rewrite
+a2enmod proxy_fcgi setenvif actions alias
+a2enmod http2
+a2enmod headers
+a2enmod proxy
 ```
 
-And enable these optional ones, which might be useful to you later on. The `proxy` module is useful for setting up reverse proxies to services like gitea, and `headers` is useful to tweak global header values:
+Tune the performance of php-fpm. You will need to edit this file according to your server specs and number of users. Edit the file at `/etc/php/8.4/pool.d/www.conf`:
 ```sh
-a2enmod proxy
-a2enmod headers
+pm = static
+pm.max_children = 50
 ```
 
+These values are what I currently use on 4get.ca, but for personal use, you can set `pm` to `ondemand` and `pm.max_children` to `20` (if you want those thumbnails to load fast!)
+
 Now, restart apache2:
 ```sh
 service apache2 restart
 ```
 
-Just for good measure, please check if your webserver is running. Access it through HTTP, not HTTPS. You should see the apache2 default landing page.
+Just for good measure, please check if your webserver is running. Access it through HTTP, not HTTPS. You should see the apache2 default landing page. Just a note, http2 won't work just yet since you don't have SSL yet.
 
 ## 000-default.conf
 Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, remove everything and carefully add each rule specified here, while making sure to replace my domains with your own:
@@ -74,16 +82,35 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 
 	DocumentRoot /var/www/4get
 	
+	<FilesMatch \.php$>
+		SetHandler "proxy:unix:/run/php/php8.1-fpm.sock|fcgi://localhost/"
+	</FilesMatch>
+	
 	Options -MultiViews
 	RewriteEngine On
 	RewriteCond %{REQUEST_FILENAME} !-d
 	RewriteCond %{REQUEST_FILENAME} !-f
 	RewriteRule ^([^\.]+)$ $1.php [NC,L]
 	
+	<Directory /var/www/4get>
+		Options -MultiViews
+		AllowOverride All
+		Require all granted
+		
+		RewriteEngine On
+		RewriteCond %{REQUEST_FILENAME} !-d
+		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteRule ^([^\.]+)$ $1.php [NC,L]
+	</Directory>
+	
 	# deny access to private resources
-	<Directory /var/www/4get/data/>
-		Order Deny,allow
-		Deny from all
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
+		Require all denied
+	</DirectoryMatch>
+
+	# remove directory listing from /icons
+	<Directory /var/www/4get/icons>
+		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```
@@ -116,6 +143,7 @@ Make sure to replace `4get.ca` with your own domain under the `SSLCertificate*`
 	ServerAdmin will@lolcat.ca
 	DocumentRoot /var/www/4get
 	
+	Protocols h2 http/1.1
 	SSLEngine On
 	SSLOptions +StdEnvVars
 	
@@ -128,6 +156,10 @@ Make sure to replace `4get.ca` with your own domain under the `SSLCertificate*`
 	AddOutputFilterByType DEFLATE text/plain
 	AddOutputFilterByType DEFLATE text/css
 	
+	<FilesMatch \.php$>
+		SetHandler "proxy:unix:/run/php/php8.1-fpm.sock|fcgi://localhost/"
+	</FilesMatch>
+	
 	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
 	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
 	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem

docs/configure.md

@@ -9,34 +9,47 @@ Welcome! This guide assumes that you have a working 4get instance. This will hel
 4. The captcha font is located in `data/fonts/captcha.ttf`
 
 # Cloudflare bypass (TLS check)
-**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed.**
+>These instructions have been updated to work with Debian 13 Trixie.
 
-Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** & the **Mwmbl** search engines. Please be aware that APT will fight against you and will re-install the openSSL-version of curl constantly when updating.
+**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed by this program!**
 
-First, follow these instructions. Only install the Firefox modules:
+Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** search engine.
 
-https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build
-
-Once you did this, you should be able to run the following inside your terminal:
+To come up with this set of instructions, I used [this guide](https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build) as a reference, but trust me you probably want to stick to what's written on this page.
 
+First, compile curl-impersonate (the firefox flavor).
 ```sh
-$ curl_ff117 --version
-curl 8.1.1 (x86_64-pc-linux-gnu) libcurl/8.1.1 NSS/3.92 zlib/1.2.13 brotli/1.0.9 zstd/1.5.4 libidn2/2.3.3 nghttp2/1.56.0
-Release-Date: 2023-05-23
-Protocols: dict file ftp ftps gopher gophers http https imap imaps mqtt pop3 pop3s rtsp smb smbs smtp smtps telnet tftp ws wss
-Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTPS-proxy IDN IPv6 Largefile libz NTLM NTLM_WB SSL threadsafe UnixSockets zstd
-```
-Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine (on my debian install) is located at `/usr/local/lib/libcurl-impersonate-ff.so`.
-
-Find the `libcurl.so.4` file used by your current installation of curl. For me, this file is located at `/usr/lib/x86_64-linux-gnu/libcurl.so.4`
-
-Now comes the sketchy part: replace `libcurl.so.4` with `libcurl-impersonate-ff.so`. You can do this in the following way:
-```sh
-sudo rm /usr/lib/x86_64-linux-gnu/libcurl.so.4
-sudo cp /usr/local/lib/libcurl-impersonate-ff.so /usr/lib/x86_64-linux-gnu/libcurl.so.4
+git clone https://github.com/lwthiker/curl-impersonate/
+cd curl-impersonate
+sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool python3-pip libnss3 libnss3-dev
+mkdir build
+cd build
+../configure
+make firefox-build
+sudo make firefox-install
+sudo ldconfig
 ```
 
-Make sure to restart your webserver and/or PHP daemon, otherwise it will keep using the old library. You should now be able to bypass Cloudflare's shitty checks!!
+Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine is located at `/usr/local/lib/libcurl-impersonate-ff.so`. Patch your PHP install so that it loads the right library:
+
+```sh
+sudo systemctl edit php8.4-fpm.service
+```
+
+^This will open a text editor. Add the following shit in there, in between those 2 comments I pasted for ya just for reference:
+
+```sh
+### Editing /etc/systemd/system/php8.4-fpm.service.d/override.conf
+### Anything between here and the comment below will become the contents of the>
+
+[Service]
+Environment="LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so"
+Environment="CURL_IMPERSONATE=firefox117"
+
+### Edits below this comment will be discarded
+```
+
+Restart php8.4-fpm. (`sudo service php8.4-fpm restart`). To test things out, try making a search on "Yep", they check for SSL. If you get results (or a timeout, this piece of shit engine is slow as fuck) that means it works!
 
 # Robots.txt
 Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the 4get.ca domain to your own domain.
@@ -66,3 +79,93 @@ Done! The scraper you chose should now be using the rotating proxies. When askin
 
 ## Important!
 If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
+
+# 4play setup
+4play is a Firefox extension I wrote that lets you control a browser from 4get. When you do a search, 4get sends a query to a nodeJS http server that communicates to the browser via websockets. The browser then reports back various payloads used for scraping. It's very similar to tools like puppeteer or playwright, except it uses websockets to communicate with the browser instead of the browser's debugger. This is needed to prevent `navigator.webdriver` from being set to `true` (also avoids a fuckton of other botchecks, that's just the tip of the iceberg).
+
+## Requirements
+- A dedicated laptop or computer running a full desktop environment (like Windows, Mac, or Linux DEs)
+- Something to make the computer think it's connected to a screen. You need it, otherwise you'll fallback to software rendering and sites can detect that. You can use:
+	- Your laptop's screen
+	- A turned-on screen connected via HDMI
+	- An EDID adapter to simulate a display (no affiliate links: [HDMI](https://www.aliexpress.com/item/1005011668390064.html), [DisplayPort](https://www.aliexpress.com/item/1005011659561729.html))
+- Firefox ESR or the latest version of Firefox
+- The [4play extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) installed
+
+## Installation
+Install the dependencies on the machine you wish to run 4play on. In a sane setup, you usually want 4play to run on a dedicated user-grade machine. For the sake of simplicity, this machine will be referred as the "render server".
+
+```sh
+git clone https://git.lolcat.ca/lolcat/4get
+cd 4get/extra/4play
+
+# install nodejs 26.x or later if it's not already installed
+
+npm install @lawlers/4play
+sudo npm install -g nodemon
+
+# edit line 7 where it says 'var password = "cnc";'. Use a strong password.
+nano page-render.js
+
+# test the server
+nodemon page-render.js
+```
+
+I recommend setting up a systemd service to auto-start the 4play server:
+```sh
+sudo nano /etc/systemd/system/fplay.service
+```
+
+Paste this in (make sure to update `User` & `WorkingDirectory`)
+```sh
+[Unit]
+Description=4play
+After=network.target
+
+[Service]
+Type=simple
+User=will
+WorkingDirectory=/home/will/Desktop/4get/extra/4play/
+ExecStart=nodemon /home/will/Desktop/4get/extra/4play/page-render.js
+Restart=always
+Environment=NODE_ENV=production
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Now, enable the service
+```sh
+sudo systemctl enable fplay
+sudo service fplay start
+sudo service fplay status # check the status. You should see it running
+```
+
+This should expose a webservice located at `http://localhost:3000`. I highly recommend you put this endpoint behind apache2 or nginx to add SSL, if the render server sits on a different network.
+
+## Setting up Firefox
+- Install Firefox. You can go for the ESR version or the latest version. You'll figure it out.
+- Head over to [this page](https://addons.mozilla.org/en-US/firefox/addon/4play/) or search "4play" on the Firefox extension store and install the extension.
+- Click the settings cog next to the extension name. Enable "Run in Private Windows" and "Allow automatic updates"
+- Click the extension in the top bar. Modify the `ws://localhost:3030/cnc` URL: replace the `cnc` path with the password you set earlier for the server.
+- Use a timeout value of `30000` to avoid getting throttled by Firefox during reconnections.
+
+If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
+
+**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
+
+## Setting up 4get
+Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
+
+```php
+const FPLAY_PASSWORD = "cnc"; # Change this to your password
+const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.com/api/example";, with the quotes and semicolon
+```
+
+**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
+
+If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
+
+In case it isn't clear, here is a visual representation of what you want to achieve:
+
+![Network map](https://i.imgur.com/jFjbBtG.png)

docs/develop.md

@@ -0,0 +1,36 @@
+# Develop
+
+Start the PHP webserver like this, assuming you have curl-impersonate installed already
+
+```sh
+LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
+```
+
+Make sure to add `.php` to paths, the embedded PHP webserver doesnt support `mod_rewrite`-like directives.
+
+If, for some reason, you're in an environment where accessing the debug logs is impossible (like those shitty shared webhosts), you can set `DISPLAY_ERRORS` to `true` in `data/config.php`.
+
+# Debug guide for docker
+
+Install:
+```sh
+git clone https://git.lolcat.ca/lolcat/4get
+# make changes to docker/apache/http/httpd.conf (comment out the lines that disable logging)
+docker build -t luuul/4get .
+```
+
+Run:
+```sh
+docker run --name 4get -p 80:80 -p 443:443 luuul/4get 
+```
+
+Get debug logs:
+```sh
+docker exec -it `docker ps -qf ancestor=luuul/4get:latest` sh # browse the container
+docker exec -it `docker ps -qf ancestor=luuul/4get:latest` tail -f /var/log/apache2/error_log
+```
+
+Restart the container:
+```sh
+docker restart 4get
+```

docs/nginx.md

@@ -1,103 +1,194 @@
-# Install on NGINX
+<h1 align=center>Installation of 4get in NGINX</h1>
 
->I do NOT recommend following this guide, only follow this if you *really* need to use nginx. I recommend you use the apache2 steps instead.
+<div align=right>
 
-Login as root.
+> NOTE: As the previous version stated, it is better to follow the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">Apache2 guide</a> instead of the Nginx one.
 
-Create a file in `/etc/nginx/sites-avaliable/` called `4get.conf` or any name you want and put this into the file:
+> NOTE: This is going to guess that you're using either a <abbr title="(Arch Linux, Artix Linux, Endeavouros, etc...) ">Arch-based system</abbr> or a <abbr title="(Debian, Ubuntu, Devuan, etc...)">Debian-based system</abbr>, although you can still follow it with minor issues.
 
-```
-server {
-    # DO YOU REALLY NEED TO LOG SEARCHES?
-    access_log /dev/null;
-    error_log /dev/null;
-    # Change this if you have 4get in other folder.
-    root /var/www/4get;
-    # Change yourdomain by your domain lol
-    server_name www.yourdomain.com yourdomain.com;
+</div>
 
-    location @php {
-        try_files $uri.php $uri/index.php =404;
-                # Change the unix socket address if it's different for you.
-        fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock;
-        fastcgi_index index.php;
-                # Change this to `fastcgi_params` if you use a debian based distro.
-        include fastcgi.conf;
-        fastcgi_intercept_errors on;
-    }
+1. Login as root.
+2. Upgrade your system:
+   * On Arch-based, run `pacman -Syu`.
+   * On Debian-based, run `apt update`, then `apt upgrade`.
+3. Install the following dependencies:
+   * `git`: So you can clone <a href="https://git.lolcat.ca/lolcat/4get">this</a> repository.
+   * `nginx`: So you can run Nginx.
+   * `php-fpm`: This is what allows Nginx to run *(and show)* PHP files.
+   * `php-imagick`, `imagemagick`: Image manipulation.
+   * `php-apcu`: Caching module.
+   * `php-curl`, `curl`:  Transferring data with URLs.
+   * `php-mbstring`: String utils.
+   * `certbot`, `certbot-nginx`: ACME client. Used to create SSL certificates.
+     * In Arch-based distributions:
+       * `pacman -S nginx certbot php-imagick certbot-nginx imagemagick curl php-apcu git`
+     * In Debian-based distributions:
+       * `apt install php-mbstring nginx certbot-nginx certbot php-imagick imagemagick php-curl curl php-apcu git`
 
-    location / {
-        try_files $uri @php;
-    }
+<div align=right>
 
-    location ~* ^(.*)\.php$ {
-        return 301 $1;
-    }
+> IMPORTANT: `php-curl`, `php-mbstring` might be a Debian-only package, but this needs further fact checking.
 
+> IMPORTANT: If having issues with `php-apcu` or `libsodium`, go to [^1].
+
+</div>
+
+4. `cd` to `/etc/nginx` and make the `conf.d/` directory if it doesn't exist:
+   * Again, this guesses you're logged in as root.
+   ```sh
+   cd /etc/nginx
+   ls -l conf.d/ # If ls shows conf.d, then it means it exists.
+   # If it does not, run:
+   mkdir conf.d
+   ```
+5. Make a file inside `conf.d/` called `4get.conf` and place the following content:
+   * First run `touch conf.d/4get.conf` then `nano conf.d/4get.conf` to open the nano editor: *(Install it if it is not, or use another editor.)*
+    ```sh
+    server {
+        access_log /dev/null; # Search log file. Do you really need to?
+        error_log /dev/null;  # Error log file.
+
+        # Change this if you have 4get in another folder.
+        root /var/www/4get;
+        # Change 'yourdomain' to your domain.
+        server_name www.yourdomain.com yourdomain.com;
+        # Port to listen to.
         listen 80;
-}
-```
 
-That is a very basic config so you will need to adapt it to your needs in case you have a more complicated nginx configuration. Anyways, you can see a real world example [here](https://git.zzls.xyz/Fijxu/etc-configs/src/branch/selfhost/nginx/sites-available/4get.zzls.xyz.conf)
+        location @php {
+            try_files $uri.php $uri/index.php =404;
+            # Change the unix socket address if it's different for you.
+            fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock;
+            fastcgi_index index.php;
+            # Change this to `fastcgi_params` if you use a debian based distribution.
+            include fastcgi.conf;
+            fastcgi_intercept_errors on;
+        }
 
-After you save the file you will need to do a symlink of the `4get.conf` file to `/etc/nignx/sites-enabled/`, you can do it with this command: 
+        location / {
+            try_files $uri @php;
+        }
 
-```sh
-ln -s /etc/nginx/sites-available/4get.conf /etc/nginx/sites-available/4get.conf
-```
+        location ~* ^(.*)\.php$ {
+            return 301 $1;
+        }
 
-Now test the nginx config with `nginx -t`, if it says that everything is good, restart nginx using `systemctl restart nginx`
-
-# Encryption setup
-
-Generate a certificate for the domain using:
-
-```sh
-certbot --nginx --key-type ecdsa -d www.yourdomain.com -d yourdomain.com
-```
-(Remember to install the nginx certbot plugin!!!)
-
-After doing that certbot should deploy the certificate automatically into your 4get nginx config file. It should be ready to use at that point.
-
-# Tor setup on NGINX
-
-Important Note: Tor onion addresses are significantly longer than traditional domain names. Before proceeding with Nginx configuration, ensure you increase the `server_names_hash_bucket_size` value in your `nginx.conf` file. This setting in your Nginx configuration controls the internal data structure used to manage multiple server names (hostnames) associated with your web server. Each hostname requires a certain amount of memory within this structure. If the size is insufficient, Nginx will encounter errors.
-
-1. Open your `nginx.conf` file (that is under `/etc/nginx/nginx.conf`).
-2. Find the line containing `# server_names_hash_bucket_size 64;`.
-3. Uncomment the line and adjust the value. Start with 64, but if you encounter issues, incrementally increase it (e.g., 128, 256) until it accommodates your configuration.
-
-Open your current 4get NGINX config (that is under `/etc/nginx/sites-available/`) and append this to the end of the file:
-
-```
-server {
-	access_log /dev/null;
-	error_log /dev/null;
-
-    listen 80;
-    server_name <youronionaddress>;
-    root /var/www/4get;
-
-    location @php {
-        try_files $uri.php $uri/index.php =404;
-        # Change the unix socket address if it's different for you.
-        fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock;
-        fastcgi_index index.php;
-        # Change this to `fastcgi_params` if you use a debian based distro.
-        include fastcgi.conf;
-        fastcgi_intercept_errors on;
     }
+    ```
+    * The above is a very basic configuration and thus will need tweaking to your personal needs. It should still work as-is, though. A 'real world' example is present in [^2].
+    * After saving the file, check that the `nginx.conf` file inside the main directory includes files inside `conf.d/`:
+      * It should be inside the the http block: *(The following is an example! Don't just Copy and Paste it!)*
+      ```sh
+      http {
+        include       mime.types;
+        include       conf.d/*.conf; 
+        types_hash_max_size 4096;
+        # ...
+      }
+      ```
+    * Now, test your configuration with `nginx -t`, if it says that everything is good, restart *(or start)* the Nginx daemon:
+      * This depends on the init manager, most distributions use `systemd`, but it's better practice to include most.
+      ```sh
+      # systemd
+      systemctl stop nginx
+      systemctl start nginxt
+      # or
+      systemctl restart nginx
+
+      # openrc
+      rc-service nginx stop
+      rc-service nginx start
+      # or
+      rc-service nginx restart
+
+      # runit
+      sv down nginx
+      sv up nginx
+      # or
+      sv restart nginx
+
+      # s6
+      s6-rc -d change nginx
+      s6-rc -u change nginx
+      # or
+      s6-svc -r /run/service/nginx
+
+      # dinit
+      dinitctl stop nginx
+      dinitctl start nginx
+      # or
+      dinitctl restart nginx
+      ```
+6. Clone the repository to `/var/www`:
+   * `git clone --depth 1 https://git.lolcat.ca/lolcat/4get 4get` - It clones the repository with the depth of one commit *(so it takes less time to download)* and saves the cloned repository as '4get'.
+7. That should be it! There are some extra steps you can take, but it really just depends on you.
+
+<h2 align=center>Encryption setup</h2>
+
+1. Generate a certificate for the domain you're using with:
+   * Note that `certbot-nginx` is needed.
+    ```sh
+    certbot --nginx --key-type ecdsa -d www.yourdomain.com -d yourdomain.com
+    ```
+2. After that, certbot will deploy the certificate automatically to your 4get conf file; It should be ready to use from there.
+
+<h2 align=center>Tor Setup</h2>
+
+<div align=right>
+
+> IMPORTANT: Tor onion addresses are very long compared to traditional domains, so, Before doing anything, edit `nginx.conf` and increase <abbr title="This setting in your Nginx configuration controls the internal data structure used to manage multiple server names (hostnames) associated with your web server. Each hostname requires a certain amount of memory within this structure. If the size is insufficient, Nginx will encounter errors."><code>server_names_hash_bucket_size</code></abbr> to your needs.
+
+</div>
+
+1. `cd` to `/etc/nginx` *(if you haven't)* and open your `nginx.conf` file.
+2. Find the line containing `# server_names_hash_bucket_size 64;` inside said file.
+3. Uncomment the line and adjust the value; start with 64, but if you encounter issues, incrementally increase it *(e.g., 128, 256)* until it accommodates your configuration.
+4. Open *(or duplicate the configuration)* and edit it:
+   * Example configuration, again:
+    ```sh
+    server {
+        access_log /dev/null; # Search log file. Do you really need to?
+        error_log /dev/null;  # Error log file.
+
+        # Change this if you have 4get in another folder.
+        root /var/www/4get;
+        # Change 'onionadress.onion' to your onion link.
+        server_name onionadress.onion;
+        # Port to listen to.
+        listen 80;
+
+        location @php {
+            try_files $uri.php $uri/index.php =404;
+            # Change the unix socket address if it's different for you.
+            fastcgi_pass unix:/var/run/php-fpm/php-fpm.sock;
+            fastcgi_index index.php;
+            # Change this to `fastcgi_params` if you use a debian based distribution.
+            include fastcgi.conf;
+            fastcgi_intercept_errors on;
+        }
+
+        location / {
+            try_files $uri @php;
+        }
+
+        location ~* ^(.*)\.php$ {
+            return 301 $1;
+        }
 
-    location / {
-        try_files $uri @php;
     }
+    ```
+    A real world example is present in [^2].
+5. Once done, check the configuration with `nginx -t`. If everything's fine and dandy, refer to <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/tor.md">the Tor guide</a> to setup your onion site.
 
-    location ~* ^(.*)\.php$ {
-        return 301 $1;
-    }
-}
-```
+<h2 align=center>Other important things</h2>
 
-Obviously replace `<youronionaddress>` by the onion address of `/var/lib/tor/4get/hostname` and then check if the nginx config is valid with `nginx -t` if yes, then restart the nginx service and try opening the onion address into the Tor Browser. You can see a real world example [here](https://git.zzls.xyz/Fijxu/etc-configs/src/branch/selfhost/nginx/sites-available/4get.zzls.xyz.conf)
+1. <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/configure.md">Configuration guide</a>: Things to do after setup.
+2. <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">Apache2 guide</a>: Fallback to this if you couldn't get something to work, or you don't know something.
 
-Once you did the above, refer to <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/tor.md">this tor guide</a> to setup your onionsite.
+<h2 align=center>Known issues</h2>
+
+1. https://git.lolcat.ca/lolcat/4get/issues
+
+[^1]: lolcat/4get#40, If having issues with `libsodium`, or `php-apcu`.
+[^2]: <a href="https://git.nadeko.net/Fijxu/etc-configs/src/branch/selfhost/nginx/conf.d/4get.conf">git.nadeko.net</a> nadeko.net's 4get instance configuration.

extra/4play/page-render.js

@@ -0,0 +1,337 @@
+const fplay = require("@lawlers/4play");
+const http = require("http");
+
+// 4play settings
+var port = 3030;
+var timeout = 30000;
+var password = "cnc";
+
+// ==================================================
+
+// webserver
+
+var global_ws = null;
+const webserver_port = 3000;
+
+var callback_q = new Map();
+
+const server = http.createServer(async function(req, res){
+	
+	const parsed = new URL(req.url, "http://localhost");
+	
+	const url = parsed.searchParams.get("url");
+	const pw = parsed.searchParams.get("pw");
+	var proxy = parsed.searchParams.get("proxy");
+	const url_match = parsed.searchParams.get("url_match");
+	const content_match = parsed.searchParams.get("content_match");
+	const fail_url_match = parsed.searchParams.get("fail_url_match");
+	const fail_content_match = parsed.searchParams.get("fail_content_match");
+	
+	var container = parsed.searchParams.get("container");
+	
+	if(fplay.browser_connected === false){
+				
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("No browser available to render the motherfucking page");
+		return;
+	}
+	
+	var errors = [];
+	
+	if(
+		pw === null ||
+		pw != password
+	){
+		
+		errors.push("pw");
+	}
+	
+	if(url === null){ errors.push("url"); }
+	if(proxy === null){ errors.push("proxy"); }
+	
+	if(errors.length !== 0){
+		
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain"
+			}
+		);
+		
+		res.end("The following parameters are missing or invalid: " + errors.join(", "));
+		return;
+	}
+	
+	console.log("Rendering " + url);
+	
+	if(
+		container === null ||
+		await fplay.container_exists(global_ws, container) === false
+	){
+		
+		container = await fplay.container_create(global_ws);
+		
+		proxy = get_proxy(proxy);
+		
+		// assign proxy
+		if(proxy.config !== false){
+			
+			await fplay.container_attach_proxy(
+				global_ws,
+				container,
+				proxy.config
+			);
+		}
+	}else{
+		
+		proxy = {raw: proxy};
+	}
+	
+	// open tab
+	const newtab = await fplay.tab_open(global_ws, url, false, container);
+	
+	if(newtab === false){
+		
+		// failed to open tab
+		res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": proxy.raw,
+				"X-Container-ID": typeof container.id == "undefined" ? container : container.id
+			}
+		);
+		
+		res.end("Failed to open tab");
+		return;
+	}
+		
+	// callback will be handled in "web_response"
+	callback_q.set(
+		newtab.id,
+		{
+			http_res: res,
+			proxy: proxy,
+			container: container,
+			url_match: url_match,
+			content_match: content_match,
+			fail_url_match: fail_url_match,
+			fail_content_match: fail_content_match
+		}
+	);
+	
+	// close tab in 30 seconds
+	await fplay.wait_random(15000, 35000);
+	await fplay.tab_close(global_ws, newtab);
+});
+
+server.listen(webserver_port, function(){
+	
+	console.log("http server running");
+});
+
+function get_proxy(proxy_line){
+	
+	parts = proxy_line.split(":");
+	// format: type:address:port:username:password
+	// PHP explode(":", $ip, 5) splits into max 5 parts
+	var type = parts[0];
+	const address = parts[1];
+	const port = parts[2];
+	const username = parts[3];
+	const password = parts[4];
+	
+	var proxy_dns = true;
+	
+	switch(type){
+		case "raw_ip":
+		case "ip":
+			// no proxy, use direct connection
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+			break;
+		
+		case "socks5":
+			type = "socks";
+			proxy_dns = false;
+			break;
+			
+		case "socks4a":
+			type = "socks4";
+			break;
+			
+		case "socks4":
+			proxy_dns = false;
+			break;
+			
+		case "socks5_hostname":
+		case "socks5h":
+		case "socks5a":
+			type = "socks";
+			break;
+		
+		case "http":
+		case "https":
+			// do nothing
+			break;
+		
+		default:
+			// invalid proxy provided
+			return {
+				config: false,
+				raw: "raw_ip::::"
+			}
+	}
+	
+	return {
+		config: {
+			type: type,
+			host: address,
+			port: port,
+			username: username,
+			password: password,
+			proxyDNS: proxy_dns
+		},
+		raw: proxy_line
+	};
+}
+
+fplay.event.on("server_ready", function(){
+	
+	console.log("listening on port " + port + " (timeout=" + timeout + ")");
+});
+
+fplay.event.on("browser_connect", async function(ws){
+	
+	global_ws = ws;
+	
+	const ua = await fplay.get_ua(ws);
+	console.log("New browser instance connected: " + ua);
+	
+	await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+	
+	fplay.web_response_whitelist(ws, ["main_frame"]);
+	
+	// clean up
+	const blanktab = await fplay.close_all_tabs(ws);
+	await fplay.delete_all_containers(ws);
+});
+
+fplay.event.on("web_response", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	var body = page.body.toString();
+	
+	var url_match = callback_data.url_match !== null ? page.url.match(new RegExp(callback_data.url_match, "i")) : null;
+	var content_match = callback_data.content_match !== null ? body.match(new RegExp(callback_data.content_match, "i")) : null;
+	var fail_url_match = callback_data.fail_url_match !== null ? page.url.match(new RegExp(callback_data.fail_url_match, "i")) : null;
+	var fail_content_match = callback_data.fail_content_match !== null ? body.match(new RegExp(callback_data.fail_content_match, "i")) : null;
+	
+	// handle negative response
+	if(
+		(
+			callback_data.fail_url_match !== null ||
+			callback_data.fail_content_match !== null
+		) &&
+		(
+			fail_url_match !== null ||
+			fail_content_match !== null
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			400,
+			{
+				"Content-Type": "text/plain",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)");
+		
+		// cleanup browser + callback
+		await fplay.tab_close(global_ws, page.id);
+		await fplay.container_delete(global_ws, callback_data.container);
+		callback_q.delete(page.id);
+		
+		return;
+	}
+	
+	// handle positive response
+	if(
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match === null
+		) ||
+		(
+			callback_data.url_match !== null &&
+			callback_data.content_match !== null &&
+			url_match &&
+			content_match
+		) ||
+		(
+			callback_data.url_match === null &&
+			callback_data.content_match !== null &&
+			content_match
+		) ||
+		(
+			callback_data.url_match !== null &&
+			callback_data.content_match === null &&
+			url_match
+		)
+	){
+		
+		callback_data.http_res.writeHead(
+			200,
+			{
+				"Content-Type": "text/plain;charset=UTF-8",
+				"X-Proxy": callback_data.proxy.raw,
+				"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+			}
+		);
+		
+		callback_data.http_res.end(body);
+		
+		callback_q.delete(page.id); // cleanup callback only
+		return;
+	}
+});
+
+fplay.event.on("dom_load_fail", async function(page){
+	
+	var callback_data = callback_q.get(page.id);
+	
+	if(typeof callback_data == "undefined"){ return; }
+	
+	callback_q.delete(page.id);
+	
+	callback_data.http_res.writeHead(
+		400,
+		{
+			"Content-Type": "text/plain",
+			"X-Proxy": callback_data.proxy.raw,
+			"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
+		}
+	);
+	
+	callback_data.http_res.end("Network error (" + page.error + ")");
+		
+	// cleanup browser
+	await fplay.tab_close(global_ws, page.id);
+	await fplay.container_delete(global_ws, callback_data.container);
+});
+
+fplay.init(port, password, timeout);

extra/gen_table.php

@@ -0,0 +1,151 @@
+<?php
+
+$scrapers = [
+	"web" => [
+		"DuckDuckGo",
+		"Brave",
+		"Yandex",
+		"Google",
+		"Google API",
+		"Google CSE",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Yep",
+		"Mwmbl",
+		"Mojeek",
+		"Naver",
+		"Baidu",
+		"Cốc Cốc",
+		"Solofield",
+		"Marginalia",
+		"Purili",
+		"wiby"
+	],
+	"images" => [
+		"DuckDuckGo",
+		"Yandex",
+		"Brave",
+		"Google",
+		"Google API",
+		"Google CSE",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Naver",
+		"Baidu",
+		"Solofield",
+		"Pinterest",
+		"Flickr",
+		"Pexels",
+		"Pixabay",
+		"Unsplash",
+		"500px",
+		"VSCO",
+		"Imgur",
+		"FindThatMeme"
+	],
+	"videos" => [
+		"YouTube",
+		"Vimeo",
+		"Sepia Search",
+		"DuckDuckGo",
+		"Brave",
+		"Yandex",
+		"Google",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Naver",
+		"Baidu",
+		"Cốc Cốc",
+		"Purili",
+		"Solofield"
+	],
+	"news" => [
+		"DuckDuckGo",
+		"Brave",
+		"Google",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Baidu"
+	],
+	"music" => [
+		"SoundCloud",
+		"Swisscows"
+	],
+	"autocomplete" => [
+		"Brave",
+		"DuckDuckGo",
+		"Yandex",
+		"Google",
+		"Qwant",
+		"Marginalia",
+		"YouTube",
+		"SoundCloud",
+		"Startpage",
+		"Kagi"
+	]
+];
+
+$lines = [
+	0 => "",
+	1 => ""
+];
+
+// get longest scraper string
+// also get list of scrapers with most entries
+$biggest_category_len = 0;
+foreach($scrapers as $category_name => $category){
+	
+	$count = count($category);
+	
+	if($count > $biggest_category_len){
+		
+		$biggest_category_len = $count;
+	}
+}
+
+foreach($scrapers as $category_name => $category){
+	
+	// get longest string of the category
+	$longest_len = mb_strlen($category_name);
+	foreach($category as $scraper){
+		
+		$strlen = mb_strlen($scraper);
+		
+		if($strlen > $longest_len){
+			
+			$longest_len = $strlen;
+		}
+	}
+	
+	// add header
+	$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
+	$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
+	
+	for($i=2; $i<count($category) + 2; $i++){
+		
+		if(!isset($lines[$i])){
+			
+			$lines[$i] = "";
+		}
+		
+		$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
+	}
+	
+	if($i - 2 < $biggest_category_len){
+		
+		for($k=$i; $k<$biggest_category_len + 2; $k++){
+			
+			if(!isset($lines[$k])){
+				
+				$lines[$k] = "";
+			}
+			
+			$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
+		}
+	}
+}
+
+foreach($lines as $line){
+	
+	echo $line . "|\n";
+}

favicon.php

@@ -15,7 +15,12 @@ class favicon{
 		
 		header("Content-Type: image/png");
 		
-		if(substr_count($url, "/") !== 2){
+		if(
+			preg_match(
+				'/^https?:\/\/[A-Za-z0-9.-]+$/',
+				$url
+			) === 0
+		){
 			
 			header("X-Error: Only provide the protocol and domain");
 			$this->defaulticon();

images.php

@@ -24,6 +24,9 @@ $payload = [
 	"nextpage" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->image($get);
 	

lib/anubis.php

@@ -0,0 +1,103 @@
+<?php
+
+//
+// Reference
+// https://github.com/TecharoHQ/anubis/blob/ecc716940e34ebe7249974f2789a99a2c7115e4e/web/js/proof-of-work.mjs
+//
+
+class anubis{
+	
+	public function __construct(){
+		
+		include_once "fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function scrape($html){
+		
+		$this->fuckhtml->load($html);
+		
+		$script =
+			$this->fuckhtml
+			->getElementById(
+				"anubis_challenge",
+				"script"
+			);
+		
+		if($script === false){
+			
+			throw new Exception("Failed to scrape anubis challenge data");
+		}
+		
+		$script =
+			json_decode(
+				$this->fuckhtml
+				->getTextContent(
+					$script
+				),
+				true
+			);
+		
+		if($script === null){
+			
+			throw new Exception("Failed to decode anubis challenge data");
+		}
+		
+		if(
+			!isset($script["challenge"]) ||
+			!is_string($script["challenge"]["randomData"]) ||
+			!isset($script["rules"]["difficulty"]) ||
+			!is_int($script["rules"]["difficulty"])
+		){
+			
+			throw new Exception("Found invalid challenge data");
+		}
+		
+		return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
+	}
+	
+	private function is_valid_hash($hash, $difficulty){
+		
+		for ($i=0; $i<$difficulty; $i++) {
+			
+			$index = (int)floor($i / 2);
+			$nibble = $i % 2;
+			
+			$byte = ord($hash[$index]);
+			$nibble = ($byte >> ($nibble === 0 ? 4 : 0)) & 0x0f;
+			
+			if($nibble !== 0){
+				return false;
+			}
+		}
+		
+		return true;
+	}
+	
+	public function rape($id, $data, $difficulty = 5){
+		
+		$nonce = 0;
+		$microtime = microtime(true);
+		
+		while(true){
+			
+			$hash_binary = hash("sha256", $data . $nonce, true);
+			
+			if($this->is_valid_hash($hash_binary, $difficulty)){
+				
+				$hash_hex = bin2hex($hash_binary);
+				
+				return [
+					"id" => $id,
+					"response" => $hash_hex,
+					//"data" => $data,
+					//"difficulty" => $difficulty,
+					"nonce" => $nonce,
+					"elapsedTime" => round((microtime(true) - $microtime) * 1000)
+				];
+			}
+			
+			$nonce++;
+		}
+	}
+}

lib/backend.php

@@ -4,14 +4,17 @@ class backend{
 	public function __construct($scraper){
 		
 		$this->scraper = $scraper;
+		$this->db = null;
+		$this->db_path_prefix = "data/sessions/";
+		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
 	}
 	
 	/*
 		Proxy stuff
 	*/
-	public function get_ip(){
+	public function get_ip($proxy_index_raw = null, $use_alt_list = false){
 		
-		$pool = constant("config::PROXY_" . strtoupper($this->scraper));
+		$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
 		if($pool === false){
 			
 			// we don't want a proxy, fuck off!
@@ -19,7 +22,10 @@ class backend{
 		}
 		
 		// indent
-		$proxy_index_raw = apcu_inc("p." . $this->scraper);
+		if($proxy_index_raw === null){
+			
+			$proxy_index_raw = apcu_inc("p." . $this->scraper);
+		}
 		
 		$proxylist = file_get_contents("data/proxies/" . $pool . ".txt");
 		$proxylist = explode("\n", $proxylist);
@@ -32,6 +38,12 @@ class backend{
 		
 		$proxylist = array_values($proxylist);
 		
+		if(count($proxylist) === 0){
+			
+			throw new Exception("A proxy list was specified but it's empty!");
+		}
+		
+		//echo $proxylist[$proxy_index_raw % count($proxylist)];
 		return $proxylist[$proxy_index_raw % count($proxylist)];
 	}
 	
@@ -75,6 +87,7 @@ class backend{
 				break;
 			
 			case "socks5_hostname":
+			case "socks5h":
 			case "socks5a":
 				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
 				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
@@ -87,6 +100,30 @@ class backend{
 		}
 	}
 	
+	// API key rotation
+	public function get_key(){
+		
+		$keys = file_get_contents("data/api_keys/" . $this->scraper . ".txt");
+		$keys = explode("\n", $keys);
+		
+		$keys = array_filter($keys, function($entry){
+			$entry = ltrim($entry);
+			return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
+		});
+		
+		$keys = array_values($keys);
+		
+		if(count($keys) === 0){
+			
+			throw new Exception("Please specify API keys in data/api_keys/" . $this->scraper . ".txt");
+		}
+		
+		$increment = apcu_inc("s." . $this->scraper) % count($keys);
+		return [
+			"key" => $keys[$increment],
+			"increment" => $increment
+		];
+	}
 	
 	
 	/*
@@ -174,4 +211,127 @@ class backend{
 			$payload[1] // proxy
 		];
 	}
+	
+	//
+	// Sessions
+	//
+	public function session_init_db($force_init = false){
+		
+		if(
+			$force_init ||
+			$this->db === null
+		){
+			
+			$this->db = new SQLite3($this->db_path);
+			
+			// create table if it doesnt exist	
+			$this->db->exec(
+				"CREATE TABLE IF NOT EXISTS sessions ( " .
+					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
+					"created_at INTEGER NOT NULL, " .
+					"proxy TEXT NOT NULL, " .
+					"headers_json TEXT NOT NULL" .
+				")"
+			);
+			
+			return true;
+		}
+		
+		return false;
+	}
+	
+	public function session_count(){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"SELECT COUNT(*) as count FROM sessions"
+		);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return (int)($row["count"] ?? 0);
+	}
+	
+	public function session_write($proxy, $headers){
+		
+		$this->session_init_db();
+		
+		$stmt = $this->db->prepare(
+			"INSERT INTO sessions ( " .
+				"created_at, " .
+				"proxy, " .
+				"headers_json " .
+			") VALUES ( " .
+				":created_at, " .
+				":proxy, " .
+				":headers_json" .
+			")"
+		);
+		
+		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
+		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
+		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
+		
+		$stmt->execute();
+		
+		return $this->db->lastInsertRowID();
+	}
+	
+	public function session_get(){
+		
+		$count = $this->session_count();
+		
+		if($count === 0){
+			
+			return false;
+		}
+		
+		$offset = apcu_inc("d." . $this->scraper) % $count;
+		
+		$stmt = $this->db->prepare(
+			"SELECT * " .
+			"FROM sessions " .
+			"ORDER BY id ASC " .
+			"LIMIT 1 OFFSET :offset"
+		);
+		
+		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
+		
+		$result = $stmt->execute();
+		$row = $result->fetchArray(SQLITE3_ASSOC);
+		
+		return [
+			"id" => $row["id"],
+			"created_at" => $row["created_at"],
+			"proxy" => $row["proxy"],
+			"headers" => json_decode($row["headers_json"], true)
+		];
+	}
+	
+	public function session_destroy_list(){
+		
+		if(
+			!preg_match(
+				'/^[A-Za-z0-9-]+$/',
+				$this->scraper
+			)
+		){
+			
+			// path traversal
+			return false;
+		}
+		
+		if(
+			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
+			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
+		){
+			
+			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
+			return true;
+		}
+		
+		return false;
+	}
 }

lib/bingcache-todo-fix.php

@@ -1,144 +0,0 @@
-<?php
-
-// https://www.bing.com/search?q=url%3Ahttps%3A%2F%2Flolcat.ca
-// https://cc.bingj.com/cache.aspx?q=url%3ahttps%3a%2f%2flolcat.ca&d=4769685974291356&mkt=en-CA&setlang=en-US&w=tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK-
-// <div class="b_attribution" u="0N|5119|4769685974291356|tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK-" tabindex="0">
-
-new bingcache();
-
-class bingcache{
-	
-	public function __construct(){
-		
-		if(
-			!isset($_GET["s"]) ||
-			$this->validate_url($_GET["s"]) === false
-		){
-			
-			var_dump($this->validate_url($_GET["s"]));
-			$this->do404("Please provide a valid URL.");
-		}
-		
-		$url = $_GET["s"];
-		
-		$curlproc = curl_init();
-		
-		curl_setopt(
-			$curlproc,
-			CURLOPT_URL,
-			"https://www.bing.com/search?q=url%3A" .
-			urlencode($url)
-		);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt(
-			$curlproc,
-			CURLOPT_HTTPHEADER,
-			["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0",
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 5);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			$this->do404("Failed to connect to bing servers. Please try again later.");
-		}
-		
-		curl_close($curlproc);
-		
-		preg_match(
-			'/<div class="b_attribution" u="(.*)" tabindex="0">/',
-			$data,
-			$keys
-		);
-		
-		print_r($keys);
-		
-		if(count($keys) === 0){
-			
-			$this->do404("Bing has not archived this URL.");
-		}
-		
-		$keys = explode("|", $keys[1]);
-		$count = count($keys);
-		
-		//header("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]);
-		echo("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]);
-	}
-	
-	public function do404($text){
-				
-		include "lib/frontend.php";
-		$frontend = new frontend();
-		
-		echo
-			$frontend->load(
-				"error.html",
-				[
-					"title" => "Shit",
-					"text" => $text
-				]
-			);
-		
-		die();
-	}
-	
-	public function validate_url($url){
-		
-		$url_parts = parse_url($url);
-		
-		// check if required parts are there
-		if(
-			!isset($url_parts["scheme"]) ||
-			!(
-				$url_parts["scheme"] == "http" ||
-				$url_parts["scheme"] == "https"
-			) ||
-			!isset($url_parts["host"])
-		){
-			return false;
-		}
-		
-		if(
-			// if its not an RFC-valid URL
-			!filter_var($url, FILTER_VALIDATE_URL)
-		){
-			return false;
-		}
-		
-		$ip = 
-			str_replace(
-				["[", "]"], // handle ipv6
-				"",
-				$url_parts["host"]
-			);
-		
-		// if its not an IP
-		if(!filter_var($ip, FILTER_VALIDATE_IP)){
-			
-			// resolve domain's IP
-			$ip = gethostbyname($url_parts["host"] . ".");
-		}
-		
-		// check if its localhost
-		return filter_var(
-			$ip,
-			FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
-		);
-	}
-}

lib/bot_protection.php

@@ -4,10 +4,96 @@ class bot_protection{
 	
 	public function __construct($frontend, $get, $filters, $page, $output){
 		
+		if(config::RATELIMIT_BURST !== 0){
+			
+			// ratelimit requests per IP
+			if(config::RATELIMIT_IP_SOURCE === false){
+				
+				$source = $_SERVER["REMOTE_ADDR"];
+			}else{
+				
+				$source = $_SERVER["HTTP_" . explode(";", strtoupper(str_replace("-", "_", config::RATELIMIT_IP_SOURCE)))[0]];
+			}
+			
+			// check if its not a local IP. Probably local tor bridge, dont ratelimit
+			if(
+				filter_var(
+					$source,
+					FILTER_VALIDATE_IP,
+					FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+				) !== false
+			){
+				
+				$source = "r." . $source;
+				
+				// counter
+				// resets on every req
+				$counter = apcu_fetch($source);
+				$counter = (int)$counter + 1;
+				
+				if(
+					(
+						CONFIG::RATELIMIT_PUNISH !== 0 &&
+						$counter > config::RATELIMIT_BURST + 1
+					) &&
+					(
+						(
+							$output === false &&
+							CONFIG::RATELIMIT_PUNISH_API === true
+						) ||
+						$output === true
+					)
+				){
+					
+					$block_time = config::RATELIMIT_TIME + (($counter - 1) * config::RATELIMIT_PUNISH);
+				}else{
+					
+					$block_time = config::RATELIMIT_TIME;
+				}
+				
+				apcu_store($source, $counter, $block_time);
+				
+				if($counter > config::RATELIMIT_BURST){
+					
+					apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
+					
+					if($output === false){
+						
+						http_response_code(429);
+						echo json_encode([
+							"status" => "You love 4get too much. Try again in " . $this->format_time($block_time) . "."
+						]);
+						die();
+					}
+					
+					$payload = [
+						"timetaken" => microtime(true),
+						"class" => "",
+						"right-left" => "",
+						"right-right" => "",
+						"left" =>
+							'<div class="infobox">' .
+								'<h1>You\'re damaging my calm</h1>' .
+								'You love 4get too much. Try again in <b>' . $this->format_time($block_time) . '</b>.' .
+							'</div>'
+					];
+					
+					$frontend->loadheader(
+						$get,
+						$filters,
+						$page,
+						false
+					);
+					
+					echo $frontend->load("search.html", $payload);
+					die();
+				}
+			}
+		}
+				
 		// check if we want captcha
 		if(config::BOT_PROTECTION !== 1){
 			
-			apcu_inc("real_requests");
 			if($output === true){
 				$frontend->loadheader(
 					$get,
@@ -46,8 +132,6 @@ class bot_protection{
 				}else{
 					
 					// the cookie is OK! dont die() and give results
-					apcu_inc("real_requests");
-					
 					if($output === true){
 						$frontend->loadheader(
 							$get,
@@ -133,6 +217,9 @@ class bot_protection{
 			$answers[] = $regex;
 		}
 		
+		// dedup
+		$answers = array_unique($answers);
+		
 		if(
 			!$invalid &&
 			$key !== false // has captcha been gen'd?
@@ -161,8 +248,7 @@ class bot_protection{
 				$key = "k" . $inc . "." . $this->randomchars();
 				
 				apcu_inc($key, 1, $stupid, 86400);
-				
-				apcu_inc("real_requests");
+				apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
 				
 				setcookie(
 					"pass",
@@ -187,6 +273,7 @@ class bot_protection{
 			}
 		}
 		
+		apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
 		$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
 		
 		$payload = [
@@ -249,13 +336,28 @@ class bot_protection{
 		$frontend->loadheader(
 			$get,
 			$filters,
-			$page
+			$page,
+			false
 		);
 		
 		echo $frontend->load("search.html", $payload);
 		die();
 	}
 	
+	private function format_time($seconds){
+		
+		$hours = floor($seconds / 3600);
+		$minutes = floor(($seconds % 3600) / 60);
+		$secs = $seconds % 60;
+		
+		if($hours > 0){
+			
+			return sprintf('%dh%02dm%02ds', $hours, $minutes, $secs);
+		}
+		
+		return sprintf('%dm%02ds', $minutes, $secs);
+	}
+	
 	private function randomchars(){
 		
 		$chars =

lib/fuckhtml.php

@@ -13,7 +13,7 @@ class fuckhtml{
 		
 		if(is_array($html)){
 			
-			if(!isset($html["innerHTML"])){
+			if(!array_key_exists("innerHTML", $html)){
 				
 				throw new Exception("(load) Supplied array doesn't contain an innerHTML index");
 			}
@@ -240,12 +240,13 @@ class fuckhtml{
 	public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){
 		
 		$elems = $this->getElementsByAttributeName($name, $collection);
+		
 		$value =
 			explode(
 				" ",
 				trim(
 					preg_replace(
-						'/ +/',
+						'/\s+/',
 						" ",
 						$value
 					)
@@ -258,7 +259,18 @@ class fuckhtml{
 			
 			foreach($elem["attributes"] as $attrib_name => $attrib_value){
 				
-				$attrib_value = explode(" ", $attrib_value);
+				$attrib_value =
+					explode(
+						" ",
+						trim(
+							preg_replace(
+								'/\s+/',
+								" ",
+								$attrib_value
+							)
+						)
+					);
+				
 				$ac = count($attrib_value);
 				$nc = count($value);
 				$cr = 0;
@@ -327,7 +339,7 @@ class fuckhtml{
 		
 		if(is_array($html)){
 			
-			if(!isset($html["innerHTML"])){
+			if(!array_key_exists("innerHTML", $html)){
 				
 				throw new Exception("(getTextContent) Supplied array doesn't contain an innerHTML index");
 			}
@@ -381,6 +393,8 @@ class fuckhtml{
 		$json_out = null;
 		$last_char = null;
 		
+		$keyword_check = null;
+		
 		for($i=0; $i<strlen($json); $i++){
 			
 			switch($json[$i]){
@@ -396,6 +410,7 @@ class fuckhtml{
 						
 						$bracket = false;
 						$is_close_bracket = true;
+						
 					}else{
 						
 						if($bracket === false){
@@ -429,6 +444,31 @@ class fuckhtml{
 				$is_close_bracket === false
 			){
 				
+				// do keyword check
+				$keyword_check .= $json[$i];
+				
+				if(in_array($json[$i], [":", "{"])){
+					
+					$keyword_check = substr($keyword_check, 0, -1);
+					
+					if(
+						preg_match(
+							'/function|array|return/i',
+							$keyword_check
+						)
+					){
+						
+						$json_out =
+							preg_replace(
+								'/[{"]*' . preg_quote($keyword_check, "/") . '$/',
+								"",
+								$json_out
+							);
+					}
+					
+					$keyword_check = null;
+				}
+				
 				// here we know we're not iterating over a quoted string
 				switch($json[$i]){
 					
@@ -498,4 +538,91 @@ class fuckhtml{
 				$string
 			);
 	}
+	
+	public function extract_json($json){
+		
+		$len = strlen($json);
+		$array_level = 0;
+		$object_level = 0;
+		$in_quote = null;
+		$start = null;
+		
+		for($i=0; $i<$len; $i++){
+			
+			switch($json[$i]){
+				
+				case "\"":
+				case "'":
+					// count preceding backslashes
+					$bsCount = 0;
+					$j = $i - 1;
+
+					while($j >= 0 && $json[$j] === "\\"){
+						$bsCount++;
+						$j--;
+					}
+
+					// quote is NOT escaped if even number of backslashes
+					if($bsCount % 2 === 0){
+						if($in_quote === null){
+							// open quote
+							$in_quote = $json[$i];
+						}elseif($in_quote === $json[$i]){
+							// close quote
+							$in_quote = null;
+						}
+					}
+					break;
+				
+				case "[":
+					if($in_quote === null){
+						
+						$array_level++;
+						if($start === null){
+							
+							$start = $i;
+						}
+					}
+					break;
+				
+				case "]":
+					if($in_quote === null){
+						
+						$array_level--;
+					}
+					break;
+				
+				case "{":
+					if($in_quote === null){
+						
+						$object_level++;
+						if($start === null){
+							
+							$start = $i;
+						}
+					}
+					break;
+				
+				case "}":
+					if($in_quote === null){
+						
+						$object_level--;
+					}
+					break;
+			}
+			
+			if(
+				$array_level === 0 &&
+				$object_level === 0 &&
+				$start !== null
+			){
+				
+				return substr($json, $start, $i - $start + 1);
+				break;
+			}
+		}
+		
+		// fallback
+		return "[]";
+	}
 }

lib/graph_gen.php

@@ -0,0 +1,388 @@
+<?php
+
+class graph{
+	
+	public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
+		
+		$graph = array_merge([
+			"title" => "Title",
+			"subtitle" => "description",
+			"grad_x" => 25,
+			"grad_y" => 20,
+			"x_label" => "<= x =>",
+			"y_label" => "<= y =>",
+			"antialias" => false,
+			"data" => []
+		], $graph);
+		
+		$size = array_merge([
+			"image_width" => 750,
+			"image_height" => 500,
+			"x" => 90,
+			"y" => 50,
+			"width" => 650,
+			"height" => 300,
+			"square" => 11,
+			"column" => 6
+		], $size);
+		
+		$colors = array_merge([
+			"bg" =>				new ImagickPixel("#1d2021"),
+			"lines_bg" =>		new ImagickPixel("#3c3836"),
+			"outline" =>		new ImagickPixel("#a89984"),
+			"text" =>			new ImagickPixel("#ebdbb2"),
+			"transparent" =>	new ImagickPixel("transparent")
+		], $colors);
+		
+		$colors_line = array_merge([
+			new ImagickPixel("#83a598"),
+			new ImagickPixel("#fb4934"),
+			new ImagickPixel("#b8bb26"),
+			new ImagickPixel("#fabd2f"),
+			new ImagickPixel("#d3869b"),
+			new ImagickPixel("#8ec07c"),
+			new ImagickPixel("#fe8019"),
+			new ImagickPixel("#458588"),
+			new ImagickPixel("#cc241d"),
+			new ImagickPixel("#98971a"),
+			new ImagickPixel("#d79921"),
+			new ImagickPixel("#b16286"),
+			new ImagickPixel("#689d6a"),
+			new ImagickPixel("#d65d0e"),
+			new ImagickPixel("#a89984") // other
+		], $colors_line);
+		
+		$not_enough_data = false;
+		
+		// compute datapoint max ranges
+		$max_range_x = 0;
+		$max_range_y = 0;
+		if(count($graph["data"]) === 0){
+			
+			$not_enough_data = true;
+		}else{
+			
+			foreach($graph["data"] as $datapoint => $value){
+				
+				if(count($value) < 2){
+					
+					$not_enough_data = true;
+				}else{
+					
+					$nv1 = [];
+					$nv2 = [];
+					foreach($value as $k => $v){
+						
+						$nv1[] = $v[0];
+						$nv2[] = $v[1];
+					}
+					
+					$max1 = max($nv1);
+					$max2 = max($nv2);
+					if($max_range_x < $max1){ $max_range_x = $max1; }
+					if($max_range_y < $max2){ $max_range_y = $max2; }
+				}
+			}
+		}
+
+		$im = new Imagick();
+		$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
+		$im->setImageFormat("png");
+
+		$draw = new ImagickDraw();
+
+		//
+		// Draw background grid
+		//
+		$draw->setStrokeColor($colors["lines_bg"]);
+		$draw->setStrokeAntialias($graph["antialias"]);
+		$draw->setStrokeWidth(1);
+		$draw->setFillColor($colors["transparent"]);
+
+		$text = new ImagickDraw();
+		$text->setFontSize(11);
+		$text->setTextAntialias($graph["antialias"]);
+		$text->setFillColor($colors["text"]);
+
+
+		// vertical lines
+		$offset = $size["width"] / ($graph["grad_x"] - 1);
+
+		for($i=0; $i<$graph["grad_x"]; $i++){
+			
+			$x = round($size["x"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_x"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$x,
+				$size["y"],
+				$x,
+				$size["y"] + $size["height"],
+			);
+			
+			if($not_enough_data === false){
+				// add numbas
+				$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
+				$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
+				
+				$im->annotateImage(
+					$text,
+					round($x - ($number_width / 2)),
+					$size["y"] + $size["height"] + 14,
+					0,
+					$number
+				);
+			}
+		}
+
+
+		// horizontal lines
+		$text->setTextAlignment(Imagick::ALIGN_RIGHT);
+		$offset = $size["height"] / ($graph["grad_y"] - 1);
+
+		$vertical_text_size = 0;
+
+		for($i=0; $i<$graph["grad_y"]; $i++){
+			
+			$y = round($size["y"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_y"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$size["x"] + 1,
+				$y,
+				$size["x"] + $size["width"] - 1,
+				$y,
+			);
+			
+			if($not_enough_data === false){
+				$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
+				$query = $im->queryFontMetrics($text, $val);
+				
+				if($query["textWidth"] > $vertical_text_size){
+					
+					$vertical_text_size = $query["textWidth"];
+				}
+				
+				// add numbas
+				$im->annotateImage(
+					$text,
+					$size["x"] - 4,
+					$y + 4,
+					0,
+					$val
+				);
+			}
+		}
+
+
+		//
+		// Draw legend
+		//
+		$text->setTextAlignment(Imagick::ALIGN_LEFT);
+		$top_padding = 44;
+		$c = count($colors_line);
+		$padding = $size["width"] / $size["column"];
+
+		$line = new ImagickDraw();
+		//$line->setStrokeWidth(1);
+		$line->setStrokeAntialias($graph["antialias"]);
+
+		if($not_enough_data === false){
+			$i = 0;
+			foreach($graph["data"] as $datafield => $data){
+				
+				$used_color = $colors_line[$i % $c];
+				$draw->setStrokeColor($used_color);
+				$row = floor($i / $size["column"]);
+				
+				$x = round($size["x"] + (($i % $size["column"]) * $padding));
+				$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
+				
+				// draw legend
+				$draw->rectangle(
+					$x,
+					$y,
+					$x + $size["square"],
+					$y + $size["square"]
+				);
+				
+				$im->annotateImage(
+					$text,
+					$x + $size["square"] + 4,
+					$y + 10,
+					0,
+					$datafield
+				);
+				
+				//
+				// Draw lines
+				//
+				$line->setStrokeColor($used_color);
+				$old_x = null;
+				$old_y = null;
+				$continue = true;
+				
+				foreach($data as $field){
+					
+					// field 0 = hour (sorted)
+					// field 1 = datapoint
+					
+					$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
+					$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
+					
+					if($continue){
+						
+						$continue = false;
+						$old_x = $x;
+						$old_y = $y;
+						continue;
+					}
+					
+					$line->line($old_x, $old_y, $x, $y);
+							
+					$old_x = $x;
+					$old_y = $y;
+				}
+				
+				$i++;
+			}
+		}
+
+		//
+		// add background labels
+		//
+		$text->setFontSize(13);
+		$text->setFillColor($colors["bg"]);
+		$draw->setFillColor($colors["text"]);
+		$draw->setStrokeColor($colors["transparent"]);
+
+		// horizontal
+		$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
+		$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
+		$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
+
+		$draw->rectangle(
+			$x_x - 2,
+			$x_y - $label_width_x["textHeight"] + 4,
+			$x_x + $label_width_x["textWidth"] + 2,
+			$x_y + 3,
+		);
+
+		// vertical
+		$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
+		$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
+		$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
+
+		$draw->rectangle(
+			$y_x + 4,
+			$y_y - 2,
+			$y_x + $label_width_y["textHeight"] + 3,
+			$y_y + $label_width_y["textWidth"] + 2,
+		);
+
+		$im->drawImage($draw);
+		$im->drawImage($line);
+
+		// horizontal text
+		$im->annotateImage(
+			$text,
+			$x_x,
+			$x_y,
+			0,
+			$graph["x_label"]
+		);
+
+		// vertical text
+		$im->annotateImage(
+			$text,
+			$y_x + $label_width_y["textHeight"],
+			$y_y + $label_width_y["textWidth"],
+			270,
+			$graph["y_label"]
+		);
+
+		//
+		// draw title + subtext
+		//
+		$text->setFontSize(20);
+		$text->setFontWeight(900);
+		$text->setFillColor($colors["text"]);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 25,
+			0,
+			$graph["title"]
+		);
+
+		$text->setFontSize(13);
+		$text->setFontWeight(100);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 10,
+			0,
+			$graph["subtitle"]
+		);
+
+		if($not_enough_data){
+			
+			// add not enough data notice
+			$rect = new ImagickDraw();
+			$rect->setFillColor($colors["bg"]);
+			$rect->setStrokeColor($colors_line[1]);
+			
+			$rect_w = 200;
+			$rect_h = 24;
+				
+			$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
+			$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
+			
+			$rect->rectangle(
+				$rect_x,
+				$rect_y,
+				$rect_x + $rect_w,
+				$rect_y + $rect_h
+			);
+			
+			$im->drawImage($rect);
+			
+			$label = "Not enough data";
+			
+			$text->setFontWeight(900);
+			$query = $im->queryFontMetrics($text, $label);
+			
+			$im->annotateImage(
+				$text,
+				round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
+				round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
+				0,
+				$label
+			);
+		}
+
+		return $im->getImageBlob();
+	}
+}

music.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->music($get);
 	

news.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->news($get);
 	

oracles/base.php

@@ -1,36 +0,0 @@
-<?php
-abstract class oracle {
-	// some info to spit out alongside the result, so the user knows
-	//  what exactly is giving out the answer. prevents confusion
-	//   about what oracle is answering them for ambiguous queries.
-	public $info = [
-		"name" => "some oracle"
-	];
-	// this function should take in a query string search from $_GET,
-	//  and return a bool determining whether or not it is a question
-	//   intended for the oracle.
-	public function check_query($q) {
-		return false;
-	}
-	// produce the correct answer for the query using the oracle.
-	//  note: if it becomes apparent /during generation/ that the
-	//   query is not in fact for the oracle, returning an empty
-	//    string will kill the oracle pane.
-	// answer format: ["ans1 title" => "ans1", ...]
-	public function generate_response($q) {
-		return "";
-	}
-}
-// backwards compatibility
-if (!function_exists('str_starts_with')) {
-	function str_starts_with($haystack, $needle) {
-		return strncmp($haystack, $needle, strlen($needle)) === 0;;
-	}
-}
-if (!function_exists('str_contains')) {
-	function str_contains($haystack, $needle) {
-		return strpos((string)$haystack, (string)$needle) !== false;
-	}
-}
-
-?>

oracles/calc.php

@@ -1,165 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class calculator extends oracle {
-	public $info = [
-		"name" => "calculator"
-	];
-	public function check_query($q) {
-		// straight numerics should go to that oracle
-		if (is_numeric($q)) {
-			return false;
-		}
-		// all chars should be number-y or operator-y
-		$char_whitelist = str_split("1234567890.+-/*^%() ");
-		foreach (str_split($q) as $char) {
-			if (!in_array($char, $char_whitelist)) {
-				return false;
-			}
-		}
-		return true;
-	}
-	// a custom parser and calculator because FUCK YUO, libraries are
-	//  gay.
-	public function generate_response($q)
-	{
-		$nums = str_split("1234567890.");
-		$ops = str_split("+-/*^%;");
-		$grouping = str_split("()");
-
-		$q = str_replace(" ", "", $q);
-
-		// backstop for the parser so it catches the last
-		//  numeric token
-		$q .= ";"; 
-
-		// the following comments refer to this example input:
-		//  21+9*(3+2^9)+1
-
-		// 2-length lists of the following patterns:
-		//  ["n" (umeric), <some number>]
-		//  ["o" (perator), "<some operator>"]
-		//  ["g" (roup explicit), <"(" or ")">]
-		// e.g. [["n", 21], ["o", "+"], ["n", 9], ["o", *],
-		//       ["g", "("], ["n", 3], ["o", "+"], ["n", 2],
-		//       ["o", "^"], ["n", 9], ["g", ")"], ["o", "+"],
-		//       ["n", "1"]]
-		$tokens = array();
-		$dragline = 0;
-		foreach(str_split($q) as $i=>$char) {
-			if (in_array($char, $nums)) {
-				continue;
-			}
-			elseif (in_array($char, $ops) || in_array($char, $grouping)) {
-				// hitting a non-numeric implies everything since the
-				//  last hit has been part of a number
-				$capture = substr($q, $dragline, $i - $dragline);
-				// prevent the int cast from creating imaginary
-				//  ["n", 0] tokens
-				if ($capture != "") {
-					if (substr_count($capture, ".") > 1) {
-						return "";
-					}
-					array_push($tokens, ["n", (float)$capture]);
-				}
-				// reset to one past the current (non-numeric) char
-				$dragline = $i + 1; 
-				// the `;' backstop is not a real token and this should
-				//  never be present in the token list
-				if ($char != ";") {
-					array_push($tokens, [
-						($char == "(" || $char == ")") ? "g" : "o",
-						$char
-					]);
-				}
-			}
-			else {
-				return "";
-			}
-		}
-
-		// two operators back to back should fail
-		for ($i = 1; $i < count($tokens); $i++) {
-			if ($tokens[$i][0] == "o" && $tokens[$i-1][0] == "o") {
-				return "";
-			}
-		}
-
-		// no implicit multiplication
-		for ($i = 0; $i < count($tokens) - 1; $i++) {
-			if ($tokens[$i][0] == "n" && $tokens[$i+1] == ["g", "("]) {
-				return "";
-			}
-		}
-
-		//strategy:
-		// traverse to group open (if there is one)
-		//  - return to start with the internals
-		// traverse to ^, attack token previous and after
-		// same but for *, then / then + then -
-		// poppers all teh way down
-		try {
-			return [
-				substr($q, 0, strlen($q)-1)." = " => $this->executeBlock($tokens)[0][1]
-			];
-		}   
-		catch (\Throwable $e) {
-			if (get_class($e) == "DivisionByZeroError") {
-				return [
-					$q." = " => "Division by Zero Error!!"
-				];
-			}
-			return "";
-		}
-	}
-	public function executeBlock($tokens) {
-		if (count($tokens) >= 2 && $tokens[0][0] == "o" && $tokens[0][1] == "-" && $tokens[1][0] == "n") {
-			array_splice($tokens, 0, 2, [["n", -1 * (float)$tokens[1][1]]]);
-		}
-		if (count($tokens) > 0 && $tokens[0][0] == "o" || $tokens[count($tokens)-1][0] == "o") {
-			throw new Exception("Error Processing Request", 1);
-		}
-		while (in_array(["g", "("], $tokens)) {
-			$first_open = array_search(["g", "("], $tokens);
-			$enclosedality = 1;
-			for ($i = $first_open+1; $i < count($tokens); $i++) {
-				if ($tokens[$i][0] == "g") {
-					$enclosedality += ($tokens[$i][1] == "(") ? 1 : -1;
-				}
-				if ($enclosedality == 0) {
-					array_splice($tokens, 
-						$first_open, 
-						$i+1 - $first_open, 
-						$this->executeBlock(
-							array_slice($tokens, $first_open+1, $i-1 - $first_open)
-						)
-					);
-					break;
-				}
-			}
-		}
-		$operators_in_pemdas_order = [
-			"^" => (fn($x, $y) => $x ** $y),
-			"*" => (fn($x, $y) => $x * $y),
-			"/" => (fn($x, $y) => $x / $y), 
-			"%" => (fn($x, $y) => $x % $y),
-			"+" => (fn($x, $y) => $x + $y), 
-			"-" => (fn($x, $y) => $x - $y)
-		];
-		foreach ($operators_in_pemdas_order as $op=>$func) {
-			while (in_array(["o", $op], $tokens)) {
-				for ($i = 0; $i < count($tokens); $i++) {
-					if ($tokens[$i] == ["o", $op]) {
-						array_splice(
-							$tokens,
-							$i-1,
-							3,
-							[["n", (string)($func((float)$tokens[$i-1][1], (float)$tokens[$i+1][1]))]]
-						);
-					}
-				}
-			}
-		}
-		return $tokens;
-	}
-}
-?>

oracles/encoder.php

@@ -1,40 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class encoder extends oracle {
-	public $info = [
-		"name" => "text encoder/hasher"
-	];
-	private $special_types = [
-		"rot13",
-		"base64"
-	];
-	public function check_query($q) {
-		$types = array_merge($this->special_types, hash_algos());
-		foreach ($types as $type) {
-			$type .= " ";
-			if (str_starts_with($q, $type)) {
-				return true;
-			}
-		}
-		return false;
-	}
-	public function generate_response($q)
-	{
-		$type = explode(" ", $q)[0];
-		$victim = substr($q, strlen($type)+1);
-		if (in_array($type, hash_algos())) {
-			return [$type." hash" => hash($type, $victim)];
-		}
-		switch ($type) {
-			case "rot13":
-				return ["rot13 encoded" => str_rot13($victim)];
-			case "base64":
-				return [
-					"base64 encoded" => base64_encode($victim),
-					"base64 decoded" => base64_decode($victim)
-				];
-		}
-		return "";
-	}
-}
-?>

oracles/numerics.php

@@ -1,54 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class numerics extends oracle {
-	public $info = [
-		"name" => "numeric base conversion"
-	];
-	public function check_query($q) {
-		if (str_contains($q, " ")) {
-			return false;
-		}
-
-		$q = strtolower($q);
-
-		$profiles = [
-			["0x", str_split("0123456789abcdef")],
-			["", str_split("1234567890")],
-			["b", str_split("10")]
-		];
-
-		foreach ($profiles as $profile) {
-			$good = true;
-			$good &= str_starts_with($q, $profile[0]);
-			$nq = substr($q, strlen($profile[0]));
-			foreach (str_split($nq) as $c) {
-				$good &= in_array($c, $profile[1]);
-			}
-			if ($good) {
-				return true;
-			}
-		}
-		return false;
-	}
-	public function generate_response($q) {
-		$n = 0;
-		if (str_starts_with($q, "0x")) {
-			$nq = substr($q, strlen("0x"));
-			$n = hexdec($nq);
-		}
-		elseif (str_starts_with($q, "b")) {
-			$nq = substr($q, strlen("b"));
-			$n = bindec($nq);
-		}
-		else {
-			$n = (int)$q;
-		}
-		return [
-			"decimal (base 10)" => (string)$n,
-			"hexadecimal (base 16)" => "0x".(string)dechex($n),
-			"binary (base 2)" => "b".(string)decbin($n),
-			"" => "binary inputs should be prefixed with 'b', hex with '0x'."
-		];
-	}
-}
-?>

oracles/time.php

@@ -1,45 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class time extends oracle {
-	public $info = [
-		"name" => "what time is it?"
-	];
-	public function check_query($q) {
-		$prompts = [
-			"what", "time", "is", "it",
-			"right", "now", "the", "current",
-			"get", "date"
-		];
-		$q = str_replace(",", "", $q);
-		$q = str_replace("?", "", $q);
-		$q = str_replace("what's", "what is", $q);
-		$oq = $q;
-		$q = explode(" ", $q);
-		$count = 0;
-		foreach ($q as $word) {
-			if (in_array($word, $prompts)) {
-				$count++;
-			}
-		}
-		// remove one from total count if a timezone is specified
-		return ($count/(count($q) + (str_contains($oq, "tz:") ? -1 : 0))) > 3/4;
-	}
-	public function generate_response($q) {
-		$timezone = timezone_name_from_abbr("UTC");
-		foreach (explode(" ", $q) as $word) {
-			if (str_starts_with($word, "tz:")) {
-				$decltz = timezone_name_from_abbr(substr($word, 3, 3));
-				if ($decltz) {
-					$timezone = $decltz;
-				}
-			}
-		}
-		date_default_timezone_set($timezone);
-		return [
-			"The time in ".$timezone => date("H:i:s"),
-			" " => date("l, F jS"),
-			"" => "include the string \"tz:XXX\" to use timezone XXX"
-		];
-	}
-}
-?>

proxy.php

@@ -34,22 +34,46 @@ try{
 		)
 	){
 		
-		if(
-			!isset($image["query"]) ||
-			!isset($image["path"]) ||
-			$image["path"] != "/th"
-		){
+		if(!isset($image["path"])){
 			
-			header("X-Error: Invalid bing image path");
+			header("X-Error: Missing bing image path");
 			$proxy->do404();
 			die();
 		}
 		
-		parse_str($image["query"], $str);
+		//
+		// get image ID
+		// formations:
+		// https://tse2.mm.bing.net/th/id/OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
+		// https://tse2.mm.bing.net/th?id=OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
+		//
+		$id = null;
+		if(isset($image["query"])){
 			
-		if(!isset($str["id"])){
+			parse_str($image["query"], $str);
 			
-			header("X-Error: Missing bing ID");
+			if(isset($str["id"])){
+				
+				$id = $str["id"];
+			}
+		}
+		
+		if($id === null){
+			
+			$id = explode("/th/id/", $image["path"], 2);
+			
+			if(count($id) !== 2){
+				
+				// malformed
+				return $url;
+			}
+			
+			$id = $id[1];
+		}
+		
+		if(is_array($id)){
+			
+			header("X-Error: Missing bing id parameter");
 			$proxy->do404();
 			die();
 		}
@@ -63,7 +87,7 @@ try{
 			case "cover": $req = "&w=207&h=270&p=0&qlt=90"; break;
 		}
 		
-		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . urlencode($str["id"]) . $req, "https://www.bing.com");
+		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . rawurlencode($id) . $req, "https://www.bing.com");
 		die();
 	}
 	

resolve/google.php

@@ -0,0 +1,107 @@
+<?php
+
+class google{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("google");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	private function get(/*$proxy, */$url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$headers =
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"];
+		
+		// i dont think proxies are necessary here...
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	function resolve($id){
+		
+		$html =
+			$this->get(
+				//$this->backend->get_ip(),
+				"https://www.google.com/goto",
+				[
+					"url" => $id
+				]
+			);
+		
+		$this->fuckhtml->load($html);
+		
+		$a =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"a"
+			);
+		
+		if(count($a) === 0){
+			
+			throw new Exception("Failed to find link");
+		}
+		
+		$link =
+			$this->fuckhtml
+			->getTextContent(
+				$a[0]["attributes"]["href"]
+			);
+		
+		if(
+			preg_match(
+				'/^(https?:|)\/\//',
+				$link
+			)
+		){
+			
+			return $link;
+		}
+		
+		throw new Exception("Got malformed link: " . $link);
+	}
+}

resolve/sc.php

@@ -0,0 +1,117 @@
+<?php
+
+class sc{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("sc");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$headers =
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"];
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	function resolve($id){
+		
+		if(
+			!preg_match(
+				'/^(t|p)[0-9]+$/',
+				$id
+			)
+		){
+			
+			throw new Exception("ID is invalid");
+		}
+		
+		$type = $id[0] == "t" ? "track" : "playlist";
+		$id = substr($id, 1);
+		
+		try{
+			$html =
+				$this->get(
+					$this->backend->get_ip(),
+					"https://w.soundcloud.com/player/",
+					[
+						"url" => "http://api.soundcloud.com/{$type}s/{$id}",
+						"show_artwork" => "true"
+					]
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch song embed page");
+		}
+		
+		$this->fuckhtml->load($html);
+		
+		$link =
+			$this->fuckhtml
+			->getElementsByAttributeValue(
+				"rel",
+				"canonical",
+				"link"
+			);
+		
+		if(count($link) === 0){
+			
+			throw new Exception("Soundcloud could not resolve the song ID to an URL");
+		}
+		
+		return
+			$this->fuckhtml
+			->getTextContent(
+				$link[0]
+				["attributes"]
+				["href"]
+			);
+	}
+}

resolver.php

@@ -0,0 +1,71 @@
+<?php
+
+$resolver = new resolver();
+
+class resolver{
+	
+	public const resolvers = [
+		"sc",
+		"google"
+	];
+	
+	public function __construct(){
+		
+		include "data/config.php";
+		
+		if(
+			!isset($_GET["scraper"]) ||
+			!is_string($_GET["scraper"]) ||
+			!in_array($_GET["scraper"], self::resolvers)
+		){
+			
+			$this->do400("Missing or invalid scraper");
+			return;
+		}
+		
+		if(
+			!isset($_GET["target"]) ||
+			!is_string($_GET["target"])
+		){
+			
+			$this->do400("Missing or invalid target");
+			return;
+		}
+		
+		$scraper = $_GET["scraper"];
+		$target = $_GET["target"];
+		
+		try{
+			
+			include "resolve/{$scraper}.php";
+			$resolver = new $scraper();
+			$link = $resolver->resolve($target);
+			
+			if(
+				is_string($link) &&
+				!empty($link)
+			){
+				
+				header("Location: {$link}");
+				echo 'If your ass is not getting redirected, <a href="' . htmlspecialchars($link) . '">click here</a>.';
+			}
+		}catch(Exception $error){
+			
+			$this->do404("Fuck! Failed to resolve URL: " . $error->getMessage());
+		}
+	}
+	
+	public function do400($message){
+		
+		header("Content-Type: text/plain");
+		http_response_code(400);
+		echo htmlspecialchars($message);
+	}
+	
+	public function do404($message){
+		
+		header("Content-Type: text/plain");
+		http_response_code(404);
+		echo htmlspecialchars($message);
+	}
+}

robots.txt

@@ -1,28 +1,23 @@
-# When the robots.txt is sus
-
-# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌
-# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌
-# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌
-# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪
-# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡
-# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐
-# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔
-# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘
-# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎
-# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗
-# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷
-# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟
-# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰
-# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊
-# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌
-# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁
-# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀
-# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀
-# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟
-# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾
-# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽
+#   44
+# 4444444      44
+# 44444444   44444       444
+# 44444444  444444    444444444
+#  44444   44444444  444444444
+#         444444444   4444444
+#        4444444444    444444
+#      4444444444444
+#    444444444444444444
+#       444444444444444
+#          44444444
+#          4444
+#           44
 
 User-agent: *
-Disallow:
-Host: 4get.ca
-Sitemap: https://4get.ca/sitemap
+Disallow: /web
+Disallow: /images
+Disallow: /videos
+Disallow: /news
+Disallow: /music
+Disallow: /proxy
+Disallow: /api/v1/ac
+Allow: /

scraper/coccoc.php

@@ -0,0 +1,923 @@
+<?php
+
+class coccoc{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("coccoc");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+		
+		$this->proc = curl_multi_init();
+		$this->handles = [];
+	}
+	
+	
+	private function get($proxy, $url, $cookie = false, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		if($cookie === false){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1",
+				"Priority: u=0, i"
+			]);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Referer: https://coccoc.com/search",
+				"Cookie: {$cookie}",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: same-origin",
+				"Priority: u=0, i",
+				"TE: trailers"
+			]);
+		}
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function getfilters($pagetype){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes", // nsfw by default????
+					"no" => "No" // &safe=1
+				]
+			],
+			"time" => [
+				"display" => "Time posted",
+				"option" => [
+					"any" => "Any time",
+					"1w" => "1 week ago",
+					"2w" => "2 weeks ago",
+					"1m" => "1 month ago",
+					"3m" => "3 months ago",
+					"6m" => "6 months ago",
+					"1Y" => "1 year ago"
+				]
+			],
+			"filter" => [
+				"display" => "Remove duplicates",
+				"option" => [
+					"no" => "No",
+					"yes" => "Yes" // &filter=0
+				]
+			]
+		];
+	}
+	
+	private function handle_redir($proxy, $html){
+		
+		$this->fuckhtml->load($html);
+		
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName("title");
+		
+		if(
+			count($title) !== 0 &&
+			$this->fuckhtml
+			->getTextContent(
+				$title[0]
+			) == "Redirecting..."
+		){
+			
+			// get cookie
+			if(
+				preg_match(
+					'/document\.cookie ?= ?"([^;"]+).*"/',
+					$html,
+					$cookie
+				) === 0
+			){
+				
+				throw new Exception("Failed to grep cookie");
+			}
+			
+			// get redirect URL
+			if(
+				preg_match(
+					'/window\.location\.replace\("([^"]+)"\)/',
+					$html,
+					$redir
+				) === 0
+			){
+				
+				throw new Exception("Failed to grep redirection URL");
+			}
+			
+			try{
+				
+				$html =
+					$this->get(
+						$proxy,
+						$redir[1],
+						$cookie[1],
+						[]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get redirection page");
+			}
+		}
+		
+		return $html;
+	}
+	
+	private function redirect_add_url($url){
+		
+		// https://coccoc.com/composer/goto?url=0AWtXcuK-mJVvFl8Qhny9LZdMOyEd08nFmTtpg82bmuqtp8p8G_OnTebNhPtNyIXMGQ1TZZ-ADMz0QEiFJhsTLIFkkPDXnJjHo3kqsobmyntiOhFag-2aqTs
+		
+		if(
+			preg_match(
+				'/\/goto\?url=([^&]+)/',
+				$url,
+				$slug
+			) === 0
+		){
+			
+			// not a coccoc redirect
+			return $url;
+		}
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, "https://coccoc.com/composer/goto?url=" . urlencode($slug[1]));
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// funny
+		curl_setopt($curlproc, CURLOPT_HEADER, true);
+		
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_multi_add_handle($this->proc, $curlproc);
+		$this->handles[] = [
+			"replacement" => $url,
+			"handle" => $curlproc
+		];
+		
+		return $url;
+	}
+	
+	private function resolve_urls(&$scraped_data){
+		
+		do{
+			do{
+				$status = curl_multi_exec($this->proc, $active);
+			}while($status === CURLM_CALL_MULTI_PERFORM);
+
+			if($active){
+				curl_multi_select($this->proc);
+			}
+
+		}while($active && $status === CURLM_OK);
+		
+		//
+		// if we reach this, we're done downloading garbage
+		//
+		
+		// parse redirect values & create data shit
+		$keys = [];
+		$replacements = [];
+		
+		for($i=0; $i<count($this->handles); $i++){
+			
+			$headers =
+				curl_multi_getcontent(
+					$this->handles[$i]["handle"]
+				);
+			
+			if(
+				!preg_match(
+					'/^location: ?([^ ]+)/im',
+					$headers,
+					$link
+				)
+			){
+				
+				// failed to extract redirect location
+				continue;
+			}
+			
+			// theyre too retarded to encode shit properly apparently.
+			// this is the best i can do, it will still yield errors when searching for "higurashi"
+			// example: https://coccoc.com/composer/goto?url=0AWtXcuIKWPOkxvarGvy3GJ36XipRirzzqAvltMM5QUfB1RPu49LrNB9qmuAQyvvI0eCTwf-FpjRjxUN8XbruhA0rvCHNNlhVNEozMUqgRcjzsdhyUMCEgwLCXVTQkXnLXqrUw8celcIZdfmZ1geOvioUcg
+			// redirects to https://vi.wikipedia.org/wiki/Higurashi_no_Naku_Koro_ni#C%E1ch_ch?i
+			// notice the encoding errors after #
+			// retards
+			
+			$link = mb_convert_encoding($link[1], "UTF-8", "ISO-8859-1");
+			
+			if(
+				preg_match(
+					'/^(https?:|)\/\//',
+					$link
+				)
+			){
+				
+				$keys[] = $this->handles[$i]["replacement"];
+				$replacements[] = $link;
+			}
+		}
+		
+		$this->recursive_replace($keys, $replacements, $scraped_data);
+		
+		curl_multi_close($this->proc);
+	}
+	
+	private function recursive_replace(&$keys, &$replacements, &$pointer){
+		
+		foreach($pointer as $key => &$value){
+			
+			if($key == "url"){
+				
+				$search =
+					array_search(
+						$value,
+						$keys
+					);
+				
+				if($search !== false){
+					
+					// replace url
+					$value = $replacements[$search];
+				}
+				
+			}elseif(is_array($value)){
+				
+				$this->recursive_replace($keys, $replacements, $value);
+			}
+		}
+	}
+	
+	public function web($get){
+		
+		if($get["npt"]){
+			
+			[$query, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"web"
+				);
+			
+			$query = json_decode($query, true);
+		}else{
+			
+			$proxy = $this->backend->get_ip();
+			
+			$query = [
+				"query" => $get["s"]
+			];
+			
+			// add filters
+			if($get["nsfw"] == "no"){
+				
+				$query["safe"] = 1;
+			}
+			
+			if($get["time"] != "any"){
+				
+				$query["tbs"] = $get["time"];
+			}
+			
+			if($get["filter"] == "yes"){
+				
+				$query["filter"] = 0;
+			}
+		}
+		
+		try{
+			
+			$html =
+				$this->get(
+					$proxy,
+					"https://coccoc.com/search",
+					false,
+					$query
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to get search page");
+		}
+		//$html = file_get_contents("scraper/coccoc.html");
+		
+		$html = $this->handle_redir($proxy, $html);
+		
+		$html = explode("window.composerResponse", $html, 2);
+		
+		if(count($html) !== 2){
+			
+			throw new Exception("Failed to grep window.composerResponse");
+		}
+		
+		$html =
+			json_decode(
+				$this->fuckhtml
+				->extract_json(
+					ltrim($html[1], " =")
+				),
+				true
+			);
+		
+		if($html === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(
+			isset($html["captcha"]) &&
+			(int)$html["captcha"] === 1
+		){
+			
+			throw new Exception("Coc Coc returned a Captcha");
+		}
+		
+		if(!isset($html["search"]["search_results"])){
+			
+			throw new Exception("Coc Coc did not return a search_results object");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		// word correction
+		foreach($html["top"] as $element){
+			
+			if(isset($element["spellChecker"][0]["query"])){
+				
+				$out["spelling"] = [
+					"type" => "not_many",
+					"using" => $html["search"]["query"],
+					"correction" => $element["spellChecker"][0]["query"]
+				];
+			}
+		}
+		
+		foreach($html["search"]["search_results"] as $result){
+			
+			if(isset($result["type"])){
+				
+				switch($result["type"]){
+					
+					//
+					// Related searches
+					//
+					case "related_queries":
+						$out["related"] = $result["queries"];
+						continue 2;
+					
+					//
+					// Videos
+					//
+					case "video_hits":
+						foreach($result["results"] as $video){
+							
+							if(
+								isset($video["image_url"]) &&
+								!empty($video["image_url"])
+							){
+								
+								$thumb = [
+									"ratio" => "16:9",
+									"url" => $video["image_url"]
+								];
+							}else{
+								
+								$thumb = [
+									"ratio" => null,
+									"url" => null
+								];
+							}
+							
+							$out["video"][] = [
+								"title" =>
+									$this->titledots(
+										$this->fuckhtml
+										->getTextContent(
+											$video["title"]
+										)
+									),
+								"description" => null,
+								"author" => [
+									"name" => $video["uploader"],
+									"url" => null,
+									"avatar" => null
+								],
+								"date" => (int)$video["date"],
+								"duration" => (int)$video["duration"],
+								"views" => null,
+								"thumb" => $thumb,
+								"url" => $video["url"]
+							];
+						}
+						continue 2;
+				}
+			}
+			
+			if(
+				!isset($result["title"]) ||
+				!isset($result["url"])
+			){
+				
+				// should not happen
+				continue;
+			}
+			
+			if(isset($result["rich"]["data"]["image_url"])){
+				
+				$thumb = [
+					"url" => $result["rich"]["data"]["image_url"],
+					"ratio" => "16:9"
+				];
+			}else{
+				
+				$thumb = [
+					"url" => null,
+					"ratio" => null
+				];
+			}
+			
+			$sublinks = [];
+			
+			if(isset($result["rich"]["data"]["linked_docs"])){
+				
+				foreach($result["rich"]["data"]["linked_docs"] as $sub){
+					
+					$sublinks[] = [
+						"title" =>
+							$this->titledots(
+								$this->fuckhtml
+								->getTextContent(
+									$sub["title"]
+								)
+							),
+						"description" =>
+							$this->titledots(
+								$this->fuckhtml
+								->getTextContent(
+									$sub["content"]
+								)
+							),
+						"date" => null,
+						"url" => $this->redirect_add_url($sub["url"])
+					];
+				}
+			}
+			
+			// get date
+			if(isset($result["date"])){
+				
+				$date = (int)$result["date"];
+			}else{
+				
+				$date = null;
+			}
+			
+			// probe for metadata
+			$table = [];
+			
+			if(isset($result["rich"]["data"]["rating"])){
+				
+				$table["Rating"] = $result["rich"]["data"]["rating"];
+				
+				if(isset($result["rich"]["data"]["num_rating"])){
+					
+					$table["Rating"] .= " (" . number_format($result["rich"]["data"]["num_rating"]) . " ratings)";
+				}
+			}
+			
+			if(isset($result["rich"]["data"]["views"])){
+				
+				$table["Views"] = number_format($result["rich"]["data"]["views"]);
+			}
+			
+			if(isset($result["rich"]["data"]["duration"])){
+				
+				$table["Duration"] = $this->int2hms($result["rich"]["data"]["duration"]);
+			}
+			
+			if(isset($result["rich"]["data"]["channel_name"])){
+				
+				$table["Author"] = $result["rich"]["data"]["channel_name"];
+			}
+			
+			if(isset($result["rich"]["data"]["video_quality"])){
+				
+				$table["Quality"] = $result["rich"]["data"]["video_quality"];
+			}
+			
+			if(isset($result["rich"]["data"]["category"])){
+				
+				$table["Category"] = $result["rich"]["data"]["category"];
+			}
+			
+			$out["web"][] = [
+				"title" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$result["title"]
+						)
+					),
+				"description" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$result["content"]
+						)
+					),
+				"url" => $this->redirect_add_url($result["url"]),
+				"date" => $date,
+				"type" => "web",
+				"thumb" => $thumb,
+				"sublink" => $sublinks,
+				"table" => $table
+			];
+		}
+		
+		//
+		// Get wikipedia head
+		//
+		if(isset($html["right"])){
+			
+			foreach($html["right"] as $wiki){
+				
+				$description = [];
+				
+				if(isset($wiki["short_intro"])){
+					
+					$description[] =
+						[
+							"type" => "quote",
+							"value" => $wiki["short_intro"],
+						];
+				}
+				
+				if(isset($wiki["intro"])){
+					
+					$description[] =
+						[
+							"type" => "text",
+							"value" => $wiki["intro"],
+						];
+				}
+				
+				// get table elements
+				$table = [];
+				
+				if(isset($wiki["fields"])){
+					
+					foreach($wiki["fields"] as $element){
+						
+						$table[$element["title"]] = implode(", ", $element["value"]);
+					}
+				}
+				
+				// get sublinks
+				$sublinks = [];
+				
+				if(isset($wiki["website"])){
+					
+					if(
+						preg_match(
+							'/^http/',
+							$wiki["website"]
+						) === 0
+					){
+						
+						$sublinks["Website"] = "https://" . $wiki["website"];
+					}else{
+						
+						$sublinks["Website"] = $wiki["website"];
+					}
+				}
+				
+				if(isset($wiki["profiles"])){
+					
+					foreach($wiki["profiles"] as $sitename => $url){
+						
+						$sitename = explode("_", $sitename);
+						$sitename = ucfirst($sitename[count($sitename) - 1]);
+						
+						$sublinks[$sitename] = $url;
+					}
+				}
+				
+				$out["answer"][] = [
+					"title" =>
+						isset($wiki["title"]) ?
+							$this->titledots(
+								$wiki["title"]
+							) : null,
+					"description" => $description,
+					"url" => null,
+					"thumb" => isset($wiki["image"]["contentUrl"]) ? $wiki["image"]["contentUrl"] : null,
+					"table" => $table,
+					"sublink" => $sublinks
+				];
+			}
+		}
+		
+		// get next page
+		if((int)$html["search"]["page"] < (int)$html["search"]["max_page"]){
+			
+			// https://coccoc.com/composer?_=1754021153532&p=0&q=zbabduiqwhduwqhdnwq&reqid=bwcAs00q&s=direct&apiV=1
+			// ^json endpoint, but we can just do &page=2 lol
+			
+			if(!isset($query["page"])){
+				
+				$query["page"] = 2;
+			}else{
+				
+				$query["page"]++;
+			}
+			
+			$out["npt"] =
+				$this->backend
+				->store(
+					json_encode($query),
+					"web",
+					$proxy
+				);
+		}
+		
+		$this->resolve_urls($out);
+		return $out;
+	}
+	
+	public function video($get){
+		
+		//$html = file_get_contents("scraper/coccoc.html");
+		if($get["npt"]){
+			
+			[$query, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"videos"
+				);
+			
+			$query = json_decode($query, true);
+		}else{
+			
+			$proxy = $this->backend->get_ip();
+			
+			$query = [
+				"query" => $get["s"],
+				"tbm" => "vid"
+			];
+			
+			// add filters
+			if($get["nsfw"] == "no"){
+				
+				$query["safe"] = 1;
+			}
+			
+			if($get["time"] != "any"){
+				
+				$query["tbs"] = $get["time"];
+			}
+			
+			if($get["filter"] == "yes"){
+				
+				$query["filter"] = 0;
+			}
+		}
+		
+		try{
+			
+			$html =
+				$this->get(
+					$proxy,
+					"https://coccoc.com/search",
+					false,
+					$query
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to get search page");
+		}
+		
+		$html = $this->handle_redir($proxy, $html);
+		
+		$html = explode("window.composerResponse", $html, 2);
+		
+		if(count($html) !== 2){
+			
+			throw new Exception("Failed to grep window.composerResponse");
+		}
+		
+		$html =
+			json_decode(
+				$this->fuckhtml
+				->extract_json(
+					ltrim($html[1], " =")
+				),
+				true
+			);
+		
+		if($html === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		if(!isset($html["search_video"]["search_results"])){
+			
+			if(isset($html["search_video"]["error"]["title"])){
+				
+				if($html["search_video"]["error"]["title"] == "Không tìm thấy kết quả nào"){
+					
+					return $out;
+				}
+				
+				throw new Exception("Coc Coc returned an error: " . $html["search_video"]["error"]["title"]);
+			}
+			
+			throw new Exception("Coc Coc did not supply a search_results object");
+		}
+		
+		foreach($html["search_video"]["search_results"] as $video){
+			
+			if(isset($video["rich"]["data"]["image_url"])){
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $video["rich"]["data"]["image_url"]
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
+			}
+			
+			$out["video"][] = [
+				"title" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$video["title"]
+						)
+					),
+				"description" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$video["content"]
+						)
+					),
+				"author" => [
+					"name" => 
+						isset($video["rich"]["data"]["channel_name"]) ?
+							$video["rich"]["data"]["channel_name"] : null,
+					"url" => null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($video["date"]) ?
+						$video["date"] : null,
+				"duration" =>
+					isset($video["rich"]["data"]["duration"]) ?
+						(int)$video["rich"]["data"]["duration"] : null,
+				"views" => null,
+				"thumb" => $thumb,
+				"url" => $this->redirect_add_url($video["url"])
+			];
+		}
+		
+		// get next page
+		if((int)$html["search_video"]["page"] < (int)$html["search_video"]["max_page"]){
+			
+			if(!isset($query["page"])){
+				
+				$query["page"] = 2;
+			}else{
+				
+				$query["page"]++;
+			}
+			
+			$out["npt"] =
+				$this->backend
+				->store(
+					json_encode($query),
+					"videos",
+					$proxy
+				);
+		}
+		
+		$this->resolve_urls($out);
+		return $out;
+	}
+	
+	private function titledots($title){
+		
+		return trim($title, " .\t\n\r\0\x0B…");
+	}
+	
+	private function int2hms($seconds){
+		
+		$hours = floor($seconds / 3600);
+		$minutes = floor(($seconds % 3600) / 60);
+		$seconds = $seconds % 60;
+		
+		return sprintf("%02d:%02d:%02d", $hours, $minutes, $seconds);
+	}
+}

scraper/crowdview.php

@@ -1,145 +0,0 @@
-<?php
-
-class crowdview{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("crowdview");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$proxy = $this->backend->get_ip();
-		
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://crowdview-next-js.onrender.com/api/search-v3",
-				[
-					"query" => $search
-				]
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$json = json_decode($json, true);
-		
-		if($json === NULL){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		foreach($json["results"] as $item){
-			
-			$description = explode("<b>", $item["snippet"], 2);
-			
-			$out["web"][] = [
-				"title" => $this->sanitize($item["title"]),
-				"description" => $this->sanitize($description[1]),
-				"url" => $item["link"],
-				"date" => strtotime($description[0]),
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function sanitize($html){
-		
-		return
-			trim(
-				$this->fuckhtml
-				->getTextContent(
-					html_entity_decode(
-						$html
-					)
-				),
-				". "
-			);
-	}
-}

scraper/curlie.php

@@ -1,309 +0,0 @@
-<?php
-
-class curlie{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("curlie");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"lang" => [
-				"display" => "Language",
-				"option" => [
-					"any" => "Any language",
-					"en" => "English",
-					"de" => "German",
-					"fr" => "French",
-					"ja" => "Japanese",
-					"it" => "Italian",
-					"es" => "Spanish",
-					"ru" => "Russian",
-					"nl" => "Dutch",
-					"pl" => "Polish",
-					"tr" => "Turkish",
-					"da" => "Danish",
-					"sv" => "Swedish",
-					"no" => "Norwegian",
-					"is" => "Icelandic",
-					"fo" => "Faroese",
-					"fi" => "Finnish",
-					"et" => "Estonian",
-					"lt" => "Lithuanian",
-					"lv" => "Latvian",
-					"cy" => "Welsh",
-					"ga" => "Irish",
-					"gd" => "Scottish Gaelic",
-					"br" => "Breton",
-					"fy" => "Frisian",
-					"frr" => "North Frisian",
-					"gem" => "Saterland Frisian",
-					"lb" => "Luxembourgish",
-					"rm" => "Romansh",
-					"pt" => "Portuguese",
-					"ca" => "Catalan",
-					"gl" => "Galician",
-					"eu" => "Basque",
-					"ast" => "Asturian",
-					"an" => "Aragonese",
-					"fur" => "Friulan",
-					"sc" => "Sardinian",
-					"scn" => "Sicilian",
-					"oc" => "Occitan",
-					"be" => "Belarusian",
-					"cs" => "Czech",
-					"hu" => "Hungarian",
-					"sk" => "Slovak",
-					"uk" => "Ukrainian",
-					"csb" => "Kashubian",
-					"tt" => "Tatar",
-					"ba" => "Bashkir",
-					"os" => "Ossetian",
-					"sl" => "Slovene",
-					"sr" => "Serbian",
-					"hr" => "Croatian",
-					"bs" => "Bosnian",
-					"bg" => "Bulgarian",
-					"sq" => "Albanian",
-					"ro" => "Romanian",
-					"mk" => "Macedonian",
-					"el" => "Greek",
-					"iw" => "Hebrew",
-					"fa" => "Persian",
-					"ar" => "Arabic",
-					"ku" => "Kurdish",
-					"az" => "Azerbaijani",
-					"hy" => "Armenian",
-					"af" => "Afrikaans",
-					"sw" => "Kiswahili",
-					"uz" => "Uzbek",
-					"kk" => "Kazakh",
-					"ky" => "Kyrgyz",
-					"tg" => "Tajik",
-					"tk" => "Turkmen",
-					"ug" => "Uyghurche",
-					"hi" => "Hindi",
-					"si" => "Sinhalese",
-					"gu" => "Gujarati",
-					"ur" => "Urdu",
-					"mr" => "Marathi",
-					"pa" => "Punjabi",
-					"bn" => "Bengali",
-					"ta" => "Tamil",
-					"te" => "Telugu",
-					"kn" => "Kannada",
-					"zh_CN" => "Chinese Simplified",
-					"zh_TW" => "Chinese Traditional",
-					"ko" => "Korean",
-					"cfr" => "Taiwanese",
-					"th" => "Thai",
-					"vi" => "Vietnamese",
-					"in" => "Indonesian",
-					"ms" => "Malay",
-					"tl" => "Tagalog",
-					"eo" => "Esperanto",
-					"ia" => "Interlingua",
-					"la" => "Latin"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/" . $query,
-					[]
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-			
-		}else{
-			$proxy = $this->backend->get_ip();
-			
-			$query = [
-				"q" => $get["s"],
-				"start" => 0,
-				"stime" => 92452189 // ?
-			];
-			
-			if($get["lang"] !== "any"){
-				
-				$query["lang"] = $get["lang"];
-			}
-			
-			try{
-				$html = $this->get(
-					$proxy,
-					"https://curlie.org/search",
-					$query
-				);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		$nextpage =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"next-page",
-				"a"
-			);
-		
-		if(count($nextpage) !== 0){
-			
-			$nextpage =
-				$this->backend->store(
-					$nextpage[0]["attributes"]["href"],
-					"web",
-					$proxy
-				);
-		}else{
-			
-			$nextpage = null;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => $nextpage,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"site-item",
-				"div"
-			);
-		
-		foreach($items as $item){
-			
-			$this->fuckhtml->load($item);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByAttributeValue(
-					"target",
-					"_blank",
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName("site-descr");
-			
-			if(count($description) !== 0){
-				
-				$description =
-					$this->fuckhtml
-					->getTextContent(
-						$description[0]
-					);
-			}else{
-				
-				$description = null;
-			}
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-}

scraper/fivehpx.php

@@ -0,0 +1,327 @@
+<?php
+
+class fivehpx{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("fivehpx");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"sort" => [
+				"display" => "Sort",
+				"option" => [
+					"relevance" => "Relevance",
+					"pulse" => "Pulse",
+					"newest" => "Newest",
+					"aqs" => "AI Score"
+				]
+			],
+			"category" => [
+				"display" => "Category",
+				"option" => [
+					"any" => "Any category",
+					"landscape,nature" => "Landscape",
+					"city" => "City",
+					"events,sports" => "Events",
+					"creative" => "Creative"
+				]
+			],/*
+			"gear" => [
+				"display" => "Gear",
+				"option" => [
+					"any" => "Any gear",
+					"camera" => "Camera",
+					"phone" => "Phone"
+				]
+			],*/
+			"downloadable" => [
+				"display" => "Downloadable",
+				"option" => [
+					"any" => "Doesn't matter",
+					"yes" => "Yes"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $post_data = null){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($post_data === null){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-User: ?1",
+				"Priority: u=0, i",
+				"TE: trailers"]
+			);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: application/graphql-response+json,application/json;q=0.9",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://500px.com/",
+				"content-type: application/json",
+				"x-500px-platform: Web",
+				"Content-Length: " . strlen($post_data),
+				"Origin: https://500px.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Alt-Used: api-neo.500px.com",
+				"Connection: keep-alive",
+				"Cookie: x-500px-device-id-prod=a7b57e48-e477-4382-8c43-70c74415eb96",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"TE: trailers"]
+			);
+						
+			// set post data
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
+		}
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$variables, $proxy] =
+				$this->backend->get(
+					$get["npt"], "images"
+				);
+			
+			$variables = json_decode($variables, true);
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			$variables = [
+				"type" => "TEXT",
+				"sort" => strtoupper($get["sort"]),
+				"first" => 20,
+				"keyword" => $search,
+				"resourceTypes" => ["PHOTO"]
+			];
+			
+			if($get["downloadable"] != "any"){
+				
+				$variables["downloadable"] = "ALLOW";
+			}
+			
+			if($get["category"] != "any"){
+				
+				$variables["categories"] = [strtoupper($get["category"])];
+			}
+			/* for some reason this filter doesnt do anything? lol
+			if($get["gear"] != "any"){
+				
+				$variables["equipments"] = [strtoupper($get["gear"])];
+			}*/
+		}
+		
+		try{
+			
+			$json =
+				$this->get(
+					$proxy,
+					"https://api-neo.500px.com/graphql",
+					[],
+					json_encode([
+						"operationName" => "searchResource",
+						"variables" => $variables,
+						"extensions" => [
+							"clientLibrary" => [
+								"name" => "@apollo/client",
+								"version" => "4.1.6"
+							]
+						],
+						"query" => "query searchResource(\$keyword: String!, \$type: SearchType = TEXT, \$first: Int!, \$after: String, \$resourceTypes: [SearchResourceType!], \$categories: [String!], \$equipments: [String!], \$downloadable: PhotoDownloadable, \$sort: SearchSortOption = RELEVANCE, \$excludeNsfw: Boolean) {\n  searchResource(\n    keyword: \$keyword\n    type: \$type\n    first: \$first\n    after: \$after\n    resourceTypes: \$resourceTypes\n    categories: \$categories\n    equipments: \$equipments\n    downloadable: \$downloadable\n    sort: \$sort\n    excludeNsfw: \$excludeNsfw\n  ) {\n    edges {\n      cursor\n      node {\n        __typename\n        ... on Photo {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          questInfo {\n            rewardQuestName\n            __typename\n          }\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          description\n          downloadable\n          licensing {\n            status\n            __typename\n          }\n          urls {\n            size_600\n            size_1024\n            __typename\n          }\n          uploader {\n            id\n            username\n            avatar\n            displayName\n            isBlockedByMe\n            isFollowedByMe\n            isFollowingMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          isNsfw\n          isLikedByMe\n          viewerHasReposted\n          width\n          height\n          dominantColorLight\n          dominantColorDark\n          hasComment\n          uploadedAt\n          __typename\n        }\n        ... on Gallery {\n          id\n          isDeleted\n          isPrivate\n          kind\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          name\n          itemCount\n          hasComment\n          isLikedByMe\n          isNsfw\n          viewerHasReposted\n          creator {\n            id\n            username\n            displayName\n            avatar\n            isFollowedByMe\n            isFollowingMe\n            isBlockedByMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          coverPhotos(first: 5) {\n            id\n            urls {\n              size_600\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on PhotoGroup {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          description\n          numCount\n          publicItemCount\n          isLikedByMe\n          isNsfw\n          viewerHasReposted\n          uploader {\n            id\n            username\n            displayName\n            avatar\n            isFollowedByMe\n            isFollowingMe\n            isBlockedByMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          coverPhotos(first: 5) {\n            id\n            urls {\n              size_600\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on User {\n          id\n          username\n          displayName\n          avatar\n          location\n          isBlockedByMe\n          isFollowedByMe\n          isFollowingMe\n          membership {\n            membership\n            __typename\n          }\n          userCoverPhotos: coverPhotos(first: 6) {\n            id\n            urls {\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on Video {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          questInfo {\n            rewardQuestName\n            __typename\n          }\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          coverUrl\n          videoUrl\n          width\n          height\n          isNsfw\n          isLikedByMe\n          uploader {\n            id\n            username\n            avatar\n            displayName\n            isBlockedByMe\n            isFollowedByMe\n            isFollowingMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n      }\n      __typename\n    }\n    pageInfo {\n      hasNextPage\n      endCursor\n      __typename\n    }\n    totalCount\n    __typename\n  }\n}\n\nfragment PhotoHonorFields on PhotoHonor {\n  __typename\n  ... on PhotoHo
Line truncated
+					])
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch graphQL object");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode graphQL object");
+		}
+		
+		if(isset($json["errors"][0]["message"])){
+			
+			throw new Exception("500px returned an API error: " . $json["errors"][0]["message"]);
+		}
+		
+		if(!isset($json["data"]["searchResource"]["edges"])){
+			
+			throw new Exception("No edges returned by API");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		foreach($json["data"]["searchResource"]["edges"] as $image){
+			
+			$image = $image["node"];
+			$title =
+				trim(
+					$this->fuckhtml
+					->getTextContent(
+						$image["title"]
+					) . ": " .
+					$this->fuckhtml
+					->getTextContent(
+						$image["description"]
+					)
+					, " :"
+				);
+			
+			$small = $this->image_ratio(600, $image["width"], $image["height"]);
+			//$large = $this->image_ratio(2048, $image["width"], $image["height"]);
+			$large = $this->image_ratio(3840, $image["width"], $image["height"]);
+			
+			// for some fuck ass reason the invalid signature error gives me the
+			// signature key they use, lol
+			// signStrPreview:
+			// 67gfdsgf!5uiyoi3fdsphoto/37070890/0.jpg21790132400
+			// 67gfdsgf!5uiyoi3fdsphoto/211496303/0.jpg111790132400
+			
+			// hash("sha256", <-key->photo/<-url id->/0.jpg<-size-><-expiry->)
+			// https://cdn-resize-prod-com.500px.cloud/photo/211496303/0.jpg?s=1&expiry=1790132400&sig=b2873066126c294d7ca0b6f74db7f2626d79d5337a4c71add910f699734fade2
+			
+			$images = [];
+			
+			preg_match(
+				'/photo\/([0-9]+)/',
+				$image["urls"]["size_600"],
+				$image_id
+			);
+			
+			if(count($image_id) !== 0){
+				
+				$expiry = time() + 3600;
+				
+				$images[] = [
+					"url" =>
+						"https://cdn-resize-prod-com.500px.cloud/photo/{$image_id[1]}/0.jpg?s=4&expiry={$expiry}&sig=" .
+						hash("sha256", "67gfdsgf!5uiyoi3fdsphoto/{$image_id[1]}/0.jpg4{$expiry}"),
+					"width" => $large[0] > $image["width"] ? $image["width"] : $large[0],
+					"height" => $large[1] > $image["height"] ? $image["height"] : $large[1]
+				];
+			}
+			
+			// add thumbnail
+			$images[] = [
+				"url" => $image["urls"]["size_600"],
+				"width" => $small[0],
+				"height" => $small[1]
+			];
+			
+			$out["image"][] = [
+				"title" => $title,
+				"source" => $images,
+				"url" => "https://500px.com/photo/{$image["id"]}"
+			];
+		}
+		
+		// get NPT token
+		
+		if($json["data"]["searchResource"]["pageInfo"]["hasNextPage"] === true){
+			
+			$variables["after"] = $json["data"]["searchResource"]["pageInfo"]["endCursor"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($variables),
+					"images",
+					$proxy
+				);
+		}
+			
+		return $out;
+	}
+	
+	private function image_ratio($longest_edge, $width, $height){
+		
+		$ratio = [
+			$longest_edge / $width,
+			$longest_edge / $height
+		];
+		
+		if($ratio[0] < $ratio[1]){
+			
+			$ratio = $ratio[0];
+		}else{
+			
+			$ratio = $ratio[1];
+		}
+		
+		return [
+			floor($width * $ratio),
+			floor($height * $ratio)
+		];
+	}
+}

scraper/flickr.php

@@ -0,0 +1,415 @@
+<?php
+
+class flickr{
+	
+	const req_web = 0;
+	const req_xhr = 1;
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("flickr");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes",
+					"maybe" => "Maybe",
+					"no" => "No",
+				]
+			],
+			"sort" => [
+				"display" => "Sort by",
+				"option" => [
+					"relevance" => "Relevance",
+					"date-posted-desc" => "Newest uploads",
+					"date-posted-asc" => "Oldest uploads",
+					"date-taken-desc" => "Newest taken",
+					"date-taken-asc" => "Oldest taken",
+					"interestingness-desc" => "Interesting"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					// color_codes=
+					"0" => "Red",
+					"1" => "Brown",
+					"2" => "Orange",
+					"b" => "Pink",
+					"4" => "Yellow",
+					"3" => "Golden",
+					"5" => "Lime",
+					"6" => "Green",
+					"7" => "Sky blue",
+					"8" => "Blue",
+					"9" => "Purple",
+					"a" => "Hot pink",
+					"c" => "White",
+					"d" => "Gray",
+					"e" => "Black",
+					// styles= override
+					"blackandwhite" => "Black & white",
+				]
+			],
+			"style" => [ // styles=
+				"display" => "Style",
+				"option" => [
+					"any" => "Any style",
+					"depthoffield" => "Depth of field",
+					"minimalism" => "Minimalism",
+					"pattern" => "Patterns"
+				]
+			],
+			"license" => [
+				"display" => "License",
+				"option" => [
+					"any" => "Any license",
+					"1,2,3,4,5,6,9,11,12,13,14,15,16" => "All creative commons",
+					"4,5,6,9,10,11,12,13" => "Commercial use allowed",
+					"1,2,4,5,9,10,11,12,14,15" => "Modifications allowed",
+					"4,5,9,10,11,12" => "Commercial use & mods allowed",
+					"7,9,10" => "No known copyright restrictions",
+					"8" => "U.S Government works"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $reqtype){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($reqtype === flickr::req_web){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: same-origin",
+				"Sec-Fetch-User: ?1",
+				"Priority: u=0, i",
+				"TE: trailers"]
+			);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"Origin: https://www.flickr.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Referer: https://www.flickr.com/",
+				// Cookie:
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"TE: trailers"]
+			);
+		}
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filters, $proxy] =
+				$this->backend->get(
+					$get["npt"], "images"
+				);
+			
+			$filters = json_decode($filters, true);
+			
+			// Workaround for the future, if flickr deprecates &page argument on html page
+			/*
+			try{
+				
+				$json =
+					$this->get(
+						$proxy,
+						"https://api.flickr.com/services/rest",
+						[
+							"sort" => $data["sort"],
+							"parse_tags" => 1,
+							// url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o
+							"extras" => "can_comment,can_print,count_comments,count_faves,description,isfavorite,license,media,needs_interstitial,owner_name,path_alias,realname,rotation,url_sq,url_q,url_t,url_s,url_n,url_w,url_m,url_z,url_c,url_l",
+							"per_page" => 100,
+							"page" => $data["page"],
+							"lang" => "en-US",
+							"text" => $data["search"],
+							"viewerNSID" => "",
+							"method" => "flickr.photos.search",
+							"csrf" => "",
+							"api_key" => $data["api_key"],
+							"format" => "json",
+							"hermes" => 1,
+							"hermesClient" => 1,
+							"reqId" => $data["reqId"],
+							"nojsoncallback" => 1
+						]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JSON");
+			}*/
+			
+		}else{
+			
+			if(strlen($get["s"]) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+						
+			// compute filters
+			$filters = [
+				"page" => 1,
+				"sort" => $get["sort"]
+			];
+			
+			if($get["style"] != "any"){
+				
+				$filters["styles"] = $get["style"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				if($get["color"] != "blackandwhite"){
+					
+					$filters["color_codes"] = $get["color"];
+				}else{
+					
+					$filters["styles"] = "blackandwhite";
+				}
+			}
+			
+			if($get["license"] != "any"){
+				
+				$filters["license"] = $get["license"];
+			}
+			
+			switch($get["nsfw"]){
+				
+				case "yes": $filters["safe_search"] = 0; break;
+				case "maybe": $filters["safe_search"] = 2; break;
+				case "no": $filters["safe_search"] = 1; break;
+			}
+		}
+		
+		$get_params = [
+			"text" => $get["s"],
+			"per_page" => 50,
+			// scrape highest resolution
+			"extras" => "url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o",
+			"view_all" => 1
+		];
+		
+		$get_params = array_merge($get_params, $filters);
+		
+		$html =
+			$this->get(
+				$proxy,
+				"https://www.flickr.com/search/",
+				$get_params,
+				flickr::req_web
+			);
+		
+		// @TODO
+		// get api_key and reqId, if flickr deprecates &page
+		
+		$this->fuckhtml->load($html);
+		
+		//
+		// get response JSON
+		//
+		$scripts =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"modelExport",
+				"script"
+			);
+		
+		$found = false;
+		foreach($scripts as $script){
+			
+			$json =	
+				preg_split(
+					'/modelExport: ?/',
+					$script["innerHTML"],
+					2
+				);
+			
+			if(count($json) !== 0){
+				
+				$found = true;
+				$json = $json[1];
+				break;
+			}
+		}
+		
+		if($found === false){
+			
+			throw new Exception("Failed to grep JSON");
+		}
+		
+		$json =
+			json_decode(
+				$this->fuckhtml
+				->extract_json(
+					$json
+				),
+				true
+			);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(!isset($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"])){
+			
+			throw new Exception("Failed to access data object");
+		}
+		
+		foreach($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"] as $image){
+			
+			if(!isset($image["data"])){
+				
+				// flickr likes to gives us empty array objects
+				continue;
+			}
+			
+			$image = $image["data"];
+			
+			$title = [];
+			
+			if(isset($image["title"])){
+				
+				$title[] =
+					$this->fuckhtml
+					->getTextContent(
+						$image["title"]
+					);
+			}
+			
+			if(isset($image["description"])){
+				
+				$title[] =
+					$this->fuckhtml
+					->getTextContent(
+						str_replace(
+							"\n",
+							" ",
+							$image["description"]
+						)
+					);
+			}
+			
+			$title = implode(": ", $title);
+			
+			$sources = array_values($image["sizes"]["data"]);
+			
+			$suitable_sizes = ["n", "m", "w", "s"];
+			
+			$thumb = &$sources[0]["data"];
+			foreach($suitable_sizes as $testing_size){
+				
+				if(isset($image["sizes"]["data"][$testing_size])){
+					
+					$thumb = &$image["sizes"]["data"][$testing_size]["data"];
+					break;
+				}
+			}
+			
+			$og = &$sources[count($sources) - 1]["data"];
+			
+			$out["image"][] = [
+				"title" => $title,
+				"source" => [
+					[
+						"url" => "https:" . $og["displayUrl"],
+						"width" => (int)$og["width"],
+						"height" => (int)$og["height"]
+					],
+					[
+						"url" => "https:" . $thumb["displayUrl"],
+						"width" => (int)$thumb["width"],
+						"height" => (int)$thumb["height"]
+					]
+				],
+				"url" => "https://www.flickr.com/photos/" . $image["ownerNsid"] . "/" . $image["id"] . "/"
+			];
+		}
+		
+		$total_items = (int)$json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["totalItems"];
+		
+		if(($filters["page"]) * 50 < $total_items){
+			
+			$filters["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filters),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+}

scraper/ftm.php

@@ -136,7 +136,7 @@ class ftm{
 				"source" => [
 					[
 						"url" =>
-							"https://findthatmeme.us-southeast-1.linodeobjects.com/" .
+							"https://s3.thehackerblog.com/findthatmeme/" .
 							$thumb,
 						"width" => null,
 						"height" => null

scraper/ghostery.php

@@ -1,320 +0,0 @@
-<?php
-
-class ghostery{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("ghostery");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		if($page != "web"){
-			
-			return [];
-		}
-		
-		return [
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"any" => "All regions",
-					"AR" => "Argentina",
-					"AU" => "Australia",
-					"AT" => "Austria",
-					"BE" => "Belgium",
-					"BR" => "Brazil",
-					"CA" => "Canada",
-					"CL" => "Chile",
-					"DK" => "Denmark",
-					"FI" => "Finland",
-					"FR" => "France",
-					"DE" => "Germany",
-					"HK" => "Hong Kong",
-					"IN" => "India",
-					"ID" => "Indonesia",
-					"IT" => "Italy",
-					"JP" => "Japan",
-					"KR" => "Korea",
-					"MY" => "Malaysia",
-					"MX" => "Mexico",
-					"NL" => "Netherlands",
-					"NZ" => "New Zealand",
-					"NO" => "Norway",
-					"CN" => "People's Republic of China",
-					"PL" => "Poland",
-					"PT" => "Portugal",
-					"PH" => "Republic of the Philippines",
-					"RU" => "Russia",
-					"SA" => "Saudi Arabia",
-					"ZA" => "South Africa",
-					"ES" => "Spain",
-					"SE" => "Sweden",
-					"CH" => "Switzerland",
-					"TW" => "Taiwan",
-					"TR" => "Turkey",
-					"GB" => "United Kingdom",
-					"US" => "United States"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $country){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"Referer: https://ghosterysearch.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: same-origin",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"]
-		);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			parse_str($query, $query);
-			
-			// country
-			$country = $query["c"];
-			unset($query["c"]);
-			
-			$query = http_build_query($query);
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search?" . $query,
-						[],
-						$country
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://ghosterysearch.com/search",
-						[
-							"q" => $get["s"]
-						],
-						$get["country"]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		$this->fuckhtml->load($html);
-		
-		$results_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"results",
-				"section"
-			);
-		
-		if(count($results_wrapper) === 0){
-			
-			throw new Exception("Failed to grep result section");
-		}
-		
-		$this->fuckhtml->load($results_wrapper[0]);
-		
-		// get search results
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"li"
-			);
-		
-		if(count($results) === 0){
-			
-			return $out;
-		}
-		
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"url",
-					"a"
-				);
-			
-			if(count($a) === 0){
-				
-				continue;
-			}
-			
-			$a = $a[0];
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"h2"
-							)[0]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$this->fuckhtml
-							->getElementsByTagName(
-								"p"
-							)[0]
-						)
-					),
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a
-						["attributes"]
-						["href"]
-					),
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		// get pagination token
-		$pagination_wrapper =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"div"
-			);
-		
-		if(count($pagination_wrapper) !== 0){
-			
-			// found next page!
-			$this->fuckhtml->load($pagination_wrapper[0]);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				);
-			
-			if(count($a) !== 0){
-				
-				$q =
-					parse_url(
-						$this->fuckhtml
-						->getTextContent(
-							$a[count($a) - 1]
-							["attributes"]
-							["href"]
-						),
-						PHP_URL_QUERY
-					);
-				
-				$out["npt"] =
-					$this->backend
-					->store(
-						$q . "&c=" . $get["country"],
-						"web",
-						$proxy
-					);
-			}
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/google_api.php

@@ -0,0 +1,872 @@
+<?php
+
+// @TODO check for consent.google.com page, if need be
+
+class google_api{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("google_api");
+	}
+	
+	public function getfilters($page){
+		
+		$base = [
+			"country" => [ // gl=<country> (image: cr=countryAF)
+				"display" => "Country",
+				"option" => [
+					"any" => "Instance's country",
+					"af" => "Afghanistan",
+					"al" => "Albania",
+					"dz" => "Algeria",
+					"as" => "American Samoa",
+					"ad" => "Andorra",
+					"ao" => "Angola",
+					"ai" => "Anguilla",
+					"aq" => "Antarctica",
+					"ag" => "Antigua and Barbuda",
+					"ar" => "Argentina",
+					"am" => "Armenia",
+					"aw" => "Aruba",
+					"au" => "Australia",
+					"at" => "Austria",
+					"az" => "Azerbaijan",
+					"bs" => "Bahamas",
+					"bh" => "Bahrain",
+					"bd" => "Bangladesh",
+					"bb" => "Barbados",
+					"by" => "Belarus",
+					"be" => "Belgium",
+					"bz" => "Belize",
+					"bj" => "Benin",
+					"bm" => "Bermuda",
+					"bt" => "Bhutan",
+					"bo" => "Bolivia",
+					"ba" => "Bosnia and Herzegovina",
+					"bw" => "Botswana",
+					"bv" => "Bouvet Island",
+					"br" => "Brazil",
+					"io" => "British Indian Ocean Territory",
+					"bn" => "Brunei Darussalam",
+					"bg" => "Bulgaria",
+					"bf" => "Burkina Faso",
+					"bi" => "Burundi",
+					"kh" => "Cambodia",
+					"cm" => "Cameroon",
+					"ca" => "Canada",
+					"cv" => "Cape Verde",
+					"ky" => "Cayman Islands",
+					"cf" => "Central African Republic",
+					"td" => "Chad",
+					"cl" => "Chile",
+					"cn" => "China",
+					"cx" => "Christmas Island",
+					"cc" => "Cocos (Keeling) Islands",
+					"co" => "Colombia",
+					"km" => "Comoros",
+					"cg" => "Congo",
+					"cd" => "Congo, the Democratic Republic",
+					"ck" => "Cook Islands",
+					"cr" => "Costa Rica",
+					"ci" => "Cote D'ivoire",
+					"hr" => "Croatia",
+					"cu" => "Cuba",
+					"cy" => "Cyprus",
+					"cz" => "Czech Republic",
+					"dk" => "Denmark",
+					"dj" => "Djibouti",
+					"dm" => "Dominica",
+					"do" => "Dominican Republic",
+					"ec" => "Ecuador",
+					"eg" => "Egypt",
+					"sv" => "El Salvador",
+					"gq" => "Equatorial Guinea",
+					"er" => "Eritrea",
+					"ee" => "Estonia",
+					"et" => "Ethiopia",
+					"fk" => "Falkland Islands (Malvinas)",
+					"fo" => "Faroe Islands",
+					"fj" => "Fiji",
+					"fi" => "Finland",
+					"fr" => "France",
+					"gf" => "French Guiana",
+					"pf" => "French Polynesia",
+					"tf" => "French Southern Territories",
+					"ga" => "Gabon",
+					"gm" => "Gambia",
+					"ge" => "Georgia",
+					"de" => "Germany",
+					"gh" => "Ghana",
+					"gi" => "Gibraltar",
+					"gr" => "Greece",
+					"gl" => "Greenland",
+					"gd" => "Grenada",
+					"gp" => "Guadeloupe",
+					"gu" => "Guam",
+					"gt" => "Guatemala",
+					"gn" => "Guinea",
+					"gw" => "Guinea-Bissau",
+					"gy" => "Guyana",
+					"ht" => "Haiti",
+					"hm" => "Heard Island and Mcdonald Islands",
+					"va" => "Holy See (Vatican City State)",
+					"hn" => "Honduras",
+					"hk" => "Hong Kong",
+					"hu" => "Hungary",
+					"is" => "Iceland",
+					"in" => "India",
+					"id" => "Indonesia",
+					"ir" => "Iran, Islamic Republic",
+					"iq" => "Iraq",
+					"ie" => "Ireland",
+					"il" => "Israel",
+					"it" => "Italy",
+					"jm" => "Jamaica",
+					"jp" => "Japan",
+					"jo" => "Jordan",
+					"kz" => "Kazakhstan",
+					"ke" => "Kenya",
+					"ki" => "Kiribati",
+					"kp" => "Korea, Democratic People's Republic",
+					"kr" => "Korea, Republic",
+					"kw" => "Kuwait",
+					"kg" => "Kyrgyzstan",
+					"la" => "Lao People's Democratic Republic",
+					"lv" => "Latvia",
+					"lb" => "Lebanon",
+					"ls" => "Lesotho",
+					"lr" => "Liberia",
+					"ly" => "Libyan Arab Jamahiriya",
+					"li" => "Liechtenstein",
+					"lt" => "Lithuania",
+					"lu" => "Luxembourg",
+					"mo" => "Macao",
+					"mk" => "Macedonia, the Former Yugosalv Republic",
+					"mg" => "Madagascar",
+					"mw" => "Malawi",
+					"my" => "Malaysia",
+					"mv" => "Maldives",
+					"ml" => "Mali",
+					"mt" => "Malta",
+					"mh" => "Marshall Islands",
+					"mq" => "Martinique",
+					"mr" => "Mauritania",
+					"mu" => "Mauritius",
+					"yt" => "Mayotte",
+					"mx" => "Mexico",
+					"fm" => "Micronesia, Federated States",
+					"md" => "Moldova, Republic",
+					"mc" => "Monaco",
+					"mn" => "Mongolia",
+					"ms" => "Montserrat",
+					"ma" => "Morocco",
+					"mz" => "Mozambique",
+					"mm" => "Myanmar",
+					"na" => "Namibia",
+					"nr" => "Nauru",
+					"np" => "Nepal",
+					"nl" => "Netherlands",
+					"an" => "Netherlands Antilles",
+					"nc" => "New Caledonia",
+					"nz" => "New Zealand",
+					"ni" => "Nicaragua",
+					"ne" => "Niger",
+					"ng" => "Nigeria",
+					"nu" => "Niue",
+					"nf" => "Norfolk Island",
+					"mp" => "Northern Mariana Islands",
+					"no" => "Norway",
+					"om" => "Oman",
+					"pk" => "Pakistan",
+					"pw" => "Palau",
+					"ps" => "Palestinian Territory, Occupied",
+					"pa" => "Panama",
+					"pg" => "Papua New Guinea",
+					"py" => "Paraguay",
+					"pe" => "Peru",
+					"ph" => "Philippines",
+					"pn" => "Pitcairn",
+					"pl" => "Poland",
+					"pt" => "Portugal",
+					"pr" => "Puerto Rico",
+					"qa" => "Qatar",
+					"re" => "Reunion",
+					"ro" => "Romania",
+					"ru" => "Russian Federation",
+					"rw" => "Rwanda",
+					"sh" => "Saint Helena",
+					"kn" => "Saint Kitts and Nevis",
+					"lc" => "Saint Lucia",
+					"pm" => "Saint Pierre and Miquelon",
+					"vc" => "Saint Vincent and the Grenadines",
+					"ws" => "Samoa",
+					"sm" => "San Marino",
+					"st" => "Sao Tome and Principe",
+					"sa" => "Saudi Arabia",
+					"sn" => "Senegal",
+					"cs" => "Serbia and Montenegro",
+					"sc" => "Seychelles",
+					"sl" => "Sierra Leone",
+					"sg" => "Singapore",
+					"sk" => "Slovakia",
+					"si" => "Slovenia",
+					"sb" => "Solomon Islands",
+					"so" => "Somalia",
+					"za" => "South Africa",
+					"gs" => "South Georgia and the South Sandwich Islands",
+					"es" => "Spain",
+					"lk" => "Sri Lanka",
+					"sd" => "Sudan",
+					"sr" => "Suriname",
+					"sj" => "Svalbard and Jan Mayen",
+					"sz" => "Swaziland",
+					"se" => "Sweden",
+					"ch" => "Switzerland",
+					"sy" => "Syrian Arab Republic",
+					"tw" => "Taiwan, Province of China",
+					"tj" => "Tajikistan",
+					"tz" => "Tanzania, United Republic",
+					"th" => "Thailand",
+					"tl" => "Timor-Leste",
+					"tg" => "Togo",
+					"tk" => "Tokelau",
+					"to" => "Tonga",
+					"tt" => "Trinidad and Tobago",
+					"tn" => "Tunisia",
+					"tr" => "Turkey",
+					"tm" => "Turkmenistan",
+					"tc" => "Turks and Caicos Islands",
+					"tv" => "Tuvalu",
+					"ug" => "Uganda",
+					"ua" => "Ukraine",
+					"ae" => "United Arab Emirates",
+					"uk" => "United Kingdom",
+					"us" => "United States",
+					"um" => "United States Minor Outlying Islands",
+					"uy" => "Uruguay",
+					"uz" => "Uzbekistan",
+					"vu" => "Vanuatu",
+					"ve" => "Venezuela",
+					"vn" => "Viet Nam",
+					"vg" => "Virgin Islands, British",
+					"vi" => "Virgin Islands, U.S.",
+					"wf" => "Wallis and Futuna",
+					"eh" => "Western Sahara",
+					"ye" => "Yemen",
+					"zm" => "Zambia",
+					"zw" => "Zimbabwe"
+				]
+			],
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes", // safe=active
+					"no" => "No" // safe=off
+				]
+			],
+			"sort" => [ // sort
+				"display" => "Sort by",
+				"option" => [
+					"any" => "Any order",
+					"date:d" => "Oldest",
+					"date:a" => "Newest"
+				]
+			],
+			"newer" => [
+				"display" => "Newer than",
+				"option" => "_DATE"
+			],			
+			"rm_dupes" => [ // filter
+				"display" => "Remove duplicates",
+				"option" => [
+					"yes" => "Yes", // 1
+					"no" => "No" // 0
+				]
+			]
+		];
+		
+		switch($page){
+			
+			case "web":
+				return array_merge(
+					$base,
+					[
+						"lang" => [ // lr=<lang> (prefix lang with "lang_")
+							"display" => "Language",
+							"option" => [
+								"any" => "Any language",
+								"ar" => "Arabic",
+								"bg" => "Bulgarian",
+								"ca" => "Catalan",
+								"cs" => "Czech",
+								"da" => "Danish",
+								"de" => "German",
+								"el" => "Greek",
+								"en" => "English",
+								"es" => "Spanish",
+								"et" => "Estonian",
+								"fi" => "Finnish",
+								"fr" => "French",
+								"hr" => "Croatian",
+								"hu" => "Hungarian",
+								"id" => "Indonesian",
+								"is" => "Icelandic",
+								"it" => "Italian",
+								"iw" => "Hebrew",
+								"ja" => "Japanese",
+								"ko" => "Korean",
+								"lt" => "Lithuanian",
+								"lv" => "Latvian",
+								"nl" => "Dutch",
+								"no" => "Norwegian",
+								"pl" => "Polish",
+								"pt" => "Portuguese",
+								"ro" => "Romanian",
+								"ru" => "Russian",
+								"sk" => "Slovak",
+								"sl" => "Slovenian",
+								"sr" => "Serbian",
+								"sv" => "Swedish",
+								"tr" => "Turkish",
+								"zh-CN" => "Chinese (Simplified)",
+								"zh-TW" => "Chinese (Traditional)"
+							]
+						]
+					]
+				);
+				break;
+			
+			case "images":
+				return array_merge(
+					$base,
+					[
+						"size" => [ // imgSize
+							"display" => "Size",
+							"option" => [
+								"any" => "Any size",
+								"icon" => "Icon",
+								"small" => "Small",
+								"medium" => "Medium",
+								"large" => "Large",
+								"xlarge" => "X-Large",
+								"xxlarge" => "XX-Large",
+								"huge" => "Huge"
+							]
+						],
+						"format" => [ // fileType
+							"display" => "Format",
+							"option" => [
+								"any" => "Any format",
+								"jpg" => "JPG",
+								"gif" => "GIF",
+								"png" => "PNG",
+								"bmp" => "BMP",
+								"svg" => "SVG",
+								"webp" => "WEBP",
+								"ico" => "ICO",
+								"craw" => "RAW"
+							]
+						],
+						"color" => [
+							"display" => "Color",
+							"option" => [
+								"any" => "Any color",
+								
+								"color" => "Full color", // imgColorType
+								"mono" => "Black & White",
+								"trans" => "Transparent background",
+								
+								"red" => "Red", // imgDominantColor
+								"orange" => "Orange",
+								"yellow" => "Yellow",
+								"green" => "Green",
+								"teal" => "Teal",
+								"blue" => "Blue",
+								"purple" => "Purple",
+								"pink" => "Pink",
+								"white" => "White",
+								"gray" => "Gray",
+								"black" => "Black",
+								"brown" => "Brown"
+							]
+						],
+						"type" => [ // imgType
+							"display" => "Type",
+							"option" => [
+								"any" => "Any type",
+								"clipart" => "Clip Art",
+								"face" => "Faces",
+								"lineart" => "Line Drawing",
+								"stock" => "Stock photos",
+								"photo" => "Photos",
+								"animated" => "Animated",
+							]
+						],
+						"rights" => [ // rights
+							"display" => "Usage rights",
+							"option" => [
+								"any" => "Any license",
+								"cc_publicdomain" => "Public domain",
+								"cc_attribute" => "Attribution required",
+								"cc_sharealike" => "Sharealike",
+								"cc_noncommercial" => "Non-commercial use only",
+								"cc_nonderived" => "Original works"
+							]
+						]
+					]
+				);
+				break;
+		}
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+			
+		$headers = [
+			"Accept: application/json",
+			"Accept-Encoding: gzip"
+		];
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// follow redirects
+		curl_setopt($curlproc, CURLOPT_FOLLOWLOCATION, true);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		
+		return $data;
+	}
+	
+	
+	public function web($get){
+		
+		// rotate proxy + key on EVERY request
+		$keydata = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($keydata["increment"]);
+		
+		if($get["npt"]){
+			
+			// $p is never used
+			[$params, $p] = $this->backend->get(
+				$get["npt"],
+				"web"
+			);
+			
+			$params = json_decode($params, true);
+			
+			$params["key"] = $keydata["key"];
+			
+		}else{
+			
+			//$json = file_get_contents("scraper/google.json");
+			$params = [
+				"q" => $get["s"],
+				"cx" => config::GOOGLE_CX_ENDPOINT,
+				"num" => 10,
+				"start" => 1,
+				"key" => $keydata["key"]
+			];
+			
+			//
+			// parse filters
+			//
+			if($get["newer"] !== false){
+				
+				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
+			}
+			
+			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
+			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
+			if($get["lang"] != "any"){ $params["lr"] = "lang_" . $get["lang"]; }
+			
+			if($get["nsfw"] == "yes"){
+				
+				$params["safe"] = "off";
+			}else{
+				
+				$params["safe"] = "active";
+			}
+			
+			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://www.googleapis.com/customsearch/v1",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		if(isset($json["error"]["message"])){
+			
+			throw new Exception(
+				"API returned an error: " .
+				$json["error"]["message"] .
+				" (key #" . $keydata["increment"] . ")"
+			);
+		}
+		
+		if(!isset($json["items"])){
+			
+			// google just doesnt return items when theres no results
+			return $out;
+		}
+		
+		foreach($json["items"] as $result){
+			
+			$date = null;
+			
+			//
+			// probe for thumbnail
+			//
+			$probes = [
+				isset($result["pagemap"]["cse_thumbnail"][0]["src"]) ? $result["pagemap"]["cse_thumbnail"][0]["src"] : null,
+				isset($result["pagemap"]["cse_image"][0]["src"]) ? $result["pagemap"]["cse_image"][0]["src"] : null,
+				isset($result["pagemap"]["metatags"][0]["twitter:image"]) ? $result["pagemap"]["metatags"][0]["twitter:image"] : null,
+				isset($result["pagemap"]["metatags"][0]["og:image"]) ? $result["pagemap"]["metatags"][0]["og:image"] : null
+			];
+			
+			$thumb = [
+				"url" => null,
+				"ratio" => null
+			];
+			
+			foreach($probes as $probe){
+				
+				if($probe !== null){
+				
+					$thumb = [
+						"url" => $probe,
+						"ratio" => "16:9"
+					];
+					break;
+				}
+			}
+			
+			//
+			// probe for page format
+			//
+			$mime = "web";
+			if(isset($result["mime"])){
+				
+				$result["mime"] =
+					explode(
+						"/",
+						$result["mime"],
+						2
+					);
+				
+				if(count($result["mime"]) === 2){
+					
+					$mime = strtoupper($result["mime"][1]);
+				}
+			}
+			
+			$description = $result["snippet"];
+			
+			//
+			// Get date
+			//
+			$description_split =
+				explode(
+					"...", $description, 2
+				);
+			
+			if(count($description_split) === 1){
+				
+				$description = $result["snippet"];
+			}elseif(strlen($description_split[0]) < 17){
+				
+				$date = trim($description_split[0]);
+				$date_probe = strtotime($date);
+				
+				if($date_probe !== false){
+					
+					$description = $description_split[1];
+				}else{
+					
+					//
+					// fallback to getting date from meta tags
+					//
+					if(isset($result["pagemap"]["metatags"][0]["creationdate"])){
+						
+						$date = $result["pagemap"]["metatags"][0]["creationdate"];
+						
+					}elseif(isset($result["pagemap"]["metatags"][0]["moddate"])){
+						
+						$date = $result["pagemap"]["metatags"][0]["moddate"];
+					}else{
+						
+						$date = null;
+					}
+					
+					$description = $result["snippet"];
+				}
+			}
+			
+			if($date !== null){
+				
+				$date =
+					strtotime(
+						trim(
+							str_replace(
+								["D:", "'"],
+								"",
+								$date
+							)
+						)
+					);
+				
+				if($date === false){
+					
+					$date = null;
+				}
+			}
+			
+			$out["web"][] = [
+				"title" =>
+					$this->titledots(
+						$result["title"]
+					),
+				"description" =>
+					$this->titledots(
+						$description
+					),
+				"url" => $result["link"],
+				"date" => $date,
+				"type" => $mime,
+				"thumb" => $thumb,
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		// get npt
+		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
+			
+			unset($params["key"]);
+			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	
+	public function image($get){
+		
+		// rotate proxy + key on EVERY request
+		$keydata = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($keydata["increment"]);
+		
+		if($get["npt"]){
+			
+			// $p is never used
+			[$params, $p] = $this->backend->get(
+				$get["npt"],
+				"web"
+			);
+			
+			$params = json_decode($params, true);
+			
+			$params["key"] = $keydata["key"];
+			
+		}else{
+			
+			//$json = file_get_contents("scraper/google.json");
+			$params = [
+				"q" => $get["s"],
+				"cx" => config::GOOGLE_CX_ENDPOINT,
+				"num" => 10,
+				"start" => 1,
+				"searchType" => "image",
+				"key" => $keydata["key"]
+			];
+			
+			//
+			// parse filters
+			//
+			if($get["newer"] !== false){
+				
+				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
+			}
+			
+			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
+			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
+			
+			if($get["nsfw"] == "yes"){
+				
+				$params["safe"] = "off";
+			}else{
+				
+				$params["safe"] = "active";
+			}
+			
+			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
+			
+			// image filters
+			if($get["size"] != "any"){ $params["imgSize"] = $get["size"]; }
+			if($get["format"] != "any"){ $params["fileType"] = $get["format"]; }
+			
+			switch($get["color"]){
+				
+				case "any":
+					break;
+				
+				case "color":
+				case "mono":
+				case "trans":
+					$params["imgColorType"] = $get["color"];
+					break;
+				
+				default:
+					$params["imgDominantColor"] = $get["color"];
+					break;
+			}
+			
+			if($get["type"] != "any"){ $params["imgType"] = $get["type"]; }
+			if($get["rights"] != "any"){ $params["rights"] = $get["rights"]; }
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://www.googleapis.com/customsearch/v1",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(isset($json["error"]["message"])){
+			
+			throw new Exception(
+				"API returned an error: " .
+				$json["error"]["message"] .
+				" (key #" . $keydata["increment"] . ")"
+			);
+		}
+		
+		if(!isset($json["items"])){
+			
+			// google just doesnt return items when theres no results
+			return $out;
+		}
+		
+		foreach($json["items"] as $image){
+			
+			$out["image"][] = [
+				"title" => $this->titledots($image["title"]),
+				"source" => [
+					[
+						"url" => $image["link"],
+						"width" => (int)$image["image"]["width"],
+						"height" => (int)$image["image"]["height"]
+					],
+					[
+						"url" => $image["image"]["thumbnailLink"],
+						"width" => (int)$image["image"]["thumbnailWidth"],
+						"height" => (int)$image["image"]["thumbnailHeight"]
+					]
+				],
+				"url" => $image["image"]["contextLink"]
+			];
+		}
+		
+		// get npt
+		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
+			
+			unset($params["key"]);
+			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	
+	private function titledots($title){
+		
+		return trim($title, " .\t\n\r\0\x0B…");
+	}
+}

scraper/greppr.php

@@ -1,435 +0,0 @@
-<?php
-
-class greppr{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("greppr");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = [], $cookie = false){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		if($cookie === false){
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"DNT: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: none",
-				"Sec-Fetch-User: ?1"]
-			);
-		}else{
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"Cookie: PHPSESSID=" . $cookie,
-				"DNT: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: none",
-				"Sec-Fetch-User: ?1"]
-			);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$headers = [];
-		
-		curl_setopt(
-			$curlproc,
-			CURLOPT_HEADERFUNCTION,
-			function($curlproc, $header) use (&$headers){
-				
-				$len = strlen($header);
-				$header = explode(':', $header, 2);
-				
-				if(count($header) < 2){
-					
-					// ignore invalid headers
-					return $len;
-				}
-				
-				$headers[strtolower(trim($header[0]))] = trim($header[1]);
-
-				return $len;
-			}
-		);
-				
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		
-		return [
-			"headers" => $headers,
-			"data" => $data
-		];
-	}
-	
-	public function web($get, $first_attempt = true){
-		
-		if($get["npt"]){
-			
-			[$q, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			$q = json_decode($q, true);
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-		}
-		
-		// get token
-		// token[0] = static token that changes once a day
-		// token[1] = dynamic token that changes on every request
-		// token[1] = PHPSESSID cookie
-		$tokens = apcu_fetch("greppr_token");
-		
-		if(
-			$tokens === false ||
-			$first_attempt === false // force token fetch
-		){
-			
-			// we haven't gotten the token yet, get it
-			try{
-				
-				$response =
-					$this->get(
-						$proxy,
-						"https://greppr.org",
-						[]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search tokens");
-			}
-			
-			$tokens = $this->parse_token($response);
-			
-			if($tokens === false){
-				
-				throw new Exception("Failed to grep search tokens");
-			}
-		}
-		
-		try{
-			
-			if($get["npt"]){
-				
-				$params = [
-					$tokens[0] => $q["q"],
-					"s" => $q["s"],
-					"l" => 30,
-					"n" => $tokens[1]
-				];
-			}else{
-				
-				$params = [
-					$tokens[0] => $search,
-					"n" => $tokens[1]
-				];
-			}
-			
-			$searchresults = $this->get(
-				$proxy,
-				"https://greppr.org/search",
-				$params,
-				$tokens[2]
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch search page");
-		}
-		
-		if(strlen($searchresults["data"]) === 0){
-			
-			// redirected to main page, which means we got old token
-			// generate a new one
-			
-			// ... unless we just tried to do that
-			if($first_attempt === false){
-				
-				throw new Exception("Failed to get a new search token");
-			}
-			
-			return $this->web($get, false);
-		}
-		
-		// refresh the token with new data (this also triggers fuckhtml load)
-		$this->parse_token($searchresults, $tokens[2]);
-		
-		// response object
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		// get results for later
-		$results =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"result",
-				"div"
-			);
-		
-		// check for next page
-		$next_elem =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"pagination",
-				"ul"
-			);
-		
-		if(count($next_elem) !== 0){
-			
-			$this->fuckhtml->load($next_elem[0]);
-			
-			$as =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"page-link",
-					"a"
-				);
-			
-			$break = false;
-			foreach($as as $a){
-				
-				if($break === true){
-					
-					parse_str(
-						$this->fuckhtml
-						->getTextContent(
-							$a["attributes"]["href"]
-						),
-						$values
-					);
-					
-					$values = array_values($values);
-					
-					$out["npt"] =
-						$this->backend->store(
-							json_encode(
-								[
-									"q" => $values[0],
-									"s" => $values[1]
-								]
-							),
-							"web",
-							$proxy
-						);
-					break;
-				}
-				
-				if($a["attributes"]["href"] == "#"){
-					
-					$break = true;
-				}
-			}
-		}
-		
-		// scrape results
-		foreach($results as $result){
-			
-			$this->fuckhtml->load($result);
-			
-			$a =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"a"
-				)[0];
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"highlightedDesc",
-					"p"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->limitstrlen(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$date =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"p"
-				);
-			
-			$date =
-				strtotime(
-					explode(
-						":",
-						$this->fuckhtml
-						->getTextContent(
-							$date[count($date) - 1]["innerHTML"]
-						)
-					)[1]
-				);
-			
-			$out["web"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["innerHTML"]
-					),
-				"description" => $description,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$a["attributes"]["href"]
-					),
-				"date" => $date,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function parse_token($response, $cookie = false){
-		
-		$this->fuckhtml->load($response["data"]);
-
-		$scripts =
-			$this->fuckhtml
-			->getElementsByTagName("script");
-		
-		$found = false;
-		foreach($scripts as $script){
-			
-			preg_match(
-				'/window\.location ?= ?\'\/search\?([^=]+).*&n=([0-9]+)/',
-				$script["innerHTML"],
-				$tokens
-			);
-			
-			if(isset($tokens[1])){
-				
-				$found = true;
-				break;
-			}
-		}
-		
-		if($found === false){
-			
-			return false;
-		}
-		
-		$tokens = [
-			$tokens[1],
-			$tokens[2]
-		];
-		
-		if($cookie !== false){
-			
-			// we already specified a cookie, so use the one we have already
-			$tokens[] = $cookie;
-			apcu_store("greppr_token", $tokens);
-			
-			return $tokens;
-		}
-		
-		if(!isset($response["headers"]["set-cookie"])){
-			
-			// server didn't send a cookie
-			return false;
-		}
-		
-		// get cookie
-		preg_match(
-			'/PHPSESSID=([^;]+)/',
-			$response["headers"]["set-cookie"],
-			$cookie
-		);
-		
-		if(!isset($cookie[1])){
-			
-			// server sent an unexpected cookie
-			return false;
-		}
-		
-		$tokens[] = $cookie[1];
-		apcu_store("greppr_token", $tokens);
-		
-		return $tokens;
-	}
-	
-	private function limitstrlen($text){
-		
-		return explode("\n", wordwrap($text, 300, "\n"))[0];
-	}
-}

scraper/imgur.php

@@ -182,6 +182,23 @@ class imgur{
 			throw new Exception("Failed to fetch HTML");
 		}
 		
+		$json = json_decode($html, true);
+		
+		if($json){
+			
+			// {"data":{"error":"Imgur is temporarily over capacity. Please try again later."},"success":false,"status":403}
+			
+			if(isset($json["data"]["error"])){
+				
+				if(stripos($json["data"]["error"], "capacity")){
+					
+					throw new Exception("Imgur IP blocked this 4get instance or request proxy. Try again");
+				}
+			}
+			
+			throw new Exception("Imgur returned an unknown error (IP ban?)");
+		}
+		
 		$this->fuckhtml->load($html);
 		
 		$posts =
@@ -197,7 +214,14 @@ class imgur{
 			
 			$image =
 				$this->fuckhtml
-				->getElementsByTagName("img")[0];
+				->getElementsByTagName("img");
+			
+			if(count($image) === 0){
+				
+				continue;
+			}
+			
+			$image = $image[0];
 			
 			$image_url = "https:" . substr($this->fuckhtml->getTextContent($image["attributes"]["src"]), 0, -5);
 			

scraper/marginalia.php

@@ -3,7 +3,10 @@
 class marginalia{
 	public function __construct(){
 		
-		include "lib/fuckhtml.php";
+		include "lib/anubis.php";
+		$this->anubis = new anubis();
+		
+		include_once "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
 		
 		include "lib/backend.php";
@@ -102,7 +105,40 @@ class marginalia{
 		);
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $get_cookies = 1){
+		
+		$curlproc = curl_init();
+		
+		switch($get_cookies){
+			
+			case 0:
+				$cookies = "";
+				$cookies_tmp = [];
+				curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
+					
+					$length = strlen($header);
+					
+					$header = explode(":", $header, 2);
+					
+					if(trim(strtolower($header[0])) == "set-cookie"){
+						
+						$cookie_tmp = explode("=", trim($header[1]), 2);
+						
+						$cookies_tmp[trim($cookie_tmp[0])] =
+							explode(";", $cookie_tmp[1], 2)[0];
+					}
+					
+					return $length;
+				});
+				break;
+			
+			case 1:
+				$cookies = "";
+				break;
+			
+			default:
+				$cookies = "Cookie: " . $get_cookies;
+		}
 		
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
@@ -110,6 +146,7 @@ class marginalia{
 			"Accept-Language: en-US,en;q=0.5",
 			"Accept-Encoding: gzip",
 			"DNT: 1",
+			$cookies,
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
 			"Sec-Fetch-Dest: document",
@@ -118,8 +155,6 @@ class marginalia{
 			"Sec-Fetch-User: ?1"
 		];
 		
-		$curlproc = curl_init();
-		
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
@@ -145,11 +180,23 @@ class marginalia{
 			throw new Exception(curl_error($curlproc));
 		}
 		
-		curl_close($curlproc);
+		if($get_cookies === 0){
+			
+			$cookie = [];
+			
+			foreach($cookies_tmp as $key => $value){
+				
+				$cookie[] = $key . "=" . $value;
+			}
+			
+			curl_close($curlproc);
+			return implode(";", $cookie);
+		}
+		
 		return $data;
 	}
 	
-	public function web($get){
+	public function web($get, $ss = ""){
 		
 		$search = [$get["s"]];
 		if(strlen($get["s"]) === 0){
@@ -220,13 +267,14 @@ class marginalia{
 			"related" => []
 		];
 		
+		// API scraper
 		if(config::MARGINALIA_API_KEY !== null){
 			
 			try{
 				$json =
 					$this->get(
 						$this->backend->get_ip(), // no nextpage
-						"https://api.marginalia.nu/" . config::MARGINALIA_API_KEY . "/search/" . urlencode($search),
+						"https://api.marginalia-search.com/" . config::MARGINALIA_API_KEY . "/search/" . urlencode($search),
 						[
 							"count" => 20
 						]
@@ -241,8 +289,18 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
+			if(strpos($json, "<title>Bad Gateway (502)</title>")){
+				
+				throw new Exception("Marginalia returned a 502 error");
+			}
+			
 			$json = json_decode($json, true);
 			
+			if($json === null){
+				
+				throw new Exception("Failed to decode JSON");
+			}
+			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -263,37 +321,215 @@ class marginalia{
 			return $out;
 		}
 		
-		// no more cloudflare!! Parse html by default
-		$params = [
-			"query" => $search
-		];
+		// HTML parser
+		$proxy = $this->backend->get_ip();
 		
-		foreach(["adtech", "recent", "intitle"] as $v){
+		//
+		// Bypass anubis check
+		//
+		/*
+		if(($anubis_key = apcu_fetch("marginalia_cookie")) === false){
 			
-			if($get[$v] == "yes"){
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/search",
+						[
+							"query" => $search
+						]
+					);
 					
-				switch($v){
+			}catch(Exception $error){
 				
-					case "adtech": $params["adtech"] = "reduce"; break;
-					case "recent": $params["recent"] = "recent"; break;
-					case "adtech": $params["searchTitle"] = "title"; break;
+				throw new Exception("Failed to get anubis challenge");
+			}
+			
+			try{
+				
+				$anubis_data = $this->anubis->scrape($html);
+			}catch(Exception $error){
+				
+				throw new Exception($error);
+			}
+			
+			// send anubis response & get cookies
+			// https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge?response=0000018966b086834f738bacba6031028adb5aa875974ead197a8b75778baf3a&nonce=39947&redir=https%3A%2F%2Fold-search.marginalia.nu%2F&elapsedTime=1164
+			
+			try{
+				
+				$anubis_key =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge",
+						[
+							"response" => $anubis_data["response"],
+							"nonce" => $anubis_data["nonce"],
+							"redir" => "https://old-search.marginalia.nu/",
+							"elapsedTime" => random_int(1000, 2000)
+						],
+						0
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to submit anubis challenge");
+			}
+			
+			apcu_store("marginalia_cookie", $anubis_key);
+		}*/
+		
+		if($get["npt"]){
+			
+			[$params, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"web"
+				);
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/search?" . $params,
+						[],
+						//$anubis_key
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get HTML");
+			}
+			
+		}else{
+			
+			if($ss == ""){
+				
+				$params = [
+					"query" => $search
+				];
+				
+				foreach(["adtech", "recent", "intitle"] as $v){
+					
+					if($get[$v] == "yes"){
+						
+						switch($v){
+							
+							case "adtech": $params["adtech"] = "reduce"; break;
+							case "recent": $params["recent"] = "recent"; break;
+							case "adtech": $params["searchTitle"] = "title"; break;
+						}
+					}
 				}
+			}else{
+				
+				$params = [];
+			}
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://old-search.marginalia.nu/search" . $ss,
+						$params,
+						//$anubis_key
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get HTML");
 			}
 		}
 		
-		try{
-			$html =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://search.marginalia.nu/search",
-					$params
-				);
-		}catch(Exception $error){
+		$this->fuckhtml->load($html);
 		
-			throw new Exception("Failed to get HTML");
+		// detect meta redirect
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"title"
+			);
+		
+		if(
+			count($title) !== 0 &&
+			$this->fuckhtml
+			->getTextContent(
+				$title[0]
+			) == "Error"
+		){
+			
+			// redirect detected
+			
+			// get timeout
+			$timeout =
+				$this->fuckhtml
+				->getElementById(
+					"countdown",
+					"b"
+				);
+			
+			if(count($timeout) === null){
+				
+				throw new Exception("Failed to find timeout value");
+			}
+			
+			$timeout =
+				$this->fuckhtml
+				->getTextContent(
+					$timeout
+				);
+			
+			preg_match(
+				'/location\.replace\(\'([^\']+)\'\)/',
+				$html,
+				$redirect
+			);
+			
+			if(!isset($redirect[1])){
+				
+				throw new Exception("Failed to grep redirect value");
+			}
+			
+			$one = 1;
+			$redirect =
+				str_replace(
+					"/search",
+					"",
+					$redirect[1],
+					$one
+				);
+			
+			sleep((int)$timeout);
+			return $this->web($get, $redirect);
 		}
 		
-		$this->fuckhtml->load($html);
+		// detect internal error
+		$infobox =
+			$this->fuckhtml
+			->getElementsByClassName("infobox");
+		
+		if(count($infobox) !== 0){
+			
+			foreach($infobox as $i){
+				
+				$this->fuckhtml->load($i);
+				
+				$h2 =
+					$this->fuckhtml
+					->getElementsByTagName("h2");
+				
+				if(
+					count($h2) !== 0 &&
+					$this->fuckhtml
+					->getTextContent(
+						$h2[0]
+					) == "Internal error"
+				){
+					
+					throw new Exception("Marginalia returned an internal server error");
+				}
+			}
+						
+			// reset
+			$this->fuckhtml->load($html);
+		}
 		
 		$sections =
 			$this->fuckhtml
@@ -387,6 +623,65 @@ class marginalia{
 			];
 		}
 		
+		// get next page
+		$this->fuckhtml->load($html);
+		
+		$pagination =
+			$this->fuckhtml
+			->getElementsByAttributeValue(
+				"aria-label",
+				"pagination",
+				"nav"
+			);
+		
+		if(count($pagination) === 0){
+			
+			// no pagination
+			return $out;
+		}
+		
+		$this->fuckhtml->load($pagination[0]);
+		
+		$pages =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"page-link",
+				"a"
+			);
+		
+		$found_current_page = false;
+		
+		foreach($pages as $page){
+			
+			if(
+				stripos(
+					$page["attributes"]["class"],
+					"active"
+				) !== false
+			){
+				
+				$found_current_page = true;
+				continue;
+			}
+			
+			if($found_current_page){
+				
+				// we found current page index, and we iterated over
+				// the next page <a>
+				
+				$out["npt"] =
+					$this->backend->store(
+						parse_url(
+							$page["attributes"]["href"],
+							PHP_URL_QUERY
+						),
+						"web",
+						$proxy
+					);
+				break;
+			}
+		}
+		
 		return $out;
 	}
 }

scraper/mojeek.php

@@ -16,18 +16,7 @@ class mojeek{
 			
 			case "web":
 				return [
-					"focus" => [
-						"display" => "Focus",
-						"option" => [
-							"any" => "No focus",
-							"blogs" => "Blogs",
-							"Dictionary" => "Dictionary",
-							"Recipes" => "Recipes",
-							"Time" => "Time",
-							"Weather" => "Weather"
-						]
-					],
-					"lang" => [
+					"lang" => [ // lb
 						"display" => "Language",
 						"option" => [
 							"any" => "Any language",
@@ -86,7 +75,7 @@ class mojeek{
 							"zu" => "Zulu"
 						]
 					],
-					"country" => [
+					"country" => [ // rb
 						"display" => "Country",
 						"option" => [
 							"any" => "No location bias",
@@ -341,27 +330,20 @@ class mojeek{
 							"zw" => "Zimbabwe"
 						]
 					],
-					"region" => [
-						"display" => "Region",
+					"sort" => [
+						"display" => "Sort by",
 						"option" => [
-							"any" => "Any region",
-							"eu" => "European Union",
-							"de" => "Germany",
-							"fr" => "France",
-							"uk" => "United Kingdom"
+							"relevance" => "Relevance",
+							"date" => "Date" // datewr=100
 						]
 					],
-					"domain" => [
-						"display" => "Results per domain",
-						"option" => [
-							"1" => "1 result",
-							"2" => "2 results",
-							"3" => "3 results",
-							"4" => "4 results",
-							"5" => "5 results",
-							"10" => "10 results",
-							"0" => "Unlimited",
-						]
+					"newer" => [
+						"display" => "Newer than",
+						"option" => "_DATE"
+					],
+					"older" => [
+						"display" => "Older than",
+						"option" => "_DATE"
 					]
 				];
 				break;
@@ -371,21 +353,29 @@ class mojeek{
 		}
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $is_api = false){
 		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"
-		];
+		if($is_api){
+						
+			$headers = [
+				"Accept: application/json"
+			];
+		}else{
+			
+			$headers = [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1"
+			];
+		}
 		
 		$curlproc = curl_init();
 		
@@ -420,6 +410,154 @@ class mojeek{
 	
 	public function web($get){
 		
+		return $this->web_api($get);
+	}
+	
+	public function web_api($get){
+				
+		$key = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($key["increment"]);
+		
+		if($get["npt"]){
+			
+			[$params, $proxy] = $this->backend->get($get["npt"], "web");
+			$params = json_decode($params, true);
+			$params["api_key"] = $key["key"];
+			
+		}else{
+			
+			if(strlen($get["s"]) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$params = [
+				"api_key" => $key["key"],
+				"q" => $get["s"],
+				"date" => 1,
+				"tlen" => 127, // title length
+				"dlen" => 511, // description length
+				"fmt" => "json"
+			];
+			
+			if($get["lang"] != "any"){
+				
+				$params["lb"] = $get["lang"];
+			}
+			
+			if($get["country"] != "any"){
+				
+				$params["rb"] = $get["country"];
+			}
+			
+			if($get["sort"] == "date"){
+				
+				$params["datewr"] = 100;
+			}
+			
+			if($get["newer"] !== false){
+				
+				$params["since"] = date("Ymd", $get["newer"]);
+			}
+			
+			if($get["older"] !== false){
+				
+				$params["before"] = date("Ymd", $get["before"]);
+			}
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://api.mojeek.com/search",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to get JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		if(!isset($json["response"]["status"])){
+			
+			throw new Exception("API did not return a status field");
+		}
+		
+		if($json["response"]["status"] != "OK"){
+			
+			throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
+		}
+		
+		if(!isset($json["response"]["results"])){
+			
+			throw new Exception("Mojeek API did not return a results object");
+		}
+		
+		foreach($json["response"]["results"] as $entry){
+			
+			$out["web"][] = [
+				"title" => html_entity_decode($entry["title"]),
+				"description" => html_entity_decode($entry["desc"]),
+				"url" => $entry["url"],
+				"date" => $entry["timestamp"],
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		// get npt
+		if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
+			
+			unset($params["api_key"]);
+			if(!isset($params["s"])){
+				
+				$params["s"] = 11;
+			}else{
+				
+				$params["s"] = $params["s"] + 10;
+			}
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	public function web_browser($get){
+		
 		if($get["npt"]){
 			
 			[$token, $proxy] = $this->backend->get($get["npt"], "web");
@@ -447,7 +585,7 @@ class mojeek{
 			$proxy = $this->backend->get_ip();
 			$lang = $get["lang"];
 			$country = $get["country"];
-			$region = $get["region"];
+			//$region = $get["region"];
 			$domain = $get["domain"];
 			$focus = $get["focus"];
 			
@@ -457,8 +595,7 @@ class mojeek{
 				"tn" => 7, // number of news results/page
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
-				"dlen" => 511, // max length of description
-				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -479,11 +616,11 @@ class mojeek{
 				
 				$params["lb"] = $lang;
 			}
-			
+			/*
 			if($region != "any"){
 				
 				$params["reg"] = $region;
-			}
+			}*/
 			
 			if($domain != "1"){
 				
@@ -501,13 +638,15 @@ class mojeek{
 				
 				throw new Exception("Failed to get HTML");
 			}
-			/*
-			$handle = fopen("scraper/mojeek.html", "r");
-			$html = fread($handle, filesize("scraper/mojeek.html"));
-			fclose($handle);*/
-			
 		}
 		
+		if(strlen($html) === 0){
+			
+			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
+		}
+		
+		$this->fuckhtml->load($html);
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -524,7 +663,7 @@ class mojeek{
 			"related" => []
 		];
 		
-		$this->fuckhtml->load($html);
+		$this->detect_block();
 		
 		$results =
 			$this->fuckhtml
@@ -695,15 +834,18 @@ class mojeek{
 				preg_match(
 					'/\/image\?img=([^&]+)/i',
 					$thumb[0]["attributes"]["src"],
-					$thumb
+					$matches
 				);
 				
-				if(count($thumb) === 2){
+				if(count($matches) === 2){
 					
+					// for some reason, if we dont get the image from mojeek
+					// it sometimes fail to fetch the right image URL
 					$answer["thumb"] =
+						"https://mojeek.com" .
 						$this->fuckhtml
 						->getTextContent(
-							$thumb[1]
+							$thumb[0]["attributes"]["src"]
 						);
 				}
 			}
@@ -995,7 +1137,12 @@ class mojeek{
 		return $out;
 	}
 	
-	public function news($get){
+	public function news(){
+		
+		throw new Exception("Currently unsupported");
+	}
+	
+	public function news_browser($get){
 		
 		$search = $get["s"];
 		
@@ -1032,6 +1179,8 @@ class mojeek{
 		
 		$this->fuckhtml->load($html);
 		
+		$this->detect_block();
+		
 		$articles =
 			$this->fuckhtml->getElementsByTagName("article");
 		
@@ -1164,6 +1313,38 @@ class mojeek{
 		return $out;
 	}
 	
+	private function detect_block(){
+		
+		$captcha =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"captcha-box",
+				"div"
+			);
+		
+		if(count($captcha) !== 0){
+			
+			throw new Exception("Mojeek returned a captcha");
+		}
+		
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"title"
+			);
+		
+		if(
+			count($title) !== 0 &&
+			$this->fuckhtml
+			->getTextContent(
+				$title[0]["innerHTML"]
+			) == "403 - Forbidden"
+		){
+			
+			throw new Exception("Mojeek blocked this instance or request proxy.");
+		}
+	}
+	
 	private function titledots($title){
 		
 		return trim($title, ". \t\n\r\0\x0B");

scraper/pexels.php

@@ -0,0 +1,285 @@
+<?php
+
+class pexels{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("pexels");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"time" => [ // date_from
+				"display" => "Time posted",
+				"option" => [
+					"any" => "Any time",
+					"last_24_hours" => "Last 24 hours",
+					"last_week" => "Last week",
+					"last_month" => "Last month",
+					"last_year" => "Last year"
+				]
+			],
+			"orientation" => [
+				"display" => "Orientation",
+				"option" => [
+					"any" => "Any orientation",
+					"landscape" => "Horizontal",
+					"portrait" => "Vertical",
+					"square" => "Square"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					"795548" => "Brown",
+					"F44336" => "Red",
+					"E91E63" => "Hot pink",
+					"9C27B0" => "Magenta",
+					"673AB7" => "Purple",
+					"3F51B5" => "Indigo",
+					"2196F3" => "Blue",
+					"03A9F4" => "Light blue",
+					"00BCD4" => "Cyan",
+					"009688" => "Forest green",
+					"4CAF50" => "Green",
+					"8BC34A" => "Lime",
+					"CDDC39" => "Pear",
+					"FFEB3B" => "Yellow",
+					"FFC107" => "Gold",
+					"FF9800" => "Orange",
+					"FF5722" => "Tomato",
+					"9E9E9E" => "Gray",
+					"607D8B" => "Teal",
+					"000000" => "Black",
+					"FFFFFF" => "White"
+				]
+			],
+			"people_count" => [
+				"display" => "Head count",
+				"option" => [
+					"any" => "Any number",
+					"0" => "0",
+					"1" => "1",
+					"2" => "2",
+					"3_plus" => "3+",
+				]
+			],
+			"people_age" => [
+				"display" => "People's age",
+				"option" => [
+					"any" => "Any age",
+					"baby" => "Baby",
+					"child" => "Child",
+					"teenager" => "Teenager",
+					"adult" => "Adult",
+					"senior_adult" => "Senior adult"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		$search = $get["query"];
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: https://www.pexels.com/search/" . rawurlencode($search) . "/",
+			"Content-Type: application/json",
+			"secret-key: H2jk9uKnhRmL6WPwh89zBezWvr", // hardcoded but like, people on github have been using this shit since 23'
+			"X-Client-Type: react",
+			"X-Next-Forwarded-CF-Connecting-IP: ",
+			"X-Next-Forwarded-CF-IPCountry: ",
+			"X-Next-Forwarded-CF-IPRegionCode: ",
+			"X-Active-Experiment: ",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Alt-Used: www.pexels.com",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=4",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filter, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$filter = json_decode($filter, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			// ?query=blue%20footed%20booby&page=1&per_page=24&seo_tags=true
+			$filter = [
+				"query" => $search,
+				"page" => 1,
+				"per_page" => 24,
+				"seo_tags" => "true"
+			];
+			
+			// add filters
+			if($get["time"] != "any"){
+				
+				$filter["date_from"] = $get["time"];
+			}
+			
+			if($get["orientation"] != "any"){
+				
+				$filter["orientation"] = $get["orientation"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				$filter["color"] = $get["color"];
+			}
+			
+			if($get["people_count"] != "any"){
+				
+				$filter["people_count"] = $get["people_count"];
+			}
+			
+			if($get["people_age"] != "any"){
+				
+				$filter["people_age"] = $get["people_age"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		try{
+			$html =
+				$this->get(
+					$proxy,
+					"https://www.pexels.com/en-us/api/v3/search/photos",
+					$filter
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($html, true);
+		
+		if(!isset($json["data"])){
+			
+			throw new Exception("Pexels did not return a data object");
+		}
+		
+		foreach($json["data"] as $image){
+			
+			$thumb_size =
+				$this->imgratio(
+					(int)$image["attributes"]["width"],
+					(int)$image["attributes"]["height"],
+					350
+				);
+			
+			$out["image"][] = [
+				"title" => $image["attributes"]["title"] . ": " . $image["attributes"]["description"],
+				"source" => [
+					[
+						"url" => $image["attributes"]["image"]["download_link"],
+						"width" => (int)$image["attributes"]["width"],
+						"height" => (int)$image["attributes"]["height"]
+					],
+					[
+						"url" =>
+							preg_replace(
+								'/(?:w|h)=[0-9]+$/',
+								"w=350",
+								$image["attributes"]["image"]["small"]
+							),
+						"width" => $thumb_size[0],
+						"height" => $thumb_size[1]
+					]
+				],
+				"url" =>
+					"https://pexels.com/photo/" .
+					$image["attributes"]["slug"] . "-" .
+					$image["attributes"]["id"] . "/"
+			];
+		}
+		
+		// get next page
+		if((int)$json["pagination"]["current_page"] < (int)$json["pagination"]["total_pages"]){
+			
+			$filter["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filter),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/pinterest.php

@@ -13,31 +13,104 @@ class pinterest{
 		return [];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], &$cookies, $header_data_post = null){
 		
 		$curlproc = curl_init();
 		
-		if($get !== []){
+		if($header_data_post === null){
+			
+			// handling GET
+						
+			// extract cookies
+			$cookies_tmp = [];
+			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
+				
+				$length = strlen($header);
+				
+				$header = explode(":", $header, 2);
+				
+				if(trim(strtolower($header[0])) == "set-cookie"){
+					
+					$cookie_tmp = explode("=", trim($header[1]), 2);
+					
+					$cookies_tmp[trim($cookie_tmp[0])] =
+						explode(";", $cookie_tmp[1], 2)[0];
+				}
+				
+				return $length;
+			});
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: application/json, text/javascript, */*, q=0.01",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"Referer: https://ca.pinterest.com/",
+				"X-Requested-With: XMLHttpRequest",
+				"X-APP-VERSION: 78f8764",
+				"X-Pinterest-AppState: active",
+				"X-Pinterest-Source-Url: /",
+				"X-Pinterest-PWS-Handler: www/index.js",
+				"screen-dpr: 1",
+				"is-preload-enabled: 1",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-origin",
+				"Connection: keep-alive",
+				"Alt-Used: ca.pinterest.com",
+				"Priority: u=0",
+				"TE: trailers"]
+			);
+			
+			if($get !== []){
+				$get = http_build_query($get);
+				$url .= "?" . $get;
+			}
+		}else{
+			
+			// handling POST (pagination)
 			$get = http_build_query($get);
-			$url .= "?" . $get;
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: application/json, text/javascript, */*, q=0.01",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"Content-Type: application/x-www-form-urlencoded",
+				"Content-Length: " . strlen($get),
+				"Referer: https://ca.pinterest.com/",
+				"X-Requested-With: XMLHttpRequest",
+				"X-APP-VERSION: 78f8764",
+				"X-CSRFToken: " . $cookies["csrf"],
+				"X-Pinterest-AppState: active",
+				"X-Pinterest-Source-Url: /search/pins/?rs=ac&len=2&q=" . urlencode($header_data_post) . "&eq=" . urlencode($header_data_post),
+				"X-Pinterest-PWS-Handler: www/search/[scope].js",
+				"screen-dpr: 1",
+				"is-preload-enabled: 1",
+				"Origin: https://ca.pinterest.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-origin",
+				"Connection: keep-alive",
+				"Alt-Used: ca.pinterest.com",
+				"Cookie: " . $cookies["cookie"],
+				"TE: trailers"]
+			);
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
 		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0",
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1"]
-		);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -54,6 +127,26 @@ class pinterest{
 			throw new Exception(curl_error($curlproc));
 		}
 		
+		if($header_data_post === null){
+			
+			if(!isset($cookies_tmp["csrftoken"])){
+				
+				throw new Exception("Failed to grep CSRF token");
+			}
+			
+			$cookies = "";
+			
+			foreach($cookies_tmp as $cookie_name => $cookie_value){
+				
+				$cookies .= $cookie_name . "=" . $cookie_value . "; ";
+			}
+			
+			$cookies = [
+				"csrf" => $cookies_tmp["csrftoken"],
+				"cookie" => rtrim($cookies, " ;")
+			];
+		}
+		
 		curl_close($curlproc);
 		return $data;
 	}
@@ -62,17 +155,68 @@ class pinterest{
 		
 		if($get["npt"]){
 			
-			// @TODO
-			// post data for next page
-			$data = [
-				"source_url" => "/search/pins/?q=" . urlencode($search) . "&rs=typed",
-				"data" =>
-					json_encode(
+			[$data, $proxy] =
+				$this->backend->get(
+					$get["npt"], "images"
+				);
+			
+			$data = json_decode($data, true);
+			
+			$search = $data["q"];
+			$cookies = $data["cookies"];
+			
+			try{
+				$json =
+					$this->get(
+						$proxy,
+						"https://ca.pinterest.com/resource/BaseSearchResource/get/",
 						[
-							// {"options":{"applied_filters":null,"appliedProductFilters":"---","article":null,"auto_correction_disabled":false,"corpus":null,"customized_rerank_type":null,"domains":null,"filters":null,"journey_depth":null,"page_size":null,"price_max":null,"price_min":null,"query_pin_sigs":null,"query":"higurashi","redux_normalize_feed":true,"rs":"typed","scope":"pins","selected_one_bar_modules":null,"source_id":null,"source_module_id":null,"top_pin_id":null,"bookmarks":["Y2JVSG81V2sxcmNHRlpWM1J5VFVad1ZsWlVRbXhpVmtreVZsZHpOV0pIU2tkV2FscFhVbXhhVkZreU1WSmtNREZWVjIxR1RrMXNTbEJXYlhSaFVtMVdjMVZ1U2xaaWEzQnpXVlJPVTJWV1pISlhhM1JYVm10V05sVldVbE5XVjBwMVVXMUdWVll6VFhoVWJYaFhWMVp3Ums1V1RsTmlSbGt5Vm10YWFtVkdWbkpOU0dSUFZsZG9XRmxzWkc5VlZscHlWbGhrYkdKR1NubFdWelZQWVVaYWRHVkVRbFppUmtwVVZrUktWMlJIVWtWV2JHaHBVakZLU0Zkc1pEUmtNVnBZVW10b2FsSXdXbkJXYlRWRFpHeGFSMWRzVG1oaGVrWllXV3RvVTFVeFpFaFZiRUpoVm5wRk1GbHFSbXRYVjA1R1YyczFWMVpHV2pSWFZtaDNVakZrY2sxWVRsaGlhM0JXV1ZSR1MyRkdiRlZTYm1SVVVteHdXbGxWVlRGVk1VbDVWRmhrVjAxdVVuWlVhMXBTWlVaT2MxcEhSbE5TTWswMVdtdGFWMU5YU2paVmJYaFRUVmhDUjFZeU5YZFVNVkY0VjJ0b1ZXRnJOVlpVVmxwTFVURndXR042VmxOV2ExcGFXVlZWTlZVeFNYZE5WRTVYVWtWYVZGWkhNVTlXTVU1WllVWk9hR1ZyV2s1WFZ6QXhZakpPVjFWWWFHRlNWbkJRVm14U1IwMUdXWGxOVkVKVlRWWnNORll5TURWV1YwVjVWV3hDV21FeGNETmFSVnByVjFkS1IyTkhhR2xYUjJkM1ZtdGFhMlF4VVhsVGJGcE9Wa1p3YjFwWGVFdFZWbFp4VW14YWJGWnRVbHBaTUdoTFZHMUtTR1ZJYUZkV2VrWjJWMVphU21ReVJYcGpSbFpwVW10d1RGZHJVa0pPVms1SFZHNVNUbFl3V2xoVmJYUldaVVpaZUZremFGUk5hM0JYVkZaYVYyRkZNSGxWYkVKYVlrWlZlRnBGV210WFIwNUpVMnMxVTFaR1dscFdWekI0VFVaV1IxTllaR3BUUlhCb1dWUkdWbVZHVm5SbFJuQnNZbFpKTWxSVlVYaFBSVGxGV1hwR1QyVnJSVEZVVlZKT1RrVXhSVkpVUWs5bGJFVXhWRmhzZDFOR1ZsWmtNMFp0VWpGYWIxZFhjRXBsUlRGSVZWaHdUbFl4YTNoVVZWSnFUVVUxV0ZadGFFOVNSVnB6Vkd0a1drMUdiRFpUVkVaT1pXMWplRmRzVWxkaFJuQllWVlJTVDJWdFRqWlVNVkpTWlZad2NWcEhkRTlsYTFwMFZGVlNhMkpWTVZWVFZFcE9Wa1pzTmxkWE1WSk9WVEYwVlcweFVGWXdXVFJXUjNSWFYwZGFRbEJVTVRoUFJHTXhUbnBCTlUxRVRUUk5SRVV3VG5wUk5VMTVjRWhWVlhkeFprUlZlRTlFVVRKWlZHc3lUMWRSTWsxVVVUSk9iVnBvV1RKWmVrNTZXWGhPTWs1cFQwUkZNVTlFVm1sTlZGcHBUV3BTYTFsWFRtcE9SR015VG1wVk5GbHFaR2haVjFacldWUmFiVmxxWkdoYVZGWnFUa1JXT0ZSclZsaG1RVDA5fFVIbzVhRkpYZUc1WFYyUlpWVEpHYkdGNk1XWk5ha1ptVFZSR09FOUVZekZPZWtFMVRVUk5ORTFFUlRCT2VsRTFUWGx3U0ZWVmQzRm1SMWw1VFZSUk1WbDZUVEJhUjFGNVQxZFNhVnB0VlRGT1JFVXdXVlJuZVU1cVRUUk5hbU40VDBSSk1VNXFWVEZOYlZwcVdsUnJlRTFFVVhwWmVsVjNXbXBvYkU1dFJYbE9ha0Y2VDFSSk5VMTZWVEJaYWtJNFZHdFdXR1pCUFQwPXxOb25lfDg3NTcwOTAzODAxNDc0OTMqR1FMKnwzMjM3YjM3ZGNhMGU3YjYyYzYzYzAyZGJkNGU1MjdlNzMyMTExMTNlMmUyMzEyOWM2MDAzYmU1ZTlmZjkwYjAwfE5FV3w="]},"context":{}}
-						]
+							"source_url" => "/search/pins/?q=" . urlencode($search) . "&rs=typed",
+							"data" => json_encode(
+								[
+									"options" => [
+										"applied_unified_filters" => null,
+										"appliedProductFilters" => "---",
+										"article" => null,
+										"auto_correction_disabled" => false,
+										"corpus" => null,
+										"customized_rerank_type" => null,
+										"domains" => null,
+										"dynamicPageSizeExpGroup" => null,
+										"filters" => null,
+										"journey_depth" => null,
+										"page_size" => null,
+										"price_max" => null,
+										"price_min" => null,
+										"query_pin_sigs" => null,
+										"query" => $data["q"],
+										"redux_normalize_feed" => true,
+										"request_params" => null,
+										"rs" => "typed",
+										"scope" => "pins",
+										"selected_one_bar_modules" => null,
+										"source_id" => null,
+										"source_module_id" => null,
+										"source_url" => "/search/pins/?q=" . urlencode($search) . "&rs=typed",
+										"top_pin_id" => null,
+										"top_pin_ids" => null,
+										"bookmarks" => [
+											$data["bookmark"]
+										]
+									],
+									"context" => []
+								],
+								JSON_UNESCAPED_SLASHES
+							)
+						],
+						$cookies,
+						$search
 					);
-			];
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JSON");
+			}
 			
 		}else{
 			
@@ -82,26 +226,44 @@ class pinterest{
 				throw new Exception("Search term is empty!");
 			}
 						
+			// https://ca.pinterest.com/resource/BaseSearchResource/get/?source_url=%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac&data=%7B%22options%22%3A%7B%22applied_unified_filters%22%3Anull%2C%22appliedProductFilters%22%3A%22---%22%2C%22article%22%3Anull%2C%22auto_correction_disabled%22%3Afalse%2C%22corpus%22%3Anull%2C%22customized_rerank_type%22%3Anull%2C%22domains%22%3Anull%2C%22dynamicPageSizeExpGroup%22%3Anull%2C%22filters%22%3Anull%2C%22journey_depth%22%3Anull%2C%22page_size%22%3Anull%2C%22price_max%22%3Anull%2C%22price_min%22%3Anull%2C%22query_pin_sigs%22%3Anull%2C%22query%22%3A%22higurashi%20when%20they%20cry%22%2C%22redux_normalize_feed%22%3Atrue%2C%22request_params%22%3Anull%2C%22rs%22%3A%22ac%22%2C%22scope%22%3A%22pins%22%2C%22selected_one_bar_modules%22%3Anull%2C%22source_id%22%3Anull%2C%22source_module_id%22%3Anull%2C%22source_url%22%3A%22%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac%22%2C%22top_pin_id%22%3Anull%2C%22top_pin_ids%22%3Anull%7D%2C%22context%22%3A%7B%7D%7D&_=1736116313987
+			// source_url=%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac
+			// &data=%7B%22options%22%3A%7B%22applied_unified_filters%22%3Anull%2C%22appliedProductFilters%22%3A%22---%22%2C%22article%22%3Anull%2C%22auto_correction_disabled%22%3Afalse%2C%22corpus%22%3Anull%2C%22customized_rerank_type%22%3Anull%2C%22domains%22%3Anull%2C%22dynamicPageSizeExpGroup%22%3Anull%2C%22filters%22%3Anull%2C%22journey_depth%22%3Anull%2C%22page_size%22%3Anull%2C%22price_max%22%3Anull%2C%22price_min%22%3Anull%2C%22query_pin_sigs%22%3Anull%2C%22query%22%3A%22higurashi%20when%20they%20cry%22%2C%22redux_normalize_feed%22%3Atrue%2C%22request_params%22%3Anull%2C%22rs%22%3A%22ac%22%2C%22scope%22%3A%22pins%22%2C%22selected_one_bar_modules%22%3Anull%2C%22source_id%22%3Anull%2C%22source_module_id%22%3Anull%2C%22source_url%22%3A%22%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac%22%2C%22top_pin_id%22%3Anull%2C%22top_pin_ids%22%3Anull%7D%2C%22context%22%3A%7B%7D%7D
+			// &_=1736116313987
+			
+			$source_url = "/search/pins/?q=" . urlencode($search) . "&rs=" . urlencode($search);
+			
 			$filter = [
-				"source_url" => "/search/pins/?q=" . urlencode($search),
+				"source_url" => $source_url,
 				"rs" => "typed",
 				"data" =>
 					json_encode(
 						[
 							"options" => [
-								"article" => null,
-								"applied_filters" => null,
+								"applied_unified_filters" => null,
 								"appliedProductFilters" => "---",
-								"auto_correction_disabled" => false,
+								"article" => null,
 								"corpus" => null,
 								"customized_rerank_type" => null,
+								"domains" => null,
+								"dynamicPageSizeExpGroup" => null,
 								"filters" => null,
-								"query" => $search,
+								"journey_depth" => null,
+								"page_size" => null,
+								"price_max" => null,
+								"price_min" => null,
 								"query_pin_sigs" => null,
+								"query" => $search,
 								"redux_normalize_feed" => true,
-								"rs" => "typed",
+								"request_params" => null,
+								"rs" => "ac",
 								"scope" => "pins", // pins, boards, videos, 
-								"source_id" => null
+								"selected_one_bar_modules" => null,
+								"source_id" => null,
+								"source_module_id" => null,
+								"source_url" => $source_url,
+								"top_pin_id" => null,
+								"top_pin_ids" => null
 							],
 							"context" => []
 						]
@@ -110,35 +272,99 @@ class pinterest{
 			];
 			
 			$proxy = $this->backend->get_ip();
-		}
+			$cookies = [];
 			
-		try{
-			$json =
-				json_decode(
+			try{
+				$json =
 					$this->get(
 						$proxy,
-						"https://www.pinterest.ca/resource/BaseSearchResource/get/",
-						$filter
-					),
-					true
-				);
+						"https://ca.pinterest.com/resource/BaseSearchResource/get/",
+						$filter,
+						$cookies,
+						null
+					);
 				
-		}catch(Exception $error){
+			}catch(Exception $error){
 				
-			throw new Exception("Failed to fetch JSON");
+				throw new Exception("Failed to fetch JSON");
+			}
 		}
 		
+		$json = json_decode($json, true);
+		
 		if($json === null){
 			
 			throw new Exception("Failed to decode JSON");
 		}
 		
+		if(
+			isset($json["client_context"]["is_bad_bot"]) &&
+			(int)$json["client_context"]["is_bad_bot"] === 1
+		){
+			
+			throw new Exception("Pinterest blocked this instance or request proxy.");
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
 			"image" => []
 		];
 		
+		if(
+			!isset(
+				$json["resource_response"]
+				["status"]
+			)
+		){
+			
+			throw new Exception("Unknown API failure");
+		}
+		
+		if($json["resource_response"]["status"] != "success"){
+			
+			$status = "Got non-OK response: " . $json["resource_response"]["status"];
+			
+			if(
+				isset(
+					$json["resource_response"]["message"]
+				)
+			){
+				
+				$status .= " - " . $json["resource_response"]["message"];
+			}
+			
+			throw new Exception($status);
+		}
+		
+		if(
+			isset(
+				$json["resource_response"]["sensitivity"]
+				["notices"][0]["description"]["text"]
+			)
+		){
+			
+			throw new Exception(
+				"Pinterest returned a notice: " .
+				$json["resource_response"]["sensitivity"]["notices"][0]["description"]["text"]
+			);
+		}
+		
+		// get NPT
+		if(isset($json["resource_response"]["bookmark"])){
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode([
+						"q" => $search,
+						"bookmark" => $json["resource_response"]["bookmark"],
+						"cookies" => $cookies
+					]),
+					"images",
+					$proxy
+				);
+		}
+		
 		foreach(
 			$json
 			["resource_response"]
@@ -150,6 +376,7 @@ class pinterest{
 			switch($item["type"]){
 				
 				case "pin":
+				case "board":
 					
 					/*
 						Handle image object
@@ -206,42 +433,15 @@ class pinterest{
 								"height" => (int)$thumb["height"]
 							]
 						],
-						"url" => "https://www.pinterest.com/pin/" . $item["id"]
+						"url" =>
+							!isset($item["link"]) ?
+							"https://ca.pinterest.com/pin/" . $item["id"] :
+							$item["link"]
 					];
 					break;
-				
-				case "board":
-					if(isset($item["cover_pin"]["image_url"])){
-						
-						$image = [
-							"url" => $item["cover_pin"]["image_url"],
-							"width" => (int)$item["cover_pin"]["size"][0],
-							"height" => (int)$item["cover_pin"]["size"][1]
-						];
-					}elseif(isset($item["image_cover_url_hd"])){
-						/*
-						$image = [
-							"url" => 
-							"width" => null,
-							"height" => null
-						];*/
-					}
-					break;
 			}
 		}
 		
 		return $out;
 	}
-	
-	private function getfullresimage($image, $has_og){
-		
-		$has_og = $has_og ? "1200x" : "originals";
-		
-		return
-			preg_replace(
-				'/https:\/\/i\.pinimg\.com\/[^\/]+\//',
-				"https://i.pinimg.com/" . $has_og . "/",
-				$image
-			);
-	}
 }

scraper/pixabay.php

@@ -0,0 +1,396 @@
+<?php
+
+class pixabay{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("pixabay");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes",
+					"no" => "No"
+				]
+			],
+			"time" => [
+				"display" => "Time posted",
+				"option" => [
+					"any" => "Any time",
+					"1d" => "< 24 hours",
+					"3d" => "< 72 hours",
+					"1w" => "< 7 days",
+					"6m" => "< 6 months",
+					"1y" => "< 12 months"
+				]
+			],
+			"category" => [
+				"display" => "Category",
+				"option" => [
+					"images" => "All images",
+					"photos" => "Photos",
+					"illustrations" => "Illustrations",
+					"vectors" => "Vectors",
+					"gifs" => "GIFs"
+				]
+			],
+			"content_type" => [
+				"display" => "Authenticity",
+				"option" => [
+					"authentic" => "Authentic only",
+					"show_all" => "Authentic + AI slop",
+					"ai" => "AI slop only"
+				]
+			],
+			"order" => [
+				"display" => "Sort by",
+				"option" => [
+					"relevance" => "Most relevant",
+					"latest" => "Latest",
+					"ec" => "Editor's choice",
+					"trending" => "Trending"
+				]
+			],
+			"orientation" => [
+				"display" => "Orientation",
+				"option" => [
+					"any" => "Any orientation",
+					"vertical" => "Vertical",
+					"horizontal" => "Horizontal"
+				]
+			],
+			"color" => [
+				"display" => "Color",
+				"option" => [
+					"any" => "Any color",
+					"transparent" => "Transparent",
+					"grayscale" => "Grayscale",
+					"red" => "Red",
+					"orange" => "Orange",
+					"yellow" => "Yellow",
+					"green" => "Green",
+					"turquoise" => "Turquoise",
+					"blue" => "Blue",
+					"brown" => "Brown",
+					"black" => "Black",
+					"gray" => "Gray",
+					"white" => "White",
+					"pink" => "Pink",
+					"lilac" => "Lilac"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $nsfw = true){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT . " Pixabay",
+			"Accept: application/json",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: {$url}",
+			"Cookie: " . ($nsfw ? "g_rated=off" : "g_rated=1"),
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: no-cors",
+			"Sec-Fetch-Site: same-origin",
+			"x-fetch-bootstrap: 1",
+			"Alt-Used: pixabay.com",
+			"Priority: u=0",
+			"Pragma: no-cache",
+			"Cache-Control: no-cache"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$npt, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$npt = json_decode($npt, true);
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$npt = [
+				"s" => $search,
+				"pagi" => 1,
+				"category" => $get["category"]
+			];
+			
+			// filters
+			if($get["content_type"] != "show_all"){
+				
+				$npt["content_type"] = $get["content_type"];
+			}
+			
+			if($get["order"] != "relevance"){
+				
+				$npt["order"] = $get["order"];
+			}
+			
+			$npt["nsfw"] = $get["nsfw"] == "yes" ? true : false;
+			
+			if($get["orientation"] != "any"){
+				
+				$npt["orientation"] = $get["orientation"];
+			}
+			
+			if($get["color"] != "any"){
+				
+				$npt["colors"] = $get["color"];
+			}
+			
+			if($get["time"] != "any"){
+				
+				$npt["date"] = $get["time"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		// https://pixabay.com/images/search/japan/?pagi=1
+		
+		$npt_pass = $npt;
+		unset($npt_pass["s"]);
+		unset($npt_pass["category"]);
+		unset($npt_pass["nsfw"]);
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://pixabay.com/{$npt["category"]}/search/" . rawurlencode($npt["s"]) . "/",
+					$npt_pass,
+					$npt["nsfw"]
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["page"]["results"])){
+			
+			throw new Exception("Pixabay API did not return a results object");
+		}
+		
+		//print_r($json);
+		
+		foreach($json["page"]["results"] as $image){
+			
+			if(
+				!(
+					$image["mediaType"] == "photo" ||
+					$image["mediaType"] == "animation"
+				)
+			){
+				
+				continue;
+			}
+			
+			// handle images that supply canvaRetouchUrl
+			if(
+				isset($image["canvaRetouchUrl"]) &&
+				$image["canvaRetouchUrl"] !== null
+			){
+				
+				parse_str(
+					parse_url(
+						$image["canvaRetouchUrl"],
+						PHP_URL_QUERY
+					),
+					$base
+				);
+				
+				if(!isset($base["image-url"])){
+					
+					// should not happen
+					continue;
+				}
+				
+				// get extension
+				$base = $base["image-url"];
+				
+				$parsed_base = parse_url($base);
+				
+				$ext =
+					explode(
+						".",
+						$parsed_base["path"]
+					);
+				
+				$ext = $ext[count($ext) - 1];
+				
+				// restructure url
+				$base =
+					explode(
+						"_",
+						$base
+					);
+				
+				unset($base[count($base) - 1]);
+				$base = implode("_", $base);
+				
+				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
+				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
+				
+				$source = [
+					[
+						"url" => "{$base}_1920.{$ext}",
+						"width" => $ratio_2x[0],
+						"height" => $ratio_2x[1]
+					],
+					[
+						"url" => "{$base}_180.{$ext}",
+						"width" => $ratio_1x[0],
+						"height" => $ratio_1x[1]
+					]
+				];
+			}else{
+				
+				// get intended sizes if canva is not set
+				// get 1x
+				$source_1x = null;
+				if(isset($image["sources"]["image_1x"])){ $source_1x = $image["sources"]["image_1x"]; }
+				elseif(isset($image["sources"]["1x"])){ $source_1x = $image["sources"]["1x"]; }
+				
+				if($source_1x === null){ continue; } // should not happen
+				
+				preg_match(
+					'/_([0-9]+)\./',
+					$image["sources"]["1x"],
+					$size_1x
+				);
+				
+				$size_1x = (int)$size_1x[1];
+				
+				// get 2x
+				$source_2x = null;
+				if(isset($image["sources"]["gif_2x"])){ $source_2x = $image["sources"]["gif_2x"]; }
+				elseif(isset($image["sources"]["2x"])){ $source_2x = $image["sources"]["2x"]; }
+				
+				if($source_2x === null){ continue; } // should not happen
+				
+				preg_match(
+					'/_([0-9]+)\./',
+					$source_2x,
+					$size_2x
+				);
+				
+				$size_2x = (int)$size_2x[1];
+				
+				// compute ratios
+				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_1x);
+				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_2x);
+				
+				// handle images that only give normal thumbnails
+				$source = [
+					[
+						"url" => $source_2x,
+						"width" => $ratio_2x[0],
+						"height" => $ratio_2x[1]
+					],
+					[
+						"url" => $source_1x,
+						"width" => $ratio_1x[0],
+						"height" => $ratio_1x[1]
+					]
+				];
+			}
+			
+			$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
+			$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
+			
+			$out["image"][] = [
+				"title" => $image["name"],
+				"source" => $source,
+				"url" => "https://pixabay.com" . $image["href"]
+			];
+		}
+		
+		// add next page
+		if((int)$json["page"]["page"] < (int)$json["page"]["pages"]){
+			
+			$npt["pagi"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($npt),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/purili.php

@@ -0,0 +1,350 @@
+<?php
+
+class purili{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("purili");
+	}
+	
+	public function getfilters($page){
+		
+		switch($page){
+			case "web":
+				return [
+					"nsfw" => [
+						"display" => "NSFW",
+						"option" => [
+							"yes" => "Yes",
+							"maybe" => "Maybe",
+							"no" => "No"
+						]
+					],
+					"language" => [
+						"display" => "Language",
+						"option" => [
+							"any" => "Any language",
+							"en" => "English only",
+							"nl" => "Dutch preferred",
+							"de" => "German preferred",
+							"fr" => "French preferred",
+							"es" => "Spanish preferred",
+							"it" => "Italian preferred",
+							"pt" => "Portuguese preferred"
+						]
+					]
+				];
+				break;
+			
+			case "videos":
+				return [];
+				break;
+		}
+	}
+	
+	private function get($proxy, $url, $get = [], $nsfw = null, $language = null){
+		
+		$curlproc = curl_init();
+		
+		switch($nsfw){
+			
+			case "yes":
+			case null:
+				$get["fv"] = "0";
+				$nsfw = "off";
+				break;
+				
+			case "maybe":
+				$get["fv"] = "1";
+				$nsfw = "moderate";
+				break;
+				
+			case "no":
+				$get["fv"] = "2";
+				$nsfw = "strict";
+				break;
+		}
+		
+		switch($language){
+			
+			case "any":
+			case null:
+				$language_cookie = "";
+				break;
+			
+			default:
+				$language_cookie = "; purili-language={$language}";
+				break;
+		}
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Cookie: purili-safesearch={$nsfw}{$language_cookie}",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=4"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function web($get){
+		
+		if($get["npt"]){
+			
+			[$q, $proxy] = $this->backend->get($get["npt"], "web");
+			$q = json_decode($q, true);
+			
+			$nsfw = $q["nsfw"];
+			unset($q["nsfw"]);
+			
+			$language = $q["language"];
+			unset($q["language"]);
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$q = [
+				"q" => $search,
+				"page" => 1,
+				// added at $this->get()
+				// "fv" => 0
+			];
+			
+			$nsfw = $get["nsfw"];
+			$language = $get["language"];
+		}
+		
+		// https://puri.li/api/search?q=4get&page=1&fv=0
+		try{
+			$json = $this->get(
+				$proxy,
+				"https://puri.li/api/search",
+				$q,
+				$nsfw,
+				$language
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to hit API");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Failed to access results array");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		foreach($json["results"] as $result){
+			
+			$out["web"][] = [
+				"title" => $result["title"],
+				"description" => $result["description"],
+				"url" => $result["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		// get next page
+		if($json["hasNext"]){
+			
+			$q["page"]++;
+			$q["nsfw"] = $nsfw;
+			$q["language"] = $language;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($q),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	public function video($get){
+		
+		
+		if($get["npt"]){
+			
+			[$q, $proxy] = $this->backend->get($get["npt"], "videos");
+			$q = json_decode($q, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$q = [
+				"q" => $search,
+				"offset" => 0,
+				"limit" => 12
+				// added at $this->get()
+				// "fv" => 0
+			];
+		}
+		
+		// https://puri.li/api/videos/search?q=higurashi&limit=12&fv=1
+		// hidden &offset parameter only accessible through the API
+		try{
+			$json = $this->get(
+				$proxy,
+				"https://puri.li/api/videos/search",
+				$q,
+				null,
+				null
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to hit API");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Failed to access results array");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		foreach($json["results"] as $result){
+			
+			if(isset($result["thumbnail"])){
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $result["thumbnail"]
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
+			}
+			
+			$out["video"][] = [
+				"title" => $result["title"],
+				"description" =>
+					str_replace(
+						"\n", " ",
+						mb_substr(
+							$result["description"], 0, 255, "utf-8"
+						)
+					),
+				"author" => [
+					"name" => null,
+					"url" => isset($result["channel_id"]) ? "https://www.youtube.com/channel/" . $result["channel_id"] : null,
+					"avatar" => null
+				],
+				"date" => strtotime($result["upload_date"]),
+				"duration" => $result["duration"],
+				"views" => $result["view_count"],
+				"thumb" => $thumb,
+				"url" => $result["url"]
+			];
+		}
+		
+		// get next page
+		if($json["limit"] + $json["offset"] < $json["total"]){
+			
+			$q["offset"] = $q["offset"] + $json["limit"];
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($q),
+					"videos",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+}

scraper/qwant.php

@@ -1,937 +0,0 @@
-<?php
-
-class qwant{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("qwant");
-	}
-	
-	public function getfilters($page){
-		
-		$base = [
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes",
-					"maybe" => "Maybe",
-					"no" => "No"
-				]
-			],
-			"country" => [
-				"display" => "Country",
-				"option" => [
-					"en_US" => "United States",
-					"fr_FR" => "France",
-					"en_GB" => "Great Britain",
-					"de_DE" => "Germany",
-					"it_IT" => "Italy",
-					"es_AR" => "Argentina",
-					"en_AU" => "Australia",
-					"es_ES" => "Spain (es)",
-					"ca_ES" => "Spain (ca)",
-					"cs_CZ" => "Czech Republic",
-					"ro_RO" => "Romania",
-					"el_GR" => "Greece",
-					"zh_CN" => "China",
-					"zh_HK" => "Hong Kong",
-					"en_NZ" => "New Zealand",
-					"fr_FR" => "France",
-					"th_TH" => "Thailand",
-					"ko_KR" => "South Korea",
-					"sv_SE" => "Sweden",
-					"nb_NO" => "Norway",
-					"da_DK" => "Denmark",
-					"hu_HU" => "Hungary",
-					"et_EE" => "Estonia",
-					"es_MX" => "Mexico",
-					"es_CL" => "Chile",
-					"en_CA" => "Canada (en)",
-					"fr_CA" => "Canada (fr)",
-					"en_MY" => "Malaysia",
-					"bg_BG" => "Bulgaria",
-					"fi_FI" => "Finland",
-					"pl_PL" => "Poland",
-					"nl_NL" => "Netherlands",
-					"pt_PT" => "Portugal",
-					"de_CH" => "Switzerland (de)",
-					"fr_CH" => "Switzerland (fr)",
-					"it_CH" => "Switzerland (it)",
-					"de_AT" => "Austria",
-					"fr_BE" => "Belgium (fr)",
-					"nl_BE" => "Belgium (nl)",
-					"en_IE" => "Ireland",
-					"he_IL" => "Israel"
-				]
-			]
-		];
-		
-		switch($page){
-			
-			case "web":
-				$base = array_merge(
-					$base,
-					[
-						"time" => [
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"day" => "Past 24 hours",
-								"week" => "Past week",
-								"month" => "Past month"
-							]
-						],
-						"extendedsearch" => [
-							// no display, wont show in interface
-							"option" => [
-								"yes" => "Yes",
-								"no" => "No"
-							]
-						]
-					]
-				);
-				break;
-			
-			case "images":
-				$base = array_merge(
-					$base,
-					[
-						"time" => [
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"day" => "Past 24 hours",
-								"week" => "Past week",
-								"month" => "Past month"
-							]
-						],
-						"size" => [
-							"display" => "Size",
-							"option" => [
-								"any" => "Any size",
-								"large" => "Large",
-								"medium" => "Medium",
-								"small" => "Small"
-							]
-						],
-						"color" => [
-							"display" => "Color",
-							"option" => [
-								"any" => "Any color",
-								"coloronly" => "Color only",
-								"monochrome" => "Monochrome",
-								"black" => "Black",
-								"brown" => "Brown",
-								"gray" => "Gray",
-								"white" => "White",
-								"yellow" => "Yellow",
-								"orange" => "Orange",
-								"red" => "Red",
-								"pink" => "Pink",
-								"purple" => "Purple",
-								"blue" => "Blue",
-								"teal" => "Teal",
-								"green" => "Green"
-							]
-						],
-						"imagetype" => [
-							"display" => "Type",
-							"option" => [
-								"any" => "Any type",
-								"animatedgif" => "Animated GIF",
-								"photo" => "Photograph",
-								"transparent" => "Transparent"
-							]
-						],
-						"license" => [
-							"display" => "License",
-							"option" => [
-								"any" => "Any license",
-								"share" => "Non-commercial reproduction and sharing",
-								"sharecommercially" => "Reproduction and sharing",
-								"modify" => "Non-commercial reproduction, sharing and modification",
-								"modifycommercially" => "Reproduction, sharing and modification",
-								"public" => "Public domain"
-							]
-						]
-					]
-				);
-				break;
-			
-			case "videos":
-				$base = array_merge(
-					$base,
-					[
-						"order" => [
-							"display" => "Order by",
-							"option" => [
-								"relevance" => "Relevance",
-								"views" => "Views",
-								"date" => "Most recent",
-							]
-						],
-						"source" => [
-							"display" => "Source",
-							"option" => [
-								"any" => "Any source",
-								"youtube" => "YouTube",
-								"dailymotion" => "Dailymotion",
-							]
-						]
-					]
-				);
-				break;
-			
-			case "news":
-				$base = array_merge(
-					$base,
-					[
-						"time" => [
-							"display" => "Time posted",
-							"option" => [
-								"any" => "Any time",
-								"hour" => "Less than 1 hour ago",
-								"day" => "Past 24 hours",
-								"week" => "Past week",
-								"month" => "Past month"
-							]
-						],
-						"order" => [
-							"display" => "Order by",
-							"option" => [
-								"relevance" => "Relevance",
-								"date" => "Most recent"
-							]
-						]
-					]
-				);
-				break;
-		}
-		
-		return $base;
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$headers = [
-			"User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Origin: https://www.qwant.com",
-			"Referer: https://www.qwant.com/",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"TE: trailers"
-		];
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		// Bypass HTTP/2 check
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			// get next page data
-			[$params, $proxy] = $this->backend->get($get["npt"], "web");
-			
-			$params = json_decode($params, true);
-			
-		}else{
-			
-			// get _GET data instead
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			if(strlen($search) > 2048){
-				
-				throw new Exception("Search term is too long!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$params = [
-				"q" => $search,
-				"freshness" => $get["time"],
-				"count" => 10,
-				"locale" => $get["country"],
-				"offset" => 0,
-				"device" => "desktop",
-				"tgp" => 3,
-				"safesearch" => 0,
-				"displayed" => "true"
-			];
-			
-			switch($get["nsfw"]){
-				
-				case "yes": $params["safesearch"] = 0; break;
-				case "maybe": $params["safesearch"] = 1; break;
-				case "no": $params["safesearch"] = 2; break;
-			}
-		}
-		/*
-		$handle = fopen("scraper/qwant_web.json", "r");
-		$json = fread($handle, filesize("scraper/qwant_web.json"));
-		fclose($handle);*/
-		
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://fdn.qwant.com/v3/search/web",
-					$params
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Could not fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === NULL){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(isset($json["data"]["message"][0])){
-			
-			throw new Exception("Server returned an error:\n" . $json["data"]["message"][0]);
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		if(
-			$json["status"] != "success" &&
-			$json["data"]["error_code"] === 5
-		){
-			
-			// no results
-			return $out;
-		}
-		
-		$this->detect_errors($json);
-		
-		if(!isset($json["data"]["result"]["items"]["mainline"])){
-			
-			throw new Exception("Server did not return a result object");
-		}
-		
-		// data is OK, parse
-		
-		// get instant answer
-		if(
-			$get["extendedsearch"] == "yes" &&
-			isset($json["data"]["result"]["items"]["sidebar"][0]["endpoint"])
-		){
-			
-			try{
-				$answer =
-					$this->get(
-						$proxy,
-						"https://api.qwant.com/v3" .
-						$json["data"]["result"]["items"]["sidebar"][0]["endpoint"],
-						[]
-					);
-				
-				$answer = json_decode($answer, true);
-				
-				if(
-					$answer === null ||
-					$answer["status"] != "success" ||
-					$answer["data"]["result"] === null
-				){
-					
-					throw new Exception();
-				}
-				
-				// parse answer
-				$out["answer"][] = [
-					"title" => $answer["data"]["result"]["title"],
-					"description" => [
-						[
-							"type" => "text",
-							"value" => $this->trimdots($answer["data"]["result"]["description"])
-						]
-					],
-					"url" => $answer["data"]["result"]["url"],
-					"thumb" =>
-						$answer["data"]["result"]["thumbnail"]["landscape"] == null ?
-						null :
-						$this->unshitimage(
-							$answer["data"]["result"]["thumbnail"]["landscape"],
-							false
-						),
-					"table" => [],
-					"sublink" => []
-				];
-				
-			}catch(Exception $error){
-				
-				// do nothing in case of failure
-			}
-			
-		}
-		
-		// get word correction
-		if(isset($json["data"]["query"]["queryContext"]["alteredQuery"])){
-			
-			$out["spelling"] = [
-				"type" => "including",
-				"using" => $json["data"]["query"]["queryContext"]["alteredQuery"],
-				"correction" => $json["data"]["query"]["queryContext"]["alterationOverrideQuery"]
-			];
-		}
-		
-		// check for next page
-		if($json["data"]["result"]["lastPage"] === false){
-			
-			$params["offset"] = $params["offset"] + 10;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($params),
-					"web",
-					$proxy
-				);
-		}
-		
-		// parse results
-		foreach($json["data"]["result"]["items"]["mainline"] as $item){
-			
-			switch($item["type"]){ // ignores ads
-				
-				case "web":
-					
-					$first_iteration = true;
-					foreach($item["items"] as $result){
-						
-						if(isset($result["thumbnailUrl"])){
-							
-							$thumb = [
-								"url" => $this->unshitimage($result["thumbnailUrl"]),
-								"ratio" => "16:9"
-							];
-						}else{
-							
-							$thumb = [
-								"url" => null,
-								"ratio" => null
-							];
-						}
-						
-						$sublinks = [];
-						if(isset($result["links"])){
-							
-							foreach($result["links"] as $link){
-								
-								$sublinks[] = [
-									"title" => $this->trimdots($link["title"]),
-									"date" => null,
-									"description" => isset($link["desc"]) ? $this->trimdots($link["desc"]) : null,
-									"url" => $link["url"]
-								];
-							}
-						}
-						
-						// detect gibberish results
-						if(
-							$first_iteration &&
-							!isset($result["urlPingSuffix"])
-						){
-							
-							throw new Exception("Qwant returned gibberish results");
-						}
-						
-						$out["web"][] = [
-							"title" => $this->trimdots($result["title"]),
-							"description" => $this->trimdots($result["desc"]),
-							"url" => $result["url"],
-							"date" => null,
-							"type" => "web",
-							"thumb" => $thumb,
-							"sublink" => $sublinks,
-							"table" => []
-						];
-						
-						$first_iteration = false;
-					}
-					break;
-				
-				case "images":
-					foreach($item["items"] as $image){
-						
-						$out["image"][] = [
-							"title" => $image["title"],
-							"source" => [
-								[
-									"url" => $image["media"],
-									"width" => (int)$image["width"],
-									"height" => (int)$image["height"]
-								],
-								[
-									"url" => $this->unshitimage($image["thumbnail"]),
-									"width" => $image["thumb_width"],
-									"height" => $image["thumb_height"]
-								]
-							],
-							"url" => $image["url"]
-						];
-					}
-					break;
-				
-				case "videos":
-					foreach($item["items"] as $video){
-						
-						$out["video"][] = [
-							"title" => $video["title"],
-							"description" => null,
-							"date" => (int)$video["date"],
-							"duration" => $video["duration"] === null ? null : $video["duration"] / 1000,
-							"views" => null,
-							"thumb" =>
-								$video["thumbnail"] === null ?
-								[
-									"url" => null,
-									"ratio" => null,
-								] :
-								[
-									"url" => $this->unshitimage($video["thumbnail"]),
-									"ratio" => "16:9",
-								],
-							"url" => $video["url"]
-						];
-					}
-					break;
-				
-				case "related_searches":
-					foreach($item["items"] as $related){
-						
-						$out["related"][] = $related["text"];
-					}
-					break;
-			}
-		}
-		
-		return $out;
-	}
-	
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$params, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$params = json_decode($params, true);
-		}else{
-			
-			$search = $get["s"];
-			
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$params = [
-				"t" => "images",
-				"q" => $search,
-				"count" => 125,
-				"locale" => $get["country"],
-				"offset" => 0, // increment by 125
-				"device" => "desktop",
-				"tgp" => 3
-			];
-			
-			if($get["time"] != "any"){
-				
-				$params["freshness"] = $get["time"];
-			}
-			
-			foreach(["size", "color", "imagetype", "license"] as $p){
-				
-				if($get[$p] != "any"){
-					
-					$params[$p] = $get[$p];
-				}
-			}
-			
-			switch($get["nsfw"]){
-				
-				case "yes": $params["safesearch"] = 0; break;
-				case "maybe": $params["safesearch"] = 1; break;
-				case "no": $params["safesearch"] = 2; break;
-			}
-		}
-		
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://api.qwant.com/v3/search/images",
-				$params,
-			);
-		}catch(Exception $err){
-			
-			throw new Exception("Failed to get JSON");
-		}
-		
-		/*
-		$handle = fopen("scraper/yandex.json", "r");
-		$json = fread($handle, filesize("scraper/yandex.json"));
-		fclose($handle);*/
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$this->detect_errors($json);
-		
-		if(isset($json["data"]["result"]["items"]["mainline"])){
-			
-			throw new Exception("Qwant returned gibberish results");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if($json["data"]["result"]["lastPage"] === false){
-			
-			$params["offset"] = $params["offset"] + 125;
-			
-			$out["npt"] = $this->backend->store(
-				json_encode($params),
-				"images",
-				$proxy
-			);
-		}
-		
-		foreach($json["data"]["result"]["items"] as $image){
-			
-			$out["image"][] = [
-				"title" => $this->trimdots($image["title"]),
-				"source" => [
-					[
-						"url" => $image["media"],
-						"width" => $image["width"],
-						"height" => $image["height"]
-					],
-					[
-						"url" => $this->unshitimage($image["thumbnail"]),
-						"width" => $image["thumb_width"],
-						"height" => $image["thumb_height"]
-					]
-				],
-				"url" => $image["url"]
-			];
-		}
-		
-		return $out;
-	}
-	
-	public function video($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$params = [
-			"t" => "videos",
-			"q" => $search,
-			"count" => 50,
-			"locale" => $get["country"],
-			"offset" => 0, // dont implement pagination
-			"device" => "desktop",
-			"tgp" => 3
-		];
-		
-		switch($get["nsfw"]){
-			
-			case "yes": $params["safesearch"] = 0; break;
-			case "maybe": $params["safesearch"] = 1; break;
-			case "no": $params["safesearch"] = 2; break;
-		}
-		
-		try{
-			$json =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://api.qwant.com/v3/search/videos",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Could not fetch JSON");
-		}
-		
-		/*
-		$handle = fopen("scraper/yandex-video.json", "r");
-		$json = fread($handle, filesize("scraper/yandex-video.json"));
-		fclose($handle);
-		*/
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Could not parse JSON");
-		}
-		
-		$this->detect_errors($json);
-		
-		if(isset($json["data"]["result"]["items"]["mainline"])){
-			
-			throw new Exception("Qwant returned gibberish results");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"video" => [],
-			"author" => [],
-			"livestream" => [],
-			"playlist" => [],
-			"reel" => []
-		];
-		
-		foreach($json["data"]["result"]["items"] as $video){
-			
-			if(empty($video["thumbnail"])){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" => $this->unshitimage($video["thumbnail"], false),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$duration = (int)$video["duration"];
-			
-			$out["video"][] = [
-				"title" => $video["title"],
-				"description" => $this->limitstrlen($video["desc"]),
-				"author" => [
-					"name" => $video["channel"],
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => (int)$video["date"],
-				"duration" => $duration === 0 ? null : $duration,
-				"views" => null,
-				"thumb" => $thumb,
-				"url" => preg_replace("/\?syndication=.+/", "", $video["url"])
-			];
-		}
-		
-		return $out;
-	}
-	
-	public function news($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$params = [
-			"t" => "news",
-			"q" => $search,
-			"count" => 50,
-			"locale" => $get["country"],
-			"offset" => 0, // dont implement pagination
-			"device" => "desktop",
-			"tgp" => 3
-		];
-		
-		switch($get["nsfw"]){
-			
-			case "yes": $params["safesearch"] = 0; break;
-			case "maybe": $params["safesearch"] = 1; break;
-			case "no": $params["safesearch"] = 2; break;
-		}
-		
-		try{
-			$json =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://api.qwant.com/v3/search/news",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Could not fetch JSON");
-		}
-		
-		/*
-		$handle = fopen("scraper/yandex-video.json", "r");
-		$json = fread($handle, filesize("scraper/yandex-video.json"));
-		fclose($handle);
-		*/
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Could not parse JSON");
-		}
-		
-		$this->detect_errors($json);
-		
-		if(isset($json["data"]["result"]["items"]["mainline"])){
-			
-			throw new Exception("Qwant returned gibberish results");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		foreach($json["data"]["result"]["items"] as $news){
-			
-			if(empty($news["media"][0]["pict_big"]["url"])){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" => $this->unshitimage($news["media"][0]["pict_big"]["url"], false),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$out["news"][] = [
-				"title" => $news["title"],
-				"author" => $news["press_name"],
-				"description" => $this->trimdots($news["desc"]),
-				"date" => (int)$news["date"],
-				"thumb" => $thumb,
-				"url" => $news["url"]
-			];
-		}
-		
-		return $out;
-	}
-	
-	private function detect_errors($json){
-		
-		if(
-			isset($json["status"]) &&
-			$json["status"] == "error"
-		){
-			
-			if(isset($json["data"]["error_data"]["captchaUrl"])){
-				
-				throw new Exception("Qwant returned a captcha");
-			}elseif(isset($json["data"]["error_data"]["error_code"])){
-				
-				throw new Exception(
-					"Qwant returned an API error: " .
-					$json["data"]["error_data"]["error_code"]
-				);
-			}
-			
-			throw new Exception("Qwant returned an API error");
-		}
-	}
-	
-	private function limitstrlen($text){
-		
-		return explode("\n", wordwrap($text, 300, "\n"))[0];
-	}
-	
-	private function trimdots($text){
-		
-		return trim($text, ". ");
-	}
-	
-	private function unshitimage($url, $is_bing = true){
-		
-		// https://s1.qwant.com/thumbr/0x0/8/d/f6de4deb2c2b12f55d8bdcaae576f9f62fd58a05ec0feeac117b354d1bf5c2/th.jpg?u=https%3A%2F%2Fwww.bing.com%2Fth%3Fid%3DOIP.vvDWsagzxjoKKP_rOqhwrQAAAA%26w%3D160%26h%3D160%26c%3D7%26pid%3D5.1&q=0&b=1&p=0&a=0
-		parse_str(parse_url($url)["query"], $parts);
-		
-		if($is_bing){
-			$parse = parse_url($parts["u"]);
-			parse_str($parse["query"], $parts);
-			
-			return "https://" . $parse["host"] . "/th?id=" . urlencode($parts["id"]);
-		}
-		
-		return $parts["u"];
-	}
-}

scraper/sc.php

@@ -135,6 +135,8 @@ class sc{
 			
 			$type = $get["type"];
 			$proxy = $this->backend->get_ip();
+			
+			// token is not tied to an IP address
 			$token = $this->get_token($proxy);
 			
 			switch($type){
@@ -149,7 +151,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -164,7 +166,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -179,7 +181,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -194,7 +196,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -209,7 +211,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -225,7 +227,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1713542117,
+						"app_version" => 1769170554,
 						"app_locale" => "en"
 					];
 					break;
@@ -247,17 +249,34 @@ class sc{
 		fclose($handle);
 		*/
 		
+		$this->fuckhtml->load($json);
+		
+		// detect for cloudfront error
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"title"
+			);
+		
+		if(
+			count($title) !== 0 &&
+			str_contains($title[0]["innerHTML"], "The request could not be satisfied")
+		){
+			
+			throw new Exception("Ratelimited by Cloudfront, add some working proxies");
+		}
+		
 		$json = json_decode($json, true);
 		
 		if($json === null){
 			
 			if($last_attempt === true){
 				
-				throw new Exception("Fetched an invalid token (please report!!)");
+				throw new Exception("Fetched an invalid token");
 			}
 			
 			// token might've expired, get a new one and re-try search
-			$this->get_token($proxy);
+			$token = $this->get_token($proxy, true);
 			return $this->music($get, true);
 		}
 		
@@ -378,7 +397,12 @@ class sc{
 					break;
 				
 				case "track":
-					if(stripos($item["monetization_model"], "TIER") === false){
+					$stream = [
+						"endpoint" => null,
+						"url" => null
+					];
+					
+					/*if(stripos($item["monetization_model"], "TIER") === false){
 						
 						$stream = [
 							"endpoint" => "sc",
@@ -394,7 +418,7 @@ class sc{
 							"endpoint" => null,
 							"url" => null
 						];
-					}
+					}*/
 					
 					// parse track
 					$out["song"][] = [
@@ -422,16 +446,19 @@ class sc{
 		return $out;
 	}
 	
-	public function get_token($proxy){
+	public function get_token($proxy, $force_refresh = false){
 		
 		$token = apcu_fetch("sc_token");
 		
-		if($token !== false){
+		if(
+			$token !== false &&
+			$force_refresh === false
+		){
 			
 			return $token;
 		}
 		
-		// search through all javascript components on the main page
+		// get token from main page
 		try{
 			$html =
 				$this->get(
@@ -447,49 +474,44 @@ class sc{
 		
 		$this->fuckhtml->load($html);
 		
-		$scripts =
-			$this->fuckhtml
-			->getElementsByTagName(
-				"script"
+		$json =
+			preg_split(
+				'/window.__sc_hydration ?= ?/',
+				$html
 			);
 		
-		foreach($scripts as $script){
+		if(!isset($json[1])){
+			
+			throw new Exception("Failed to find JSON containing token");
+		}
+		
+		$json =
+			json_decode(
+				$this->fuckhtml
+				->extract_json(
+					$json[1]
+				),
+				true
+			);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON containing token");
+		}
+		
+		foreach($json as $item){
 			
 			if(
-				!isset($script["attributes"]["src"]) ||
-				strpos($script["attributes"]["src"], "sndcdn.com") === false
+				isset($item["hydratable"]) &&
+				isset($item["data"]["id"])
 			){
 				
-				continue;
-			}
-			
-			try{
-				$js =
-					$this->get(
-						$proxy,
-						$script["attributes"]["src"],
-						[]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch search token");
-			}
-			
-			preg_match(
-				'/client_id=([^"]+)/',
-				$js,
-				$token
-			);
-			
-			if(isset($token[1])){
-				
-				apcu_store("sc_token", $token[1]);
-				return $token[1];
-				break;
+				apcu_store("sc_token", $item["data"]["id"]);
+				return $item["data"]["id"];
 			}
 		}
 		
-		throw new Exception("Did not find a Soundcloud token in the Javascript blobs");
+		throw new Exception("Failed to find a valid token");
 	}
 	
 	private function limitstrlen($text){

scraper/sepiasearch.php

@@ -0,0 +1,541 @@
+<?php
+
+class sepiasearch{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("sepiasearch");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"nsfw" => [
+				"display" => "NSFW",
+				"option" => [
+					"yes" => "Yes", // &sensitiveContent=both
+					"no" => "No" // &sensitiveContent=false
+				]
+			],
+			"language" => [
+				"display" => "Language", // &language=
+				"option" => [
+					"any" => "Any language",
+					"en" => "English",
+					"fr" => "Français",
+					"ar" => "العربية",
+					"ca" => "Català",
+					"cs" => "Čeština",
+					"de" => "Deutsch",
+					"el" => "ελληνικά",
+					"eo" => "Esperanto",
+					"es" => "Español",
+					"eu" => "Euskara",
+					"fa" => "فارسی",
+					"fi" => "Suomi",
+					"gd" => "Gàidhlig",
+					"gl" => "Galego",
+					"hr" => "Hrvatski",
+					"hu" => "Magyar",
+					"is" => "Íslenska",
+					"it" => "Italiano",
+					"ja" => "日本語",
+					"kab" => "Taqbaylit",
+					"nl" => "Nederlands",
+					"no" => "Norsk",
+					"oc" => "Occitan",
+					"pl" => "Polski",
+					"pt" => "Português (Brasil)",
+					"pt-PT" => "Português (Portugal)",
+					"ru" => "Pусский",
+					"sk" => "Slovenčina",
+					"sq" => "Shqip",
+					"sv" => "Svenska",
+					"th" => "ไทย",
+					"tok" => "Toki Pona",
+					"tr" => "Türkçe",
+					"uk" => "украї́нська мо́ва",
+					"vi" => "Tiếng Việt",
+					"zh-Hans" => "简体中文（中国）",
+					"zh-Hant" => "繁體中文（台灣）"
+				]
+			],
+			"type" => [
+				"display" => "Result type", // i handle this
+				"option" => [
+					"videos" => "Videos",
+					"playlists" => "Playlists",
+					"channels" => "Channels"
+				]
+			],
+			"sort" => [
+				"display" => "Sort by",
+				"option" => [
+					"best" => "Best match", // no filter
+					"-publishedAt" => "Newest", // sort=-publishedAt
+					"publishedAt" => "Oldest" // sort=publishedAt
+				]
+			],
+			"newer" => [ // &startDate=2025-07-26T04:00:00.000Z
+				"display" => "Newer than",
+				"option" => "_DATE"
+			],
+			"duration" => [
+				"display" => "Duration",
+				"option" => [
+					"any" => "Any duration",
+					"short" => "Short (0-4mins)", // &durationRange=short
+					"medium" => "Medium (4-10 mins)",
+					"long" => "Long (10+ mins)",
+				]
+			],
+			"category" => [
+				"display" => "Category", // &categoryOneOf[]=
+				"option" => [
+					"any" => "Any category",
+					"1" => "Music",
+					"2" => "Films",
+					"3" => "Vehicles",
+					"4" => "Art",
+					"5" => "Sports",
+					"6" => "Travels",
+					"7" => "Gaming",
+					"8" => "People",
+					"9" => "Comedy",
+					"10" => "Entertainment",
+					"11" => "News & Politics",
+					"12" => "How To",
+					"13" => "Education",
+					"14" => "Activism",
+					"15" => "Science & Technology",
+					"16" => "Animals",
+					"17" => "Kids",
+					"18" => "Food"
+				]
+			],
+			"display" => [
+				"display" => "Display",
+				"option" => [
+					"any" => "Everything",
+					"true" => "Live videos", // &isLive=true
+					"false" => "VODs" // &isLive=false
+				]
+			],
+			"license" => [
+				"display" => "License", // &license=
+				"option" => [
+					"any" => "Any license",
+					"1" => "Attribution",
+					"2" => "Attribution - Share Alike",
+					"3" => "Attribution - No Derivatives",
+					"4" => "Attribution - Non Commercial",
+					"5" => "Attribution - Non Commercial - Share Alike",
+					"6" => "Attribution - Non Commercial - No Derivatives",
+					"7" => "Public Domain Dedication"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		curl_setopt(
+			$curlproc,
+			CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: application/json, text/plain, */*",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Referer: https://sepiasearch.org/search",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=0",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function video($get){
+		
+		if($get["npt"]){
+			
+			[$npt, $proxy] =
+				$this->backend
+				->get(
+					$get["npt"],
+					"videos"
+				);
+			
+			$npt = json_decode($npt, true);
+			$type = $npt["type"];
+			$npt = $npt["npt"];
+		}else{
+			
+			$proxy = $this->backend->get_ip();
+						
+			$npt = [
+				"search" => $get["s"],
+				"start" => 0,
+				"count" => 20
+			];
+			
+			if($get["type"] == "videos"){
+				
+				//
+				// Parse video filters
+				//
+				switch($get["nsfw"]){
+					
+					case "yes": $npt["nsfw"] = "both"; break;
+					case "no": $npt["nsfw"] = "false"; break;
+				}
+				
+				$npt["boostLanguages[]"] = "en";
+				if($get["language"] != "any"){
+					
+					$npt["languageOneOf[]"] = $get["language"];
+				}
+				
+				if($get["sort"] != "best"){
+					
+					$npt["sort"] = $get["sort"];
+				}
+				
+				if($get["newer"] !== false){
+					
+					$date = new DateTime("@{$get["newer"]}");
+					$date->setTimezone(new DateTimeZone("UTC"));
+					$formatted = $date->format("Y-m-d\TH:i:s.000\Z");
+					
+					$npt["startDate"] = $formatted;
+				}
+				
+				switch($get["duration"]){
+					
+					case "short":
+						$npt["durationMax"] = 240;
+						break;
+					
+					case "medium":
+						$npt["durationMin"] = 240;
+						$npt["durationMax"] = 600;
+						break;
+					
+					case "long":
+						$npt["durationMin"] = 600;
+						break;
+				}
+				
+				if($get["category"] != "any"){
+					
+					$npt["categoryOneOf[]"] = $get["category"];
+				}
+				
+				if($get["display"] != "any"){
+					
+					$npt["isLive"] = $get["display"];
+				}
+				
+				if($get["license"] != "any"){
+					
+					// typo in license, lol
+					$npt["licenceOneOf[]"] = $get["license"];
+				}
+			}
+			
+			$type = $get["type"];
+		}
+		
+		switch($type){
+			
+			case "videos":
+				$url = "https://sepiasearch.org/api/v1/search/videos";
+				break;
+			
+			case "channels":
+				$url = "https://sepiasearch.org/api/v1/search/video-channels";
+				break;
+			
+			case "playlists":
+				$url = "https://sepiasearch.org/api/v1/search/video-playlists";
+				break;
+		}
+		
+		//$json = file_get_contents("scraper/sepia.json");
+		try{
+			
+			$json =
+				$this->get(
+					$proxy,
+					$url,
+					$npt
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to parse JSON");
+		}
+		
+		if(isset($json["errors"])){
+			
+			$msg = [];
+			foreach($json["errors"] as $error){
+				
+				if(isset($error["msg"])){
+					
+					$msg[] = $error["msg"];
+				}
+			}
+			
+			throw new Exception("Sepia Search returned error(s): " . implode(", ", $msg));
+		}
+		
+		if(!isset($json["data"])){
+			
+			throw new Exception("Sepia Search did not return a data object");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		
+		switch($get["type"]){
+			
+			case "videos":
+				foreach($json["data"] as $video){
+					
+					if(count($video["account"]["avatars"]) !== 0){
+						
+						$avatar =
+							$video["account"]["avatars"][count($video["account"]["avatars"]) - 1]["url"];
+					}else{
+						
+						$avatar = null;
+					}
+					
+					if($video["thumbnailUrl"] === null){
+						
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}else{
+						
+						$thumb  = [
+							"ratio" => "16:9",
+							"url" => $video["thumbnailUrl"]
+						];
+					}
+					
+					if($video["isLive"]){
+						
+						$append = "livestream";
+					}else{
+						
+						$append = "video";
+					}
+					
+					$out[$append][] = [
+						"title" => $video["name"],
+						"description" =>
+							$this->limitstrlen(
+								$this->titledots(
+									$video["description"]
+								)
+							),
+						"author" => [
+							"name" => $video["account"]["displayName"] . " ({$video["account"]["name"]})",
+							"url" => $video["account"]["url"],
+							"avatar" => $avatar
+						],
+						"date" => strtotime($video["publishedAt"]),
+						"duration" => $video["isLive"] ? "_LIVE" : $video["duration"],
+						"views" => $video["views"],
+						"thumb" => $thumb,
+						"url" => $video["url"]
+					];
+				}
+				break;
+			
+			case "playlists":
+				foreach($json["data"] as $playlist){
+					
+					if(count($playlist["ownerAccount"]["avatars"]) !== 0){
+						
+						$avatar =
+							$playlist["ownerAccount"]["avatars"][count($playlist["ownerAccount"]["avatars"]) - 1]["url"];
+					}else{
+						
+						$avatar = null;
+					}
+					
+					if($playlist["thumbnailUrl"] === null){
+						
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}else{
+						
+						$thumb  = [
+							"ratio" => "16:9",
+							"url" => $playlist["thumbnailUrl"]
+						];
+					}
+					
+					$out["playlist"][] = [
+						"title" => $playlist["displayName"],
+						"description" =>
+							$this->limitstrlen(
+								$this->titledots(
+									$playlist["description"]
+								)
+							),
+						"author" => [
+							"name" => $playlist["ownerAccount"]["displayName"] . " ({$playlist["ownerAccount"]["name"]})",
+							"url" => $playlist["ownerAccount"]["url"],
+							"avatar" => $avatar
+						],
+						"date" => strtotime($playlist["createdAt"]),
+						"duration" => $playlist["videosLength"],
+						"views" => null,
+						"thumb" => $thumb,
+						"url" => $playlist["url"]
+					];
+				}
+				break;
+			
+			case "channels":
+				foreach($json["data"] as $channel){
+					
+					if(count($channel["avatars"]) !== 0){
+						
+						$thumb = [
+							"ratio" => "1:1",
+							"url" => $channel["avatars"][count($channel["avatars"]) - 1]["url"]
+						];
+					}else{
+						
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}
+					
+					$out["author"][] = [
+						"title" => $channel["displayName"] . " ({$channel["name"]})",
+						"followers" => $channel["followersCount"],
+						"description" =>
+							$channel["videosCount"] . " videos. " .
+							$this->limitstrlen(
+								$this->titledots(
+									$channel["description"]
+								)
+							),
+						"thumb" => $thumb,
+						"url" => $channel["url"]
+					];
+				}
+				break;
+		}
+		
+		// get next page
+		if($json["total"] - 20 > $npt["start"]){
+			
+			$npt["start"] += 20;
+			
+			$npt = [
+				"type" => $get["type"],
+				"npt" => $npt
+			];
+			
+			$out["npt"] =
+				$this->backend
+				->store(
+					json_encode($npt),
+					"videos",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function titledots($title){
+		
+		$substr = substr($title, -3);
+		
+		if(
+			$substr == "..." ||
+			$substr == "…"
+		){
+			
+			return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0");
+		}
+		
+		return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0");
+	}
+	
+	private function limitstrlen($text){
+		
+		return
+			explode(
+				"\n",
+				wordwrap(
+					str_replace(
+						["\n\r", "\r\n", "\n", "\r"],
+						" ",
+						$text
+					),
+					300,
+					"\n"
+				),
+				2
+			)[0];
+	}
+}

scraper/swisscows.php

@@ -0,0 +1,252 @@
+<?php
+
+class swisscows{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("swisscows");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"type" => [
+				"display" => "Type",
+				"option" => [
+					"track" => "Tracks",
+					"playlist" => "Playlists"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $web_req = false){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		// use http2
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Access-Control-Request-Method: GET",
+			"Access-Control-Request-Headers: cache-control",
+			"Referer: https://swisscows.com/",
+			"Origin: https://swisscows.com",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-site",
+			"Priority: u=4",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function music($get, $last_attempt = false){
+		
+		if($get["npt"]){
+			
+			[$params, $proxy] = $this->backend->get($get["npt"], "music");
+			$params = json_decode($params, true);
+			$type = $params["type"];
+			$search = $params["s"];
+			$offset = $params["offset"];
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$type = $get["type"];
+			$offset = 0;
+			$proxy = $this->backend->get_ip();
+		}
+		
+		try{
+			
+			$json =
+				$this->get(
+					$proxy,
+					"https://api.swisscows.com/audio/search/{$type}s",
+					[
+						"query" => $search,
+						"offset" => $offset,
+						"itemsCount" => 20,
+						"region" => "en-US"
+					]
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"song" => [],
+			"playlist" => [],
+			"album" => [],
+			"podcast" => [],
+			"author" => [],
+			"user" => []
+		];
+		
+		if(!isset($json["items"])){
+			
+			throw new Exception("Swisscows did not return an items object");
+		}
+		
+		if($type == "track"){
+			foreach($json["items"] as $item){
+				
+				$tags = $item["tags"];
+				if(!empty($item["genre"])){
+					
+					$tags[] = $item["genre"];
+				}
+				
+				$out["song"][] = [
+					"title" => $item["title"],
+					"description" => implode(", ", $tags),
+					"url" => "/resolver?scraper=sc&target=t{$item["id"]}",
+					"views" => null,
+					"author" => [
+						"name" => null,
+						"url" => null,
+						"avatar" => null
+					],
+					"thumb" => [
+						"ratio" => "1:1",
+						"url" => $item["artworkUrl"]
+					],
+					"date" => null,
+					"duration" => $this->convert_time($item["duration"]),
+					"stream" => [					
+						"endpoint" => null,
+						"url" => null
+					]
+				];
+			}
+		}else{
+			
+			foreach($json["items"] as $item){
+				
+				$out["playlist"][] = [
+					"title" => $item["title"],
+					"description" => $this->limitstrlen($item["description"]),
+					"author" => [
+						"name" => null,
+						"url" => null,
+						"avatar" => null
+					],
+					"thumb" => [
+						"ratio" => "1:1",
+						"url" => $item["artworkUrl"]
+					],
+					"date" => null,
+					"duration" => $this->convert_time($item["duration"]),
+					"url" => "/resolver?scraper=sc&target=p{$item["id"]}",
+				];
+			}
+		}
+		
+		//
+		// get NPT
+		//
+		if(
+			isset($json["nextOffset"]) &&
+			$json["nextOffset"] !== null
+		){
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode(
+						[
+							"type" => $type,
+							"s" => $search,
+							"offset" => $json["nextOffset"]
+						]
+					),
+					"music",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function limitstrlen($text){
+		
+		return
+			explode(
+				"\n",
+				wordwrap(
+					str_replace(
+						["\n\r", "\r\n", "\n", "\r"],
+						" ",
+						$text
+					),
+					300,
+					"\n"
+				),
+				2
+			)[0];
+	}
+	
+	private function convert_time($time){
+		
+		list($hours, $minutes, $seconds) = explode(':', $time);
+		
+		return
+			((int)$hours * 3600) +
+			((int)$minutes * 60) +
+			(float)$seconds;
+	}
+}

scraper/unsplash.php

@@ -0,0 +1,267 @@
+<?php
+
+class unsplash{
+	
+	public function __construct(){
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+		
+		include "lib/backend.php";
+		$this->backend = new backend("unsplash");
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"order_by" => [
+				"display" => "Order by",
+				"option" => [
+					"relevance" => "Relevance",
+					"latest" => "Newest",
+					"editorial" => "Curated"
+				]
+			],
+			"orientation" => [
+				"display" => "Order by",
+				"option" => [
+					"any" => "Any orientation",
+					"landscape" => "Landscape",
+					"portrait" => "Portrait",
+					"squarish" => "Square"
+				]
+			],
+			"license" => [
+				"display" => "License",
+				"option" => [
+					"any" => "Any license",
+					"only" => "Unsplash+",
+					"none" => "Free"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $referer){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: */*",
+			"Accept-Language: en-US",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"Referer: {$referer}",
+			"client-geo-region: global",
+			"x-client-version: 8999df28be3f138bf2c646df5d656e4dc6970ba0",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Sec-Fetch-Dest: empty",
+			"Sec-Fetch-Mode: cors",
+			"Sec-Fetch-Site: same-origin",
+			"Priority: u=0",
+			"TE: trailers"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$filter, $proxy] =
+				$this->backend->get(
+					$get["npt"],
+					"images"
+				);
+			
+			$filter = json_decode($filter, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			$filter = [
+				"page" => 1,
+				"per_page" => 20,
+				"query" => $search
+			];
+			
+			// add filters
+			if($get["order_by"] != "relevance"){
+				
+				$filter["order_by"] = $get["order_by"];
+			}
+			
+			if($get["orientation"] != "any"){
+				
+				$filter["orientation"] = $get["orientation"];
+			}
+			
+			if($get["license"] != "any"){
+				
+				$filter["plus"] = $get["license"];
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+				
+		// https://unsplash.com/s/photos/shibuya-night?order_by=latest&orientation=landscape&license=free
+		// https://unsplash.com/s/photos/%3Ctest-%3F!-haha == <test ?! haha
+		
+		// ?page=1&per_page=20&query=japan
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://unsplash.com/napi/search/photos",
+					$filter,
+					"https://unsplash.com/s/photos/" . rawurlencode(str_replace(" ", "-", $filter["query"])),
+				);
+			
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Could not decode the JSON payload");
+		}
+		
+		foreach($json["results"] as $image){
+			
+			$base = explode("?", $image["urls"]["raw"])[0];
+			
+			if(
+				(bool)$image["premium"] ||
+				(bool)$image["plus"]
+			){
+				
+				// when we get "premium" images, give
+				// 1. highest resolution with watermarks
+				// 2. highest resolution without watermarks
+				// (if width of image is above 900px, it has watermarks)
+				// 3. thumbnail without watermark
+				$x900 = $this->imgratio((int)$image["width"], (int)$image["height"], 900);
+				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
+				
+				$source = [
+					[
+						"url" => $base,
+						"width" => (int)$image["width"],
+						"height" => (int)$image["height"]
+					],
+					[
+						"url" => $base . "?w=900",
+						"width" => $x900[0],
+						"height" => $x900[1]
+					],
+					[
+						"url" => $base . "?w=500",
+						"width" => $x500[0],
+						"height" => $x500[1]
+					]
+				];
+			}else{
+				
+				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
+				
+				// image is free as in freedom(tm)
+				$source = [
+					[
+						"url" => $base,
+						"width" => (int)$image["width"],
+						"height" => (int)$image["height"]
+					],
+					[
+						"url" => $base . "?w=500",
+						"width" => $x500[0],
+						"height" => $x500[1]
+					]
+				];
+			}
+			
+			$title = [];
+			
+			$image["description"] = trim($image["description"]);
+			$image["alt_description"] = trim($image["alt_description"]);
+			
+			if(!empty($image["description"])){ $title[] = $image["description"]; }
+			if(!empty($image["alt_description"])){ $title[] = $image["alt_description"]; }
+			
+			$title = implode(": ", $title);
+			
+			$out["image"][] = [
+				"title" => $title,
+				"source" => $source,
+				"url" => "https://unsplash.com/photos/" . $image["slug"]
+			];
+		}
+		
+		// next page stuff
+		if($filter["page"] < (int)$json["total_pages"]){
+			
+			$filter["page"]++;
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($filter),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function imgratio($width, $height, $max_width){
+		
+		$ratio = $max_width / $width;
+		$new_height = floor($height * $ratio);
+		
+		return [
+			$max_width,
+			$new_height
+		];
+	}
+}

scraper/vimeo.php

@@ -0,0 +1,754 @@
+<?php
+
+class vimeo{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("vimeo");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [
+			"time" => [
+				"display" => "Date uploaded", // &filter_uploaded=
+				"option" => [
+					"any" => "Any time",
+					"today" => "Last 24 hours",
+					"this-week" => "Last 7 days",
+					"this-month" => "Last 30 days",
+					"this-year" => "Last 365 days",
+				]
+			], 
+			"display" => [
+				"display" => "Display",
+				"option" => [
+					"video" => "Videos",
+					"ondemand" => "On-Demand ($$)",
+					"people" => "People",
+					"channel" => "Channels",
+					"group" => "Groups"
+				]
+			],
+			"sort" => [
+				"display" => "Sort by",
+				"option" => [
+					"relevance" => "Relevance", // no param
+					"recent" => "Newest", // &sort=latest&direction=desc
+					"popular" => "Most popular", // &sort=popularity&direction=desc
+					"a_z" => "Title, A to Z", // &sort=alphabetical&direction=asc
+					"z_a" => "Title, Z to A", // &sort=alphabetical&direction=desc
+					"longest" => "Longest", // &sort=duration&direction=desc
+					"shortest" => "Shortest", // &sort=duration&direction=asc
+				]
+			],
+			"duration" => [
+				"display" => "Duration", // &filter_duration=
+				"option" => [
+					"any" => "Any duration",
+					"short" => "Short (less than 4 minutes)",
+					"medium" => "Medium (4-10 minutes)",
+					"long" => "Long (over 10 minutes)"
+				]
+			],
+			"resolution" => [
+				"display" => "Resolution",
+				"option" => [
+					"any" => "Any resolution",
+					"4k" => "4K" // &filter_resolution=4k
+				]
+			],
+			"category" => [
+				"display" => "Category", // &filter_category=
+				"option" => [
+					"any" => "Any category",
+					"animation" => "Animation",
+					"comedy" => "Comedy",
+					"music" => "Music",
+					"experimental" => "Experimental",
+					"documentary" => "Documentary",
+					"identsandanimatedlogos" => "Idents and Animated Logos",
+					"industry" => "Industry",
+					"instructionals" => "Instructionals",
+					"narrative" => "Narrative",
+					"personal" => "Personal"
+				]
+			],
+			"live" => [
+				"display" => "Live events",
+				"option" => [
+					"any" => "Any",
+					"yes" => "Live now" // &filter_live=now
+				]
+			],
+			"hdr" => [
+				"display" => "HDR", // &filter_hdr=
+				"option" => [
+					"any" => "Any",
+					"hdr" => "Any HDR",
+					"dolby_vision" => "Dolby Vision",
+					"hdr10" => "HDR10",
+					"hdr10+" => "HDR10+"
+				]
+			],
+			"vimeo_360" => [
+				"display" => "Vimeo 360°", // &filter_vimeo_360
+				"option" => [
+					"any" => "Any",
+					"spatial" => "Spatial",
+					"360" => "360°"
+				]
+			],
+			"price" => [ // &filter_price=
+				"display" => "Price",
+				"option" => [
+					"any" => "Any price",
+					"free" => "Free",
+					"paid" => "Paid"
+				]
+			],
+			"collection" => [
+				"display" => "Vimeo collections",
+				"option" => [
+					"any" => "Any collection",
+					"staff_pick" => "Staff picks" // &filter_staffpicked=true
+				]
+			],
+			"license" => [ // &filter_license=
+				"display" => "License",
+				"option" => [
+					"any" => "Any license",
+					"by-nc-nd" => "CC BY-NC-ND",
+					"by" => "CC BY",
+					"by-nc" => "CC BY-NC",
+					"by-nc-sa" => "CC BY-NC-SA",
+					"by-nd" => "CC BY-ND",
+					"by-sa" => "CC BY-SA",
+					"cc0" => "CC0"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $jwt = false){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($jwt === false){
+			
+			curl_setopt(
+				$curlproc,
+				CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://vimeo.com/search",
+				"X-Requested-With: XMLHttpRequest",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-origin",
+				"Priority: u=4"]
+			);
+			
+		}else{
+			
+			curl_setopt(
+				$curlproc,
+				CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: application/vnd.vimeo.*+json;version=3.3",
+				"Accept-Language: en",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://vimeo.com/",
+				"Content-Type: application/json",
+				"Authorization: jwt $jwt",
+				"Vimeo-Page: /search/[[...slug]]",
+				"Origin: https://vimeo.com",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4"]
+			);
+		}
+		
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function video($get){
+		
+		// parse shit
+		if($get["npt"]){
+			
+			[$npt, $proxy] =
+				$this->backend
+				->get(
+					$get["npt"],
+					"videos"
+				);
+			
+			$npt = json_decode($npt, true);
+			$pagetype = $npt["pagetype"];
+			$npt = $npt["npt"];
+			
+			$jwt = $this->get_jwt($proxy);
+			
+			try{
+				
+				$json =
+					$this->get(
+						$proxy,
+						"https://api.vimeo.com" . $npt,
+						[],
+						$jwt
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JSON");
+			}
+			
+		}else{
+			
+			$proxy = null;
+			$jwt = $this->get_jwt($proxy); // this gives us a proxy by reference
+			
+			// parse filters
+			$npt = [
+				"query" => $get["s"],
+				"page" => 1,
+				"per_page" => 24,
+				"facets" => "type"
+			];
+			
+			switch($get["display"]){
+				
+				case "video":
+					$npt["filter_type"] = "clip";
+					$npt["fields"] = "clip.name,stats.plays,clip.pictures,clip.user.name,clip.user.link,clip.user.pictures.sizes,clip.uri,clip.stats.plays,clip.duration,clip.created_time,clip.link,clip.description";
+					break;
+				
+				case "ondemand":
+					$npt["filter_type"] = "ondemand";
+					$npt["sizes"] = "296x744";
+					$npt["fields"] = "ondemand.link,ondemand.name,ondemand.pictures.sizes,ondemand.metadata.interactions.buy,ondemand.metadata.interactions.rent,ondemand.uri";
+					break;
+				
+				case "people":
+					$npt["filter_type"] = "people";
+					$npt["fetch_user_profile"] = "1";
+					$npt["fields"] = "people.name,people.location_details.formatted_address,people.metadata.public_videos.total,people.pictures.sizes,people.link,people.metadata.connections.followers.total,people.skills.name,people.skills.uri,people.background_video,people.uri";
+					break;
+				
+				case "channel":
+					$npt["filter_type"] = "channel";
+					$npt["fields"] = "channel.name,channel.metadata.connections.users.total,channel.metadata.connections.videos.total,channel.pictures.sizes,channel.link,channel.uri";
+					break;
+				
+				case "group":
+					$npt["filter_type"] = "group";
+					$npt["fields"] = "group.name,group.metadata.connections.users.total,group.metadata.connections.videos.total,group.pictures.sizes,group.link,group.uri";
+					break;
+			}
+			
+			// only apply filters if we're searching for videos
+			if($get["display"] == "video"){
+				
+				switch($get["sort"]){
+					
+					case "relevance": break; // do nothing
+					
+					case "recent":
+						$npt["sort"] = "latest";
+						$npt["direction"] = "desc";
+						break;
+					
+					case "popular":
+						$npt["sort"] = "popularity";
+						$npt["direction"] = "desc";
+						break;
+					
+					case "a_z":
+						$npt["sort"] = "alphabetical";
+						$npt["direction"] = "asc";
+						break;
+					
+					case "z_a":
+						$npt["sort"] = "alphabetical";
+						$npt["direction"] = "desc";
+						break;
+					
+					case "longest":
+						$npt["sort"] = "duration";
+						$npt["direction"] = "desc";
+						break;
+					
+					case "shortest":
+						$npt["sort"] = "duration";
+						$npt["direction"] = "asc";
+						break;
+				}
+				
+				if($get["time"] != "any"){
+					
+					$npt["filter_uploaded"] = $get["time"];
+				}
+				
+				if($get["duration"] != "any"){
+					
+					$npt["filter_duration"] = $get["duration"];
+				}
+				
+				if($get["resolution"] != "any"){
+					
+					$npt["filter_resolution"] = $get["resolution"];
+				}
+				
+				if($get["category"] != "any"){
+					
+					$npt["filter_category"] = $get["category"];
+				}
+				
+				if($get["live"] != "any"){
+					
+					$npt["filter_live"] = "now";
+				}
+				
+				if($get["hdr"] != "any"){
+					
+					$npt["filter_hdr"] = $get["hdr"];
+				}
+				
+				if($get["vimeo_360"] != "any"){
+					
+					$npt["filter_vimeo_360"] = $get["vimeo_360"];
+				}
+				
+				if($get["price"] != "any"){
+					
+					$npt["filter_price"] = $get["price"];
+				}
+				
+				if($get["collection"] == "staff_pick"){
+					
+					$npt["filter_staffpicked"] = "true";
+				}
+				
+				if($get["license"] != "any"){
+					
+					$npt["filter_license"] = $get["license"];
+				}
+			}
+			
+			$pagetype = $npt["filter_type"];
+			
+			try{
+				
+				$json =
+					$this->get(
+						$proxy,
+						"https://api.vimeo.com/search",
+						$npt,
+						$jwt
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JSON");
+			}
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to parse JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"video" => [],
+			"author" => [],
+			"livestream" => [],
+			"playlist" => [],
+			"reel" => []
+		];
+		
+		if(isset($json["error"])){
+			
+			$error = $json["error"];
+			if(isset($json["developer_message"])){
+				
+				$error .= " ({$json["developer_message"]})";
+			}
+			
+			throw new Exception("Vimeo returned an error: " . $error);
+		}
+		
+		if(!isset($json["data"])){
+			
+			throw new Exception("Vimeo did not return a data object");
+		}
+		
+		switch($pagetype){
+			
+			case "clip":
+				foreach($json["data"] as $video){
+					
+					$video = $video["clip"];
+					
+					if(isset($video["user"]["pictures"]["sizes"])){
+						
+						$avatar = $video["user"]["pictures"]["sizes"][count($video["user"]["pictures"]["sizes"]) - 1]["link"];
+					}else{
+						
+						$avatar = null;
+					}
+					
+					$out["video"][] = [
+						"title" => $video["name"],
+						"description" =>
+							$this->limitstrlen(
+								$video["description"]
+							),
+						"author" => [
+							"name" => $video["user"]["name"],
+							"url" => $video["user"]["link"],
+							"avatar" => $avatar
+						],
+						"date" => strtotime($video["created_time"]),
+						"duration" => (int)$video["duration"],
+						"views" => (int)$video["stats"]["plays"],
+						"thumb" => [
+							"ratio" => "16:9",
+							"url" => $video["pictures"]["base_link"]
+						],
+						"url" => $video["link"]
+					];
+				}
+				break;
+			
+			case "ondemand":
+				foreach($json["data"] as $video){
+					
+					$video = $video["ondemand"];
+					
+					$description = [];
+					if(isset($video["metadata"]["interactions"]["rent"]["display_price"])){
+						
+						$description[] = "Rent for " . $video["metadata"]["interactions"]["rent"]["display_price"];
+					}
+					
+					if(isset($video["metadata"]["interactions"]["buy"]["display_price"])){
+						
+						$description[] = "Buy for " . $video["metadata"]["interactions"]["buy"]["display_price"];
+					}
+					
+					$description = implode(", ", $description);
+					
+					$out["video"][] = [
+						"title" => $video["name"],
+						"description" => $description,
+						"author" => [
+							"name" => null,
+							"url" => null,
+							"avatar" => null
+						],
+						"date" => null,
+						"duration" => null,
+						"views" => null,
+						"thumb" => [
+							"ratio" => "9:16",
+							"url" => $video["pictures"]["sizes"][0]["link"]
+						],
+						"url" => $video["link"]
+					];
+				}
+				break;
+			
+			case "people":
+				foreach($json["data"] as $user){
+					
+					$user = $user["people"];
+					
+					if(
+						isset($user["pictures"]["sizes"]) &&
+						count($user["pictures"]["sizes"]) !== 0
+					){
+						
+						$thumb = [
+							"ratio" => "1:1",
+							"url" => $user["pictures"]["sizes"][count($user["pictures"]["sizes"]) - 1]["link"]
+						];
+					}else{
+						
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}
+					
+					$out["author"][] = [
+						"title" => $user["name"],
+						"followers" => (int)$user["metadata"]["connections"]["followers"]["total"],
+						"description" => $user["metadata"]["public_videos"]["total"] . " videos.",
+						"thumb" => $thumb,
+						"url" => $user["link"]
+					];
+				}
+				break;
+			
+			case "channel":
+			case "group":
+				foreach($json["data"] as $channel){
+					
+					$channel = $channel[$npt["filter_type"]];
+					
+					if(
+						isset($channel["pictures"]["sizes"]) &&
+						count($channel["pictures"]["sizes"]) !== 0
+					){
+						
+						$thumb = [
+							"ratio" => "16:9",
+							"url" => $channel["pictures"]["sizes"][count($channel["pictures"]["sizes"]) - 1]["link"]
+						];
+					}else{
+						
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}
+					
+					$out["author"][] = [
+						"title" => $channel["name"],
+						"followers" => (int)$channel["metadata"]["connections"]["users"]["total"],
+						"description" => $channel["metadata"]["connections"]["videos"]["total"] . " videos.",
+						"thumb" => $thumb,
+						"url" => $channel["link"]
+					];
+				}
+				break;
+		}
+		
+		//
+		// get next page
+		//
+		if(
+			isset($json["paging"]["next"]) &&
+			is_string($json["paging"]["next"])
+		){
+			
+			$out["npt"] =
+				$this->backend
+				->store(
+					json_encode([
+						"npt" => $json["paging"]["next"],
+						"pagetype" => $pagetype
+					]),
+					"videos",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function get_jwt(&$proxy){
+		
+		//
+		// get jwt token
+		// it's probably safe to cache this across proxies, cause the jwt doesnt contain an userID
+		// only an appID, whatever shit that is
+		// we can only cache it for 5 minutes though, otherwise vimeo cries about it
+		//
+		if($proxy === null){
+			
+			$proxy = $this->backend->get_ip();
+		}
+		
+		$jwt = apcu_fetch("vimeo_jwt");
+		
+		if($jwt === false){
+			/*
+			$html =
+				$this->get(
+					$proxy,
+					"https://vimeo.com/search",
+					[],
+					false
+				);
+			
+			$this->fuckhtml->load($html);
+			
+			$captcha =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"title"
+				);
+			
+			if(
+				count($captcha) !== 0 &&
+				$this->fuckhtml
+				->getTextContent(
+					$captcha[0]
+				) == "Vimeo / CAPTCHA Challenge"
+			){
+				
+				throw new Exception("Vimeo returned a Captcha");
+			}
+			
+			$html =
+				explode(
+					'<script id="viewer-bootstrap" type="application/json">',
+					$html,
+					2
+				);
+			
+			if(count($html) !== 2){
+				
+				throw new Exception("Failed to find JWT json");
+			}
+			
+			$jwt =
+				json_decode(
+					$this->fuckhtml
+					->extract_json(
+						$html[1]
+					),
+					true
+				);
+			
+			if($jwt === null){
+				
+				throw new Exception("Failed to decode JWT json");
+			}
+			
+			if(!isset($jwt["jwt"])){
+				
+				throw new Exception("Failed to grep JWT");
+			}
+			
+			$jwt = $jwt["jwt"];
+			*/
+			
+			try{
+				$json =
+					$this->get(
+						$proxy,
+						"https://vimeo.com/_next/jwt",
+						[],
+						false
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch JWT token");
+			}
+			
+			$this->fuckhtml->load($json);
+			
+			$captcha =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"title"
+				);
+			
+			if(
+				count($captcha) !== 0 &&
+				$this->fuckhtml
+				->getTextContent(
+					$captcha[0]
+				) == "Vimeo / CAPTCHA Challenge"
+			){
+				
+				throw new Exception("Vimeo returned a Captcha");
+			}
+				
+			$json = json_decode($json, true);
+			
+			if($json === null){
+				
+				throw new Exception("The JWT object could not be decoded");
+			}
+			
+			if(!isset($json["token"])){
+				
+				throw new Exception("Vimeo did not return a JWT");
+			}
+			
+			$jwt = $json["token"];
+			
+			apcu_store("vimeo_jwt", $jwt, 300);
+		}
+		
+		return $jwt;
+	}
+	
+	private function titledots($title){
+		
+		$substr = substr($title, -3);
+		
+		if(
+			$substr == "..." ||
+			$substr == "…"
+		){
+			
+			return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0");
+		}
+		
+		return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0");
+	}
+	
+	private function limitstrlen($text){
+		
+		return
+			explode(
+				"\n",
+				wordwrap(
+					str_replace(
+						["\n\r", "\r\n", "\n", "\r"],
+						" ",
+						$text
+					),
+					300,
+					"\n"
+				),
+				2
+			)[0];
+	}
+}

scraper/vsco.php

@@ -0,0 +1,256 @@
+<?php
+
+class vsco{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("vsco");
+	}
+	
+	public function getfilters($page){
+		
+		return [];
+	}
+	
+	private function get($proxy, $url, $get = [], $bearer = null){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get_tmp = http_build_query($get);
+			$url .= "?" . $get_tmp;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($bearer === null){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1",
+				"Priority: u=0, i"]
+			);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US",
+				"Accept-Encoding: gzip",
+				"Referer: https://vsco.co/search/images/" . urlencode($get["query"]),
+				"authorization: Bearer " . $bearer,
+				"content-type: application/json",
+				"x-client-build: 1",
+				"x-client-platform: web",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-origin",
+				"Priority: u=0",
+				"TE: trailers"]
+			);
+		}
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// http1 bypass?!?!?!
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function image($get){
+		
+		if($get["npt"]){
+			
+			[$data, $proxy] =
+				$this->backend->get(
+					$get["npt"], "images"
+				);
+			
+			$data = json_decode($data, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			
+			// get bearer token
+			try{
+				
+				$html =
+					$this->get(
+						$proxy,
+						"https://vsco.co/feed"
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch feed page");
+			}
+			
+			preg_match(
+				'/"tkn":"([A-z0-9]+)"/',
+				$html,
+				$bearer
+			);
+			
+			if(!isset($bearer[1])){
+				
+				throw new Exception("Failed to grep bearer token");
+			}
+			
+			$data = [
+				"pagination" => [
+					"query" => $search,
+					"page" => 0,
+					"size" => 100
+				],
+				"bearer" => $bearer[1]
+			];
+		}
+		
+		try{
+			
+			$json =
+				$this->get(
+					$proxy,
+					"https://vsco.co/api/2.0/search/images",
+					$data["pagination"],
+					$data["bearer"]
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"image" => []
+		];
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Failed to access results object");
+		}
+		
+		foreach($json["results"] as $image){
+			
+			$image_domain = parse_url("https://" . $image["responsive_url"], PHP_URL_HOST);
+			$thumbnail = explode($image_domain, $image["responsive_url"], 2)[1];
+			
+			if(substr($thumbnail, 0, 3) != "/1/"){
+				
+				$thumbnail =
+					preg_replace(
+						'/^\/[^\/]+/',
+						"",
+						$thumbnail
+					);
+			}
+			
+			$thumbnail = "https://img.vsco.co/cdn-cgi/image/width=480,height=360" . $thumbnail;
+			$size =
+				$this->image_ratio(
+					(int)$image["dimensions"]["width"],
+					(int)$image["dimensions"]["height"]
+				);
+			
+			$out["image"][] = [
+				"title" => $image["description"],
+				"source" => [
+					[
+						"url" => "https://" . $image["responsive_url"],
+						"width" => (int)$image["dimensions"]["width"],
+						"height" => (int)$image["dimensions"]["height"]
+					],
+					[
+						"url" => $thumbnail,
+						"width" => $size[0],
+						"height" => $size[1]
+					]
+				],
+				"url" => "https://" . $image["grid"]["domain"] . "/media/" . $image["imageId"]
+			];
+		}
+		
+		// get NPT
+		$max_page = ceil($json["total"] / 100);
+		$data["pagination"]["page"]++;
+		
+		if($max_page > $data["pagination"]["page"]){
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($data),
+					"images",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	private function image_ratio($width, $height){
+		
+		$ratio = [
+			480 / $width,
+			360 / $height
+		];
+		
+		if($ratio[0] < $ratio[1]){
+			
+			$ratio = $ratio[0];
+		}else{
+			
+			$ratio = $ratio[1];
+		}
+		
+		return [
+			floor($width * $ratio),
+			floor($height * $ratio)
+		];
+	}
+}

scraper/yandex.php

@@ -14,7 +14,7 @@ class yandex{
 		// backend included in the scraper functions
 	}
 	
-	private function get($proxy, $url, $get = [], $nsfw){
+	private function get($proxy, $url, $get = [], $nsfw, $get_cookie = 1){
 		
 		$curlproc = curl_init();
 		
@@ -25,19 +25,55 @@ class yandex{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
+		// extract "i" cookie
+		if($get_cookie === 0){
+			
+			$cookies_tmp = [];
+			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
+				
+				$length = strlen($header);
+				
+				$header = explode(":", $header, 2);
+				
+				if(trim(strtolower($header[0])) == "set-cookie"){
+					
+					$cookie_tmp = explode("=", trim($header[1]), 2);
+					
+					$cookies_tmp[trim($cookie_tmp[0])] =
+						explode(";", $cookie_tmp[1], 2)[0];
+				}
+				
+				return $length;
+			});
+		}
+		
 		switch($nsfw){
 			case "yes": $nsfw = "0"; break;
 			case "maybe": $nsfw = "1"; break;
 			case "no": $nsfw = "2"; break;
 		}
 		
+		switch($get_cookie){
+			
+			case 0:
+				$cookie = "";
+				break;
+			
+			case 1:
+				$cookie = "Cookie: yp=" . (time() - 4000033) . ".szm.1:1920x1080:876x1000#" . time() . ".sp.family:" . $nsfw;
+				break;
+			
+			default:
+				$cookie = "Cookie: i=" . $get_cookie;
+		}
+		
 		$headers =
 			["User-Agent: " . config::USER_AGENT,
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
 			"Accept-Encoding: gzip",
 			"Accept-Language: en-US,en;q=0.5",
 			"DNT: 1",
-			"Cookie: yp=1716337604.sp.family%3A{$nsfw}#1685406411.szm.1:1920x1080:1920x999",
+			$cookie,
 			"Referer: https://yandex.com/images/search",
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
@@ -59,6 +95,17 @@ class yandex{
 		
 		$data = curl_exec($curlproc);
 		
+		if($get_cookie === 0){
+			
+			if(isset($cookies_tmp["i"])){
+				
+				return $cookies_tmp["i"];
+			}else{
+				
+				throw new Exception("Failed to get Yandex clearance cookie");
+			}
+		}
+		
 		if(curl_errno($curlproc)){
 			
 			throw new Exception(curl_error($curlproc));
@@ -217,6 +264,23 @@ class yandex{
 		// https://yandex.com/search/site/?text=minecraft&web=1&frame=1&v=2.0&searchid=3131712
 		// &within=777&from_day=26&from_month=8&from_year=2023&to_day=26&to_month=8&to_year=2023
 		
+		// get clearance cookie
+		if(($cookie = apcu_fetch("yandexweb_cookie")) === false){
+			
+			$proxy = $this->backend->get_ip();
+			
+			$cookie =
+				$this->get(
+					$proxy,
+					"https://yandex.ru/support2/smart-captcha/ru/",
+					[],
+					false,
+					0
+				);
+			
+			apcu_store("yandexweb_cookie", $cookie);
+		}
+		
 		if($get["npt"]){
 			
 			[$npt, $proxy] = $this->backend->get($get["npt"], "web");
@@ -226,7 +290,8 @@ class yandex{
 					$proxy,
 					"https://yandex.com" . $npt,
 					[],
-					"yes"
+					"yes",
+					$cookie
 				);
 		}else{
 			
@@ -236,7 +301,7 @@ class yandex{
 				throw new Exception("Search term is empty!");
 			}
 			
-			$proxy = $this->backend->get_ip();
+			$proxy = !isset($proxy) ? $this->backend->get_ip() : $proxy;
 			$lang = $get["lang"];
 			$older = $get["older"];
 			$newer = $get["newer"];
@@ -283,7 +348,8 @@ class yandex{
 						$proxy,
 						"https://yandex.com/search/site/",
 						$params,
-						"yes"
+						"yes",
+						$cookie
 					);
 			}catch(Exception $error){
 				
@@ -314,6 +380,19 @@ class yandex{
 		
 		$this->fuckhtml->load($html);
 		
+		// Scrape page blocked error
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName("title");
+		
+		if(
+			count($title) !== 0 &&
+			$title[0]["innerHTML"] == "403"
+		){
+			
+			throw new Exception("Yandex blocked this proxy or 4get instance.");
+		}
+		
 		// get nextpage
 		$npt =
 			$this->fuckhtml
@@ -668,7 +747,6 @@ class yandex{
 		foreach($json["blocks"] as $block){
 			
 			$html .= $block["html"];
-			
 			// get next page
 			if(
 				isset($block["params"]["nextPageUrl"]) &&
@@ -790,123 +868,71 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$get, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$params = json_decode($params, true);
-			
-			$nsfw = $params["nsfw"];
-			unset($params["nsfw"]);
+			$get = json_decode($get, true);
 		}else{
 			
-			$search = $get["s"];
-			if(strlen($search) === 0){
+			if(strlen($get["s"]) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$time = $get["time"];
-			$duration = $get["duration"];
-			
-			// https://yandex.com/video/search
-			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-			// &format=json
-			// &request=
-			// {
-			//		"blocks":[
-			//			{"block":"extra-content","params":{},"version":2},
-			//			{"block":"i-global__params:ajax","params":{},"version":2},
-			//			{"block":"search2:ajax","params":{},"version":2},
-			//			{"block":"vital-incut","params":{},"version":2},
-			//			{"block":"content_type_search","params":{},"version":2},
-			//			{"block":"serp-controller","params":{},"version":2},
-			//			{"block":"cookies_ajax","params":{},"version":2}
-			//		],
-			//		"metadata":{
-			//			"bundles":{"lb":"^G]!q<X120"},
-			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
-			//			"extraContent":{"names":["i-react-ajax-adapter"]}
-			//		}
-			// }
-			// &yu=4861394161661655015
-			// &from=tabbar
-			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
-			// &suggest_reqid=486139416166165501562797413447032
-			// &text=minecraft
-			
-			$params = [
-				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-				"format" => "json",
-				"request" => json_encode([
-					"blocks" => [
-						(object)[
-							"block" => "extra-content",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "i-global__params:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "search2:ajax",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "vital-incut",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "content_type_search",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "serp-controller",
-							"params" => (object)[],
-							"version" => 2
-						],
-						(object)[
-							"block" => "cookies_ajax",
-							"params" => (object)[],
-							"version" => 2
-						]
-					],
-					"metadata" => (object)[
-						"bundles" => (object)[
-							"lb" => "^G]!q<X120"
-						],
-						"assets" => (object)[
-							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
-						],
-						"extraContent" => (object)[
-							"names" => [
-								"i-react-ajax-adapter"
-							]
-						]
-					]
-				]),
-				"text" => $search
-			];
-			
-			if($duration != "any"){
-				
-				$params["duration"] = $duration;
-			}
-			
-			if($time != "any"){
-				
-				$params["within"] = $time;
-			}
 		}
+		
+		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		// https://yandex.com/video/search
+		// ?text=skycamefalling
+		// &from=tabbar
+		// &format=json
+		// &ncrnd=7271
+		// &p=0
+		// &parent-reqid=
+		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+		// &yu=3091577281773194415
+		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+		
+		$params = [
+			"text" => $get["s"],
+			"from" => "tabbar",
+			"format" => "json",
+			"ncrnd" => 7271,
+			"p" => 0,
+			"parent-reqid" => "",
+			"request" => json_encode((object)[
+				"blocks" => [
+					(object)[
+						"block" => "video-app",
+						"params" => (object)[]
+					]
+				]
+			]),
+			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+			"yu" => 3091577281773194415,
+			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
+		];
+		
+		if(isset($get["p"])){
+			
+			$params["p"] = $get["p"];
+		}
+		
+		if($get["duration"] != "any"){
+			
+			$params["duration"] = $get["duration"];
+		}
+		
+		if($get["time"] != "any"){
+			
+			$params["within"] = $get["time"];
+		}
+		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -918,7 +944,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$nsfw,
+					$get["nsfw"],
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -933,7 +959,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["blocks"])){
+		if(!isset($json["results"]["clips"]["items"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -948,209 +974,120 @@ class yandex{
 			"reel" => []
 		];
 		
-		$html = null;
-		foreach($json["blocks"] as $block){
+		foreach($json["results"]["clips"]["items"] as $k => $data){
 			
-			if(isset($block["html"])){
+			if(isset($data["preview"]["posterSrc"])){
 				
-				$html .= $block["html"];
+				$poster = $data["preview"]["posterSrc"];
+				
+				if(
+					preg_match(
+						'/^\/\//',
+						$data["preview"]["posterSrc"]
+					)
+				){
+					
+					$poster = "https:" . $poster;
+				}
+				
+				$thumb = [
+					"ratio" => "16:9",
+					"url" => $poster
+				];
+			}else{
+				
+				$thumb = [
+					"ratio" => null,
+					"url" => null
+				];
 			}
+			
+			$out["video"][] = [
+				"title" => $data["relatedParams"]["text"],
+				"description" => $this->titledots($data["description"]),
+				"author" => [
+					"name" =>
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+					"url" => 
+						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
+					"avatar" => null
+				],
+				"date" =>
+					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
+					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
+				"duration" =>
+					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
+					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
+				"views" =>
+					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
+					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
+				"thumb" => $thumb,
+				"url" =>
+					preg_replace(
+						'/^http:\/\//',
+						"https://",
+						$data["relatedParams"]["related_url"]
+					)
+			];
 		}
 		
-		$this->fuckhtml->load($html);
+		// get npt
+		if($json["results"]["search"]["hasNextPage"]){
 			
-		$div =
-			$this->fuckhtml
-			->getElementsByTagName("div");
+			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
-		/*
-			Get nextpage
-		*/
-		$npt =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"more more_direction_next i-bem",
-				$div
-			);
-		
-		if(count($npt) !== 0){
-			
-			$params["p"] = "1";
-			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($params),
+					json_encode($get),
 					"video",
 					$proxy
 				);
 		}
 		
-		$items =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"serp-item",
-				$div
-			);
-		
-		foreach($items as $item){
-			
-			$data =
-				json_decode(
-					$this->fuckhtml
-					->getTextContent(
-						$item["attributes"]["data-video"]
-					),
-					true
-				);
-			
-			$this->fuckhtml->load($item);
-			
-			$thumb =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"thumb-image__image",
-					"img"
-				);
-			
-			$c = 1;
-			if(count($thumb) === 0){
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}else{
-				
-				$thumb = [
-					"url" =>
-						str_replace(
-							"//",
-							"https://",
-							$this->fuckhtml
-							->getTextContent(
-								$thumb
-								[0]
-								["attributes"]
-								["src"]
-							),
-							$c
-						),
-					"ratio" => "16:9"
-				];
-			}
-			
-			$smallinfos =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__sitelinks-item",
-					"div"
-				);
-			
-			$date = null;
-			$views = null;
-			$first = true;
-			
-			foreach($smallinfos as $info){
-				
-				if($first){
-					
-					$first = false;
-					continue;
-				}
-				
-				$info =
-					$this->fuckhtml
-					->getTextContent(
-						$info
-					);
-				
-				if($temp_date = strtotime($info)){
-					
-					$date = $temp_date;
-				}else{
-					
-					$views = $this->parseviews($info);
-				}
-			}
-			
-			$description =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"serp-item__text serp-item__text_visibleText_always",
-					"div"
-				);
-			
-			if(count($description) === 0){
-				
-				$description = null;
-			}else{
-				
-				$description =
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$description[0]
-						)
-					);
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->fuckhtml
-					->getTextContent(
-						$this->titledots(
-							$data["title"]
-						)
-					),
-				"description" => $description,
-				"author" => [
-					"name" => null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" => $date,
-				"duration" =>
-					(int)$data
-					["counters"]
-					["toHostingLoaded"]
-					["stredParams"]
-					["duration"],
-				"views" => $views,
-				"thumb" => $thumb,
-				"url" =>
-					str_replace(
-						"http://",
-						"https://",
-						$this->fuckhtml
-						->getTextContent(
-							$data["counters"]
-							["toHostingLoaded"]
-							["postfix"]
-							["href"]
-						),
-						$c
-					)
-			];
-		}
-		
 		return $out;
 	}
 	
-	private function parseviews($text){
+	private function parseviews($number){
 		
-		$text = explode(" ", $text);
+		// decimal should always be 1 number long
+		$number = explode(" ", $number, 2);
+		$number = $number[0];
 		
-		$num = (float)$text[0];
-		$mod = $text[1];
+		$unit = strtolower($number[strlen($number) - 1]);
 		
-		switch($mod){
+		$tmp = explode(".", $number, 2);
+		$number = (int)$number;
 		
-			case "bln.": $num = $num * 1000000000; break;
-			case "mln.": $num = $num * 1000000; break;
-			case "thsd.": $num = $num * 1000; break;
+		if(count($tmp) === 2){
+			
+			$decimal = (int)$tmp[1];
+		}else{
+			
+			$decimal = 0;
 		}
 		
-		return $num;
+		switch($unit){
+			
+			case "k":
+				$exponant = 1000;
+				break;
+			
+			case "m":
+				$exponant = 1000000;
+				break;
+			
+			case "b";
+				$exponant = 1000000000;
+				break;
+			
+			default:
+				$exponant = 1;
+				break;
+		}
+		
+		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

scraper/yep.php

@@ -14,234 +14,209 @@ class yep{
 	public function getfilters($page){
 		
 		return [
-			"country" => [
-				"display" => "Country",
+			"lang" => [
+				"display" => "Language",
 				"option" => [
-					"all" => "All regions",
-					"af" => "Afghanistan",
-					"al" => "Albania",
-					"dz" => "Algeria",
-					"as" => "American Samoa",
-					"ad" => "Andorra",
-					"ao" => "Angola",
-					"ai" => "Anguilla",
-					"ag" => "Antigua and Barbuda",
-					"ar" => "Argentina",
-					"am" => "Armenia",
-					"aw" => "Aruba",
-					"au" => "Australia",
-					"at" => "Austria",
-					"az" => "Azerbaijan",
-					"bs" => "Bahamas",
-					"bh" => "Bahrain",
-					"bd" => "Bangladesh",
-					"bb" => "Barbados",
-					"by" => "Belarus",
-					"be" => "Belgium",
-					"bz" => "Belize",
-					"bj" => "Benin",
-					"bt" => "Bhutan",
-					"bo" => "Bolivia",
-					"ba" => "Bosnia and Herzegovina",
-					"bw" => "Botswana",
-					"br" => "Brazil",
-					"bn" => "Brunei Darussalam",
-					"bg" => "Bulgaria",
-					"bf" => "Burkina Faso",
-					"bi" => "Burundi",
-					"cv" => "Cabo Verde",
-					"kh" => "Cambodia",
-					"cm" => "Cameroon",
-					"ca" => "Canada",
-					"ky" => "Cayman Islands",
-					"cf" => "Central African Republic",
-					"td" => "Chad",
-					"cl" => "Chile",
-					"cn" => "China",
-					"co" => "Colombia",
-					"cg" => "Congo",
-					"cd" => "Congo, Democratic Republic",
-					"ck" => "Cook Islands",
-					"cr" => "Costa Rica",
-					"hr" => "Croatia",
-					"cu" => "Cuba",
-					"cy" => "Cyprus",
-					"cz" => "Czechia",
-					"ci" => "Côte d'Ivoire",
-					"dk" => "Denmark",
-					"dj" => "Djibouti",
-					"dm" => "Dominica",
-					"do" => "Dominican Republic",
-					"ec" => "Ecuador",
-					"eg" => "Egypt",
-					"sv" => "El Salvador",
-					"gq" => "Equatorial Guinea",
-					"ee" => "Estonia",
-					"et" => "Ethiopia",
-					"fo" => "Faroe Islands",
-					"fj" => "Fiji",
-					"fi" => "Finland",
-					"fr" => "France",
-					"gf" => "French Guiana",
-					"pf" => "French Polynesia",
-					"ga" => "Gabon",
-					"gm" => "Gambia",
-					"ge" => "Georgia",
-					"de" => "Germany",
-					"gh" => "Ghana",
-					"gi" => "Gibraltar",
-					"gr" => "Greece",
-					"gl" => "Greenland",
-					"gd" => "Grenada",
-					"gp" => "Guadeloupe",
-					"gu" => "Guam",
-					"gt" => "Guatemala",
-					"gg" => "Guernsey",
-					"gn" => "Guinea",
-					"gy" => "Guyana",
-					"ht" => "Haiti",
-					"hn" => "Honduras",
-					"hk" => "Hong Kong",
-					"hu" => "Hungary",
-					"is" => "Iceland",
-					"in" => "India",
-					"id" => "Indonesia",
-					"iq" => "Iraq",
-					"ie" => "Ireland",
-					"im" => "Isle of Man",
-					"il" => "Israel",
-					"it" => "Italy",
-					"jm" => "Jamaica",
-					"jp" => "Japan",
-					"je" => "Jersey",
-					"jo" => "Jordan",
-					"kz" => "Kazakhstan",
-					"ke" => "Kenya",
-					"ki" => "Kiribati",
-					"kw" => "Kuwait",
-					"kg" => "Kyrgyzstan",
-					"la" => "Lao People's Democratic Republic",
-					"lv" => "Latvia",
-					"lb" => "Lebanon",
-					"ls" => "Lesotho",
-					"ly" => "Libya",
-					"li" => "Liechtenstein",
-					"lt" => "Lithuania",
-					"lu" => "Luxembourg",
-					"mk" => "Macedonia",
-					"mg" => "Madagascar",
-					"mw" => "Malawi",
-					"my" => "Malaysia",
-					"mv" => "Maldives",
-					"ml" => "Mali",
-					"mt" => "Malta",
-					"mq" => "Martinique",
-					"mr" => "Mauritania",
-					"mu" => "Mauritius",
-					"yt" => "Mayotte",
-					"mx" => "Mexico",
-					"fm" => "Micronesia, Federated States of",
-					"md" => "Moldova",
-					"mc" => "Monaco",
-					"mn" => "Mongolia",
-					"me" => "Montenegro",
-					"ms" => "Montserrat",
-					"ma" => "Morocco",
-					"mz" => "Mozambique",
-					"mm" => "Myanmar",
-					"na" => "Namibia",
-					"nr" => "Nauru",
-					"np" => "Nepal",
-					"nl" => "Netherlands",
-					"nc" => "New Caledonia",
-					"nz" => "New Zealand",
-					"ni" => "Nicaragua",
-					"ne" => "Niger",
-					"ng" => "Nigeria",
-					"nu" => "Niue",
-					"no" => "Norway",
-					"om" => "Oman",
-					"pk" => "Pakistan",
-					"ps" => "Palestine, State of",
-					"pa" => "Panama",
-					"pg" => "Papua New Guinea",
-					"py" => "Paraguay",
-					"pe" => "Peru",
-					"ph" => "Philippines",
-					"pn" => "Pitcairn",
-					"pl" => "Poland",
-					"pt" => "Portugal",
-					"pr" => "Puerto Rico",
-					"qa" => "Qatar",
-					"ro" => "Romania",
-					"ru" => "Russian Federation",
-					"rw" => "Rwanda",
-					"re" => "Réunion",
-					"sh" => "Saint Helena",
-					"kn" => "Saint Kitts and Nevis",
-					"lc" => "Saint Lucia",
-					"vc" => "Saint Vincent and the Grenadines",
-					"ws" => "Samoa",
-					"sm" => "San Marino",
-					"st" => "Sao Tome and Principe",
-					"sa" => "Saudi Arabia",
-					"sn" => "Senegal",
-					"rs" => "Serbia",
-					"sc" => "Seychelles",
-					"sl" => "Sierra Leone",
-					"sg" => "Singapore",
-					"sk" => "Slovakia",
-					"si" => "Slovenia",
-					"sb" => "Solomon Islands",
-					"so" => "Somalia",
-					"kr" => "Sourth Korea",
-					"za" => "South Africa",
-					"es" => "Spain",
-					"lk" => "Sri Lanka",
-					"sr" => "Suriname",
-					"se" => "Sweden",
-					"ch" => "Switzerland",
-					"tw" => "Taiwan",
-					"tj" => "Tajikistan",
-					"tz" => "Tanzania",
-					"th" => "Thailand",
-					"tl" => "Timor-Leste",
-					"tg" => "Togo",
-					"tk" => "Tokelau",
+					"any" => "Any language",
+					"aa" => "Afar",
+					"ab" => "Abkhazian",
+					"ae" => "Avestan",
+					"af" => "Afrikaans",
+					"ak" => "Akan",
+					"am" => "Amharic",
+					"an" => "Aragonese",
+					"ar" => "Arabic",
+					"as" => "Assamese",
+					"av" => "Avaric",
+					"ay" => "Aymara",
+					"az" => "Azerbaijani",
+					"ba" => "Bashkir",
+					"be" => "Belarusian",
+					"bg" => "Bulgarian",
+					"bh" => "Bihari",
+					"bi" => "Bislama",
+					"bm" => "Bambara",
+					"bn" => "Bengali",
+					"bo" => "Tibetan",
+					"br" => "Breton",
+					"bs" => "Bosnian",
+					"ca" => "Catalan",
+					"ce" => "Chechen",
+					"ch" => "Chamorro",
+					"co" => "Corsican",
+					"cr" => "Cree",
+					"cs" => "Czech",
+					"cu" => "Church Slavic",
+					"cv" => "Chuvash",
+					"cy" => "Welsh",
+					"da" => "Danish",
+					"de" => "German",
+					"dv" => "Divehi",
+					"dz" => "Dzongkha",
+					"ee" => "Ewe",
+					"el" => "Greek",
+					"en" => "English",
+					"eo" => "Esperanto",
+					"es" => "Spanish",
+					"et" => "Estonian",
+					"eu" => "Basque",
+					"fa" => "Persian",
+					"ff" => "Fulah",
+					"fi" => "Finnish",
+					"fj" => "Fijian",
+					"fo" => "Faroese",
+					"fr" => "French",
+					"fy" => "Western Frisian",
+					"ga" => "Irish",
+					"gd" => "Scottish Gaelic",
+					"gl" => "Galician",
+					"gn" => "Guarani",
+					"gu" => "Gujarati",
+					"gv" => "Manx",
+					"ha" => "Hausa",
+					"he" => "Hebrew",
+					"hi" => "Hindi",
+					"ho" => "Hiri Motu",
+					"hr" => "Croatian",
+					"ht" => "Haitian",
+					"hu" => "Hungarian",
+					"hy" => "Armenian",
+					"hz" => "Herero",
+					"ia" => "Interlingua",
+					"id" => "Indonesian",
+					"ie" => "Interlingue",
+					"ig" => "Igbo",
+					"ii" => "Sichuan Yi",
+					"ik" => "Inupiaq",
+					"io" => "Ido",
+					"is" => "Icelandic",
+					"it" => "Italian",
+					"iu" => "Inuktitut",
+					"ja" => "Japanese",
+					"jv" => "Javanese",
+					"ka" => "Georgian",
+					"kg" => "Kongo",
+					"ki" => "Kikuyu",
+					"kj" => "Kuanyama",
+					"kk" => "Kazakh",
+					"kl" => "Kalaallisut",
+					"km" => "Central Khmer",
+					"kn" => "Kannada",
+					"ko" => "Korean",
+					"kr" => "Kanuri",
+					"ks" => "Kashmiri",
+					"ku" => "Kurdish",
+					"kv" => "Komi",
+					"kw" => "Cornish",
+					"ky" => "Kyrgyz",
+					"la" => "Latin",
+					"lb" => "Luxembourgish",
+					"lg" => "Ganda",
+					"li" => "Limburgish",
+					"ln" => "Lingala",
+					"lo" => "Lao",
+					"lt" => "Lithuanian",
+					"lu" => "Luba-Katanga",
+					"lv" => "Latvian",
+					"mg" => "Malagasy",
+					"mh" => "Marshallese",
+					"mi" => "Maori",
+					"mk" => "Macedonian",
+					"ml" => "Malayalam",
+					"mn" => "Mongolian",
+					"mr" => "Marathi",
+					"ms" => "Malay",
+					"mt" => "Maltese",
+					"my" => "Burmese",
+					"na" => "Nauru",
+					"nb" => "Norwegian Bokmål",
+					"nd" => "North Ndebele",
+					"ne" => "Nepali",
+					"ng" => "Ndonga",
+					"nl" => "Dutch",
+					"nn" => "Norwegian Nynorsk",
+					"no" => "Norwegian",
+					"nr" => "South Ndebele",
+					"nv" => "Navajo",
+					"ny" => "Chichewa",
+					"oc" => "Occitan",
+					"oj" => "Ojibwa",
+					"om" => "Oromo",
+					"or" => "Oriya",
+					"os" => "Ossetian",
+					"pa" => "Punjabi",
+					"pi" => "Pali",
+					"pl" => "Polish",
+					"ps" => "Pashto",
+					"pt" => "Portuguese",
+					"qu" => "Quechua",
+					"rm" => "Romansh",
+					"rn" => "Rundi",
+					"ro" => "Romanian",
+					"ru" => "Russian",
+					"rw" => "Kinyarwanda",
+					"sa" => "Sanskrit",
+					"sc" => "Sardinian",
+					"sd" => "Sindhi",
+					"se" => "Northern Sami",
+					"sg" => "Sango",
+					"si" => "Sinhala",
+					"sk" => "Slovak",
+					"sl" => "Slovenian",
+					"sm" => "Samoan",
+					"sn" => "Shona",
+					"so" => "Somali",
+					"sq" => "Albanian",
+					"sr" => "Serbian",
+					"ss" => "Swati",
+					"st" => "Southern Sotho",
+					"su" => "Sundanese",
+					"sv" => "Swedish",
+					"sw" => "Swahili",
+					"ta" => "Tamil",
+					"te" => "Telugu",
+					"tg" => "Tajik",
+					"th" => "Thai",
+					"ti" => "Tigrinya",
+					"tk" => "Turkmen",
+					"tl" => "Tagalog",
+					"tn" => "Tswana",
 					"to" => "Tonga",
-					"tt" => "Trinidad and Tobago",
-					"tn" => "Tunisia",
-					"tr" => "Turkey",
-					"tm" => "Turkmenistan",
-					"ug" => "Uganda",
-					"ua" => "Ukraine",
-					"ae" => "United Arab Emirates",
-					"gb" => "United Kingdom",
-					"us" => "United States",
-					"uy" => "Uruguay",
-					"uz" => "Uzbekistan",
-					"vu" => "Vanuatu",
-					"ve" => "Venezuela",
-					"vn" => "Vietnam",
-					"vg" => "Virgin Islands, British",
-					"vi" => "Virgin Islands, U.S.",
-					"ye" => "Yemen",
-					"zm" => "Zambia",
-					"zw" => "Zimbabwe"
+					"tr" => "Turkish",
+					"ts" => "Tsonga",
+					"tt" => "Tatar",
+					"tw" => "Twi",
+					"ty" => "Tahitian",
+					"ug" => "Uyghur",
+					"uk" => "Ukrainian",
+					"ur" => "Urdu",
+					"uz" => "Uzbek",
+					"ve" => "Venda",
+					"vi" => "Vietnamese",
+					"vo" => "Volapük",
+					"wa" => "Walloon",
+					"wo" => "Wolof",
+					"xh" => "Xhosa",
+					"yi" => "Yiddish",
+					"yo" => "Yoruba",
+					"za" => "Zhuang",
+					"zh" => "Chinese",
+					"zh-cn" => "Chinese (Simplified)",
+					"zh-tw" => "Chinese (Traditional)",
+					"zu" => "Zulu"
 				]
 			],
 			"nsfw" => [
 				"display" => "NSFW",
 				"option" => [
 					"yes" => "Yes",
-					"maybe" => "Maybe",
 					"no" => "No"
 				]
 			]
 		];
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
 		
 		$curlproc = curl_init();
 		
@@ -252,32 +227,43 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		// set ciphers
-		curl_setopt(
-			$curlproc,
-			CURLOPT_SSL_CIPHER_LIST,
-			"aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha"
-		);
+		// use http1 lol
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: https://yep.com/",
-			"Origin: https://yep.com",
-			"DNT: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"Priority: u=4",
-			"TE: trailers"]
-		);
+		
+		if($use_api){
+			
+			$post_data = json_encode($post_data);
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["Content-Type: application/json",
+				"Authorization: Bearer $bearer",
+				"Content-Length: " . strlen($post_data)]
+			);
+			
+			curl_setopt($curlproc, CURLOPT_POST, true);
+			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: */*",
+				"Accept-Language: en-US,en;q=0.9",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"Referer: https://yep.com/",
+				"Origin: https://yep.com",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Sec-Fetch-Dest: empty",
+				"Sec-Fetch-Mode: cors",
+				"Sec-Fetch-Site: same-site",
+				"Priority: u=4",
+				"Pragma: no-cache",
+				"Cache-Control: no-cache",
+				"TE: trailers"]
+			);
+		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -302,22 +288,17 @@ class yep{
 	
 	public function web($get){
 		
+		if(config::YEP_USE_API){
+			
+			return $this->web_api($get);
+		}
+		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -334,23 +315,25 @@ class yep{
 			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"limit" => 100, // wwwwwwwwwwwwwww
+			"query" => $search,
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
+		if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
+		
 		try{
 			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
+			// https://api.yep.com/search?limit=20&query=asmr
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "web"
-					]
+					"https://api.yep.com/search",
+					$filters
 				);
+			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
@@ -414,7 +397,7 @@ class yep{
 									)
 								),
 							"url" => $item["url"],
-							"date" => strtotime($item["first_seen"]),
+							"date" => null,
 							"type" => "web",
 							"thumb" => [
 								"url" => null,
@@ -428,83 +411,11 @@ class yep{
 			}
 		}
 		
-		if(isset($json[1]["featured_news"])){
-			
-			foreach($json[1]["featured_news"] as $news){
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$news["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($news["first_seen"]),
-					"thumb" =>
-						isset($news["img"]) ?
-						[
-							"url" => $this->unshiturl($news["img"]),
-							"ratio" => "16:9"
-						] :
-						[
-							"url" => null,
-							"ratio" => null
-						],
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		if(isset($json[1]["featured_images"])){
-			
-			foreach($json[1]["featured_images"] as $image){
-				
-				if(
-					$image["width"] !== 0 &&
-					$image["height"] !== 0
-				){
-					
-					$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
-					$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
-					
-					$width = $image["width"];
-					$height = $image["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $image["title"],
-					"source" => [
-						[
-							"url" => $image["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $image["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $image["host_page"]
-				];
-			}
-		}
-		
 		return $out;
 	}
 	
 	
-	
-	public function image($get){
+	private function web_api($get){
 		
 		$search = $get["s"];
 		if(strlen($search) === 0){
@@ -512,142 +423,53 @@ class yep{
 			throw new Exception("Search term is empty!");
 		}
 		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
 		$out = [
 			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
 			"npt" => null,
-			"image" => []
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
 		];
 		
+		// parse filters
+		$filters = [
+			"query" => $search,
+			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
+		];
+		
+		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
+		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
+		
+		// add api key
+		$key_data = $this->backend->get_key();
+		
 		try{
 			
 			$json =
 				$this->get(
-					$this->backend->get_ip(), // no nextpage!
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "images"
-					]
+					$this->backend->get_ip($key_data["increment"]),
+					"https://platform.yep.com/api/search",
+					[],
+					true,
+					$filters,
+					$key_data["key"]
 				);
+			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
 		}
 		
-		$this->detect_cf($json);
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
-				
-				if(
-					$item["width"] !== 0 &&
-					$item["height"] !== 0
-				){
-					
-					$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
-					$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
-					
-					$width = $item["width"];
-					$height = $item["height"];
-				}else{
-					
-					$thumb_width = null;
-					$thumb_height = null;
-					$width = null;
-					$height = null;
-				}
-				
-				$out["image"][] = [
-					"title" => $item["title"],
-					"source" => [
-						[
-							"url" => $item["image_id"],
-							"width" => $width,
-							"height" => $height
-						],
-						[
-							"url" => $item["src"],
-							"width" => $thumb_width,
-							"height" => $thumb_height
-						]
-					],
-					"url" => $item["host_page"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
-	
-	public function news($get){
-		
-		$search = $get["s"];
-		if(strlen($search) === 0){
-			
-			throw new Exception("Search term is empty!");
-		}
-		
-		$country = $get["country"];
-		$nsfw = $get["nsfw"];
-		
-		switch($nsfw){
-			
-			case "yes": $nsfw = "off"; break;
-			case "maybe": $nsfw = "moderate"; break;
-			case "no": $nsfw = "strict"; break;
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		try{
-			
-			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
-			$json =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://api.yep.com/fs/2/search",
-					[
-						"client" => "web",
-						"gl" => $country == "all" ? $country : strtoupper($country),
-						"limit" => "99999",
-						"no_correct" => "false",
-						"q" => $search,
-						"safeSearch" => $nsfw,
-						"type" => "news"
-					]
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$this->detect_cf($json);
+		// should never happen
+		//$this->detect_cf($json);
 		
 		$json = json_decode($json, true);
 		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
@@ -657,34 +479,53 @@ class yep{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		if(isset($json[1]["results"])){
-			foreach($json[1]["results"] as $item){
+		if(isset($json["error"])){
 			
-				$out["news"][] = [
-					"title" => $item["title"],
-					"author" => null,
-					"description" =>
-						$this->titledots(
-							strip_tags(
-								html_entity_decode(
-									$item["snippet"]
-								)
-							)
-						),
-					"date" => strtotime($item["first_seen"]),
-					"thumb" =>
-						isset($item["img"]) ?
-						[
-							"url" => $this->unshiturl($item["img"]),
-							"ratio" => "16:9"
-						] :
-						[
-							"url" => null,
-							"ratio" => null
-						],
-					"url" => $item["url"]
-				];
+			throw new Exception("Yep API returned an error: " . $json["error"]);
+		}
+		
+		if(isset($json["errors"])){
+			
+			throw new Exception("Yep API returned the following errors: {$json["message"]}");
+		}
+		
+		if(
+			isset($json["success"]) &&
+			$json["success"] !== true
+		){
+			
+			throw new Exception("Yep API returned a false-y success value");
+		}
+		
+		if(!isset($json["results"])){
+			
+			throw new Exception("Yep API did not return a results object");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			if(
+				$item["url"] === null ||
+				$item["url"] == ""
+			){
+				
+				// sometimes API fucks up
+				continue;
 			}
+			
+			$out["web"][] = [
+				"title" => $item["title"],
+				"description" => $item["description"],
+				"url" => $item["url"],
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
 		}
 		
 		return $out;

settings.php

@@ -87,14 +87,6 @@ $settings = [
 						"value" => "kagi",
 						"text" => "Kagi"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -133,30 +125,26 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "google_api",
+						"text" => "Google API"
+					],
+					[
+						"value" => "google_cse",
+						"text" => "Google CSE"
+					],
+					[
+						"value" => "yahoo_japan",
+						"text" => "Yahoo! JAPAN",
+					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
 					],
-					[
-						"value" => "greppr",
-						"text" => "Greppr"
-					],
-					[
-						"value" => "crowdview",
-						"text" => "Crowdview"
-					],
 					[
 						"value" => "mwmbl",
 						"text" => "Mwmbl"
@@ -165,6 +153,18 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
+					[
+						"value" => "naver",
+						"text" => "Naver"
+					],
+					[
+						"value" => "baidu",
+						"text" => "Baidu"
+					],
+					[
+						"value" => "coccoc",
+						"text" => "Cốc Cốc"
+					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
@@ -174,12 +174,12 @@ $settings = [
 						"text" => "Marginalia"
 					],
 					[
-						"value" => "wiby",
-						"text" => "wiby"
+						"value" => "purili",
+						"text" => "Purili"
 					],
 					[
-						"value" => "curlie",
-						"text" => "Curlie"
+						"value" => "wiby",
+						"text" => "wiby"
 					]
 				]
 			],
@@ -203,26 +203,62 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "google_api",
+						"text" => "Google API"
+					],
+					[
+						"value" => "google_cse",
+						"text" => "Google CSE"
+					],
+					[
+						"value" => "yahoo_japan",
+						"text" => "Yahoo! JAPAN",
+					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
 					[
-						"value" => "qwant",
-						"text" => "Qwant"
+						"value" => "naver",
+						"text" => "Naver"
 					],
 					[
-						"value" => "yep",
-						"text" => "Yep"
+						"value" => "baidu",
+						"text" => "Baidu"
 					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
 					],
-					/*[
+					[
 						"value" => "pinterest",
 						"text" => "Pinterest"
-					],*/
+					],
+					[
+						"value" => "flickr",
+						"text" => "Flickr"
+					],
+					[
+						"value" => "pexels",
+						"text" => "Pexels"
+					],
+					[
+						"value" => "pixabay",
+						"text" => "Pixabay"
+					],
+					[
+						"value" => "unsplash",
+						"text" => "Unsplash"
+					],
+					[
+						"value" => "fivehpx",
+						"text" => "500px"
+					],
+					[
+						"value" => "vsco",
+						"text" => "VSCO"
+					],
 					[
 						"value" => "imgur",
 						"text" => "Imgur"
@@ -241,6 +277,14 @@ $settings = [
 						"value" => "yt",
 						"text" => "YouTube"
 					],
+					[
+						"value" => "vimeo",
+						"text" => "Vimeo"
+					],
+					[
+						"value" => "sepiasearch",
+						"text" => "Sepia Search"
+					],
 					[
 						"value" => "ddg",
 						"text" => "DuckDuckGo"
@@ -257,13 +301,29 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "yahoo_japan",
+						"text" => "Yahoo! JAPAN",
+					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
 					[
-						"value" => "qwant",
-						"text" => "Qwant"
+						"value" => "naver",
+						"text" => "Naver"
+					],
+					[
+						"value" => "baidu",
+						"text" => "Baidu"
+					],
+					[
+						"value" => "coccoc",
+						"text" => "Cốc Cốc"
+					],
+					[
+						"value" => "purili",
+						"text" => "Purili"
 					],
 					[
 						"value" => "solofield",
@@ -287,21 +347,21 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
+					[
+						"value" => "yahoo_japan",
+						"text" => "Yahoo! JAPAN",
+					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
-					[
-						"value" => "yep",
-						"text" => "Yep"
-					],
-					[
+					/*[
 						"value" => "mojeek",
 						"text" => "Mojeek"
+					],*/
+					[
+						"value" => "baidu",
+						"text" => "Baidu"
 					]
 				]
 			],
@@ -312,11 +372,15 @@ $settings = [
 					[
 						"value" => "sc",
 						"text" => "SoundCloud"
-					]//,
-					//[
-					//	"value" => "spotify",
-					//	"text" => "Spotify"
-					//]
+					],
+					[
+						"value" => "swisscows",
+						"text" => "Swisscows"
+					]
+					/*[
+						"value" => "spotify",
+						"text" => "Spotify"
+					]*/
 				]
 			]
 		]

static/serverping.js

@@ -22,7 +22,7 @@ var list = [];
 var pinged_list = [];
 var reqs = 0;
 var errors = 0;
-var sort = 6; // highest version first
+var sort = 12;
 
 // check for instance redirect stuff
 var redir = [];
@@ -68,8 +68,8 @@ table.innerHTML =
 		'<th class="extend">Server</th>' +
 		'<th>Address</th>' +
 		'<th>Bot protection</th>' +
-		'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
-		'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
+		'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
+		'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
 		'<th>API</th>' +
 		'<th><div class="arrow up"></div>Version</th>' +
 	'</tr>' +
@@ -80,6 +80,15 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
+
+// server popup without inline javascript
+tbody.addEventListener('click', function(event){
+	var row = event.target.closest('.show_server');
+	if(!row) return;
+
+	show_server(row.id);
+})
+
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -343,7 +352,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
+		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -406,6 +415,7 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
+
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/style.css

@@ -1013,6 +1013,23 @@ table tr a:last-child{
 	font-weight:bold;
 }
 
+/* booru */
+#images .image .i-title-wrapper{
+	margin-bottom:7px;
+	white-space:nowrap;
+	overflow:hidden;
+	text-overflow:ellipsis;
+}
+#images .image .i-title-wrapper a{
+	display:inline;
+}
+
+#images .image .i-title-wrapper .title{
+	display:inline;
+	margin-bottom:0;
+}
+/* end booru */
+
 #images .image .description{
 	overflow:hidden;
 	height:45px;
@@ -1113,6 +1130,17 @@ table tr a:last-child{
 /*
 	Settings page
 */
+.web .settings select{
+	all:unset;
+	user-select:none;
+	display:block;
+	border:1px solid var(--504945);
+	border-radius:2px;
+	font-size:14px;
+	padding:0 20px 0 4px;
+	height:24px;
+}
+
 .web .settings{
 	margin-top:17px;
 	border:1px dashed var(--504945);

static/themes/Gore's shitty theme.css

@@ -1,47 +1,45 @@
 :root{
-	--1d2021: #1d2021;
-	--282828: #282828;
-	--3c3836: #3c3836;
-	--504945: #504945;
+	--1d2021:#1d2021;
+	--282828:#282828;
+	--3c3836:#3c3836;
+	--504945:#504945;
 
 	/* font */
-	--928374: #928374;
-	--a89984: #c9c5bf;
-	--bdae93: #bdae93;
-	--8ec07c: #8ec07c;
-	--ebdbb2: #ebdbb2;
+	--928374:#928374;
+	--a89984:#c9c5bf;
+	--bdae93:#bdae93;
+	--8ec07c:#8ec07c;
+	--ebdbb2:#ebdbb2;
 }
 
-
-
 body{
 	padding:15px 4% 40px;
+	margin:unset;
 }
 
-h1,h2,h3,h4,h5,h6{
+h1, h2, h3, h4, h5, h6{
 	padding:0;
 	margin:0 0 7px 0;
 	line-height:initial;
 	color:var(--bdae93);
 }
 
-h3,h4,h5,h6{
+h3, h4, h5, h6{
 	margin-bottom:14px;
 }
 
 /*
-	Web styles
+  Web styles
 */
 
 .searchbox input[type="submit"]{
 	float:right;
 	cursor:pointer;
 	padding:0 10px;
-  border-left: 1px solid var(--504945);
-  background: #723c0b;
+	border-left:1px solid var(--504945);
+	background:#723c0b;
 }
 
-
 .searchbox input{
 	all:unset;
 	line-height:36px;
@@ -96,7 +94,6 @@ h3,h4,h5,h6{
 	display:inline-block;
 }
 
-
 .tabs .tab.selected{
 	border-bottom:2px solid #fc92a5;
 }
@@ -106,7 +103,7 @@ h3,h4,h5,h6{
 	padding-bottom:12px;
 	padding-top:7px;
 	margin-bottom:7px;
-	background-color:#232525
+	background-color:#232525;
 }
 
 .filters .filter{
@@ -169,7 +166,6 @@ h3,h4,h5,h6{
 	font-size:12px;
 }
 
-
 .web .hover{
 	display:block;
 	text-decoration:none;
@@ -193,16 +189,13 @@ h3,h4,h5,h6{
 	color:#9760b1 !important;
 }
 
-
 .web .text-result .greentext{
 	font-size:14px;
 	color:var(--bdae93);
 }
 
-
 /* favicon */
 
-
 .favicon-dropdown a{
 	text-decoration:none;
 	color:#d3d0c1;
@@ -211,39 +204,33 @@ h3,h4,h5,h6{
 	font-size:13px;
 }
 
-
-.web .favicon img,
-.favicon-dropdown img{
+.web .favicon img, .favicon-dropdown img{
 	margin:3px 7px 0 0;
 	height:16px;
 	font-size:12px;
-	line-height:16px;;
+	line-height:16px;
 	display:block;
 	text-align:left;
 }
 
-
 .web .sublinks{
 	padding:17px 10px;
 	font-size:15px;
 	color:var(--#928374);
 }
 
-
 .web .text-result .sublinks:last-child{
 	padding-bottom:0;
 }
 
-
 /* Wikipedia head */
 .wiki-head{
 	padding:5px;
-	background-color: #322f2b
+	background-color:#322f2b;
 }
 
-
 /*
-	Images tab
+  Images tab
 */
 
 #images{
@@ -257,17 +244,14 @@ h3,h4,h5,h6{
 	float:left;
 }
 
-
-
 #images .image .title{
 	white-space:nowrap;
 	overflow:hidden;
 	margin-bottom:7px;
 	font-weight:bold;
-    color:var(--bdae93);
+	color:var(--bdae93);
 }
 
-
 #popup-status{
 	display:none;
 	position:fixed;
@@ -280,43 +264,59 @@ h3,h4,h5,h6{
 }
 
 /*
-	Settings page
+  Settings page
 */
 
-
 .web .settings-submit a{
 	margin-right:17px;
 	color:#bdae93;
 }
 
+/*
+  Responsive image
+*/
+@media only screen and (max-width:1454px){
+	#images .image-wrapper{
+		width:25%;
+	}
+}
+
+@media only screen and (max-width:1161px){
+	#images .image-wrapper{
+		width:25%;
+	}
+}
+
+@media only screen and (max-width:750px){
+	#images .image-wrapper{
+		width:50%;
+	}
+}
+
+@media only screen and (max-width:450px){
+	#images .image-wrapper{
+		width:100%;
+	}
+}
 
 /*
-	Responsive image
+  Responsive design
 */
-@media only screen and (max-width: 1454px){ #images .image-wrapper{ width:25%; } }
-@media only screen and (max-width: 1161px){ #images .image-wrapper{ width:25%; } }
-@media only screen and (max-width: 750px){ #images .image-wrapper{ width:50%; } }
-@media only screen and (max-width: 450px){ #images .image-wrapper{ width:100%; } }
 
-
-/*
-	Responsive design
-*/
-@media only screen and (max-width: 1550px){
-	
-	
-	.web .left,
+@media only screen and (max-width:1550px){
+.web .left,
 	.searchbox{
 		width:60%;
 	}
-
 }
 
-@media only screen and (max-width: 1000px){
-
+@media only screen and (max-width:1100px){
+.web .left,
+	.searchbox{
+		width:100%;
+	}
 }
 
-    .type{
-    color:var(--bdae93);
-    }
+.type{
+	color:var(--bdae93);
 }

static/themes/Kuuro.css

@@ -0,0 +1,17 @@
+:root{
+	--1d2021: #101010;
+	--282828: #1a1a1a;
+	--3c3836: #1e1e1e;
+	--504945: #232323;
+
+	--928374: #949494;
+	--a89984: #d2d2d2;
+	--bdae93: #d2d2d2;
+	--8ec07c: #99c794;
+	--ebdbb2: #d2d2d2;
+
+	--comment: #5f6364;
+	--default: #cccece;
+	--keyword: #c594c5;
+	--string:  #99c794;
+}

static/themes/Rosé Pine.css

@@ -0,0 +1,20 @@
+:root{
+	/* background */
+	--1d2021: #191724;
+	--282828: #1f1d2e;
+	--3c3836: #26233a;
+	--504945: #524f67;
+
+	/* font */
+	--928374: #e0def4;
+	--a89984: #908caa;
+	--bdae93: #6e6a86;
+	--8ec07c: #9ccfd8;
+	--ebdbb2: #f6c177;
+
+	/* code highlighter */
+	--comment: #6e6a86;
+	--default: #c4a7e7;
+	--keyword: #31748f;
+	--string: #f6c177;
+}

static/themes/Wine.css

@@ -0,0 +1,40 @@
+:root
+{
+        --accent : #f79e98;
+        --1d2021 : #180d0c;
+        --282828 : #180d0c;
+        --3c3836 : #251615;
+        --504945 : #251615;
+        --928374 : var(--accent);
+        --a89984 : #d8c5c4;
+        --bdae93 : #d8c5c4;
+        --8ec07c : var(--accent);
+        --ebdbb2 : #d8c5c4;
+        --comment: #928374;
+        --default: #DCC9BC;
+        --keyword: #F07342;
+        --string : var(--accent);
+        --green  : #959A6B;
+        --yellow : #E39C45;
+        --red    : #CF223E;
+        --white  : var(--a89984);
+        --black  : var(--1d2021);
+        --hover  : #b18884
+}
+
+      a.link, a { color: var(--accent); text-decoration: none; }
+      .searchbox { width: 23%; }
+      .filters filter select { color: #E39C45; }
+      .web .separator::before { color: var(--white) }
+      .searchbox input[type="text"]::placeholder { color: var(--white); }
+      a.link:hover
+      {
+        color: var(--hover);
+        text-shadow: 0 0 .2rem var(--hover);
+      }
+      .code-inline
+      { border-color: var(--default); font-family: monospace;}
+      .home #center a
+      { color: var(--accent); }
+      .home .subtext
+      { color: var(--white); }

static/themes/evergarden.css

@@ -0,0 +1,25 @@
+:root {
+	/* background */
+	--1d2021: #232a2e;
+	--282828: #2b3337;
+	--3c3836: #374145;
+	--504945: #4a585c;
+
+	/* font */
+	--928374: #839e9a;
+	--a89984: #adc9bc;
+	--bdae93: #f8f9e8;
+	--8ec07c: #cbe3b3;
+	--ebdbb2: #adc9bc;
+
+	/* code highlighter */
+	--comment: #839e9a;
+	--default: #f8f9e8;
+	--keyword: #6f8788;
+	--string: #cbe3b3;
+
+	/* color codes for instance list */
+	--green: #cbe3b3;
+	--yellow: #f5d098;
+	--red: #f57f82;
+}

template/donate.html

@@ -1,9 +1,18 @@
 <a href="/" class="link">&lt; Go back</a>
 
 <h1>Donate to the project</h1>
-This project takes up most of my free time and costs money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
+This project takes up most of my free time and costs (a lot of) money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
 
-To run smoothly, 4get requires 20$/month for the server fees and 15-25$ for residential proxies. I also have plans to build my own index to provide better search results; funds raised here will go directly towards better hardware for these purposes. According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> serves between 800-1400 users every day, or around 7000-10000 searches!
+To run smoothly, 4get requires:
+<ul>
+	<li>$350/year for server hosting at OVH</li>
+	<li>$60-80/month for proxies</li>
+	<li>$54/month for home internet</li>
+</ul>
+
+I also bought a dedicated machine for browser rendering hosted at home coming at around $380.<br><br>
+
+According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> currently serves between <a href="https://4get.ca/ami4get">2000-2500 users every day</a>, and serves around <a href="https://4get.ca/api/v2/graph">20,000-22,000 searches</a>!
 
 <h2>Ko-fi</h2>
 This is the most convenient way to donate. Supports PayPal.
@@ -17,4 +26,4 @@ Due to popular demand, I have added an XMR addy. I'm planning to reward users wh
 	<li><a href="monero:85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB?tx_description=4get+donation">85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB</a></li>
 </ul>
 
-Thank you all for your generous support!!
+I'm immensely grateful for every donation. Thank you all for your generous support!!

template/header.html

@@ -15,7 +15,7 @@
 		<div class="navigation">
 			<a href="/">Home</a>
 			<a href="/settings">Settings</a>
-			<a href="https://git.lolcat.ca/lolcat/4get_news" target="_BLANK">News</a>
+			<a href="https://bsky.app/profile/4getisdown.bsky.social" target="_BLANK">News</a>
 		</div>
 		<form method="GET" autocomplete="off">
 			<div class="searchbox">