Add actions support for 4get docker image builds

.dockerignore

@@ -1 +0,0 @@
-.git

.gitea/workflows/ci.yml

@@ -0,0 +1,48 @@
+name: '4get CI'
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - '*'
+    paths-ignore:
+      - 'README.md'
+      - 'docker-compose.yaml'
+      - '.gitignore'
+      - 'docs/**'
+
+jobs:
+  build:
+    runs-on: docker
+
+    steps:
+    - uses: actions/checkout@v4
+      name: Checkout 4get repository
+
+    - uses: docker/setup-buildx-action@v3
+      name: Setup Docker BuildX system
+
+    - name: Login to Docker Container Registry
+      uses: docker/login-action@v3
+      with:
+        registry: git.lolcat.ca
+        username: ${{ secrets.USERNAME }}
+        password: ${{ secrets.TOKEN }}
+
+    - name: Docker meta
+      id: meta
+      uses: docker/metadata-action@v5
+      with:
+        images: git.lolcat.ca/ckg/4get
+        tags: |
+          type=sha,format=short,prefix={{date 'YYYY.MM.DD'}}-,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+          type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'master') }}
+
+    - uses: docker/build-push-action@v6
+      name: Build images
+      with:
+        context: .
+        file: Dockerfile
+        tags: ${{ steps.meta.outputs.tags }}
+        platforms: linux/amd64
+        push: true

.gitignore

@@ -26,4 +26,3 @@ data/captcha/minecraft/
 !banner/*default*
 scraper/curlie.html
 icons/*
-node_modules/

Dockerfile

@@ -1,18 +1,10 @@
-FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate
+FROM alpine:latest
-
-FROM alpine:3.21
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates 
+RUN apk add php apache2-ssl php83-fileinfo php83-openssl php83-iconv php83-common php83-dom php83-sodium php83-curl curl php83-pecl-apcu php83-apache2 imagemagick php83-pecl-imagick php-mbstring imagemagick-webp imagemagick-jpeg
-
-COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
-COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
-
-ENV LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so
-ENV CURL_IMPERSONATE=ff117
-ENV CURL_IMPERSONATE_HEADERS=no
 
+COPY ./docker/apache/ /etc/apache2/
 COPY . .
 
 RUN chmod 777 /var/www/html/4get/icons
@@ -22,5 +14,4 @@ EXPOSE 443
 
 ENV FOURGET_PROTO=http
 
-ENTRYPOINT  ["./docker/docker-entrypoint.sh"]
+CMD  ["./docker/docker-entrypoint.sh"]
-CMD ["start"]

README.md

@@ -9,15 +9,13 @@ https://4get.ca/about
 ## Official instance
 https://4get.ca , or visit the official instance list: https://4get.ca/instances
 
-_NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
-
 ## Totally unbiased comparison between alternatives
 
-|                            | 4get                    | searx(ng) | whoogle    | degoog                                    |
+|                            | 4get                    | searx(ng) | libreY      | araa      | hearch            |
-|----------------------------|-------------------------|-----------|------------|-------------------------------------------|
+|----------------------------|-------------------------|-----------|-------------|-----------|-------------------|
-| RAM usage                  | 100-400mb~              | 400mb-1GB | 100mb      | 100mb-250mb                               |
+| RAM usage                  | 200-400mb~              | 2GB~      | 200-400mb~  | 2GB~      | idk               |
-| Does it suck               | no (debunked by snopes) | yes       | kind of?   | hit and miss with search filters          |
+| Does it suck               | no (debunked by snopes) | yes       | yes         | a little  | better than searx |
-| Does it work               | ye                      | lmao      | shits dead | works $rightNow, it's actually kinda cool |
+| Does it work               | ye                      | sometimes | sometimes   | sometimes | yes               |
 
 ## Features
 1. Rotating proxies on a per-scraper basis
@@ -25,41 +23,36 @@ _NOT to be confused with 4get.ch, 4get.lol and friends! I **don't** host these._
 3. Bot protection that *actually* filters out the bots (when configured)
 4. Interface doesn't require javascript
 5. Favicon fetcher with caching support & image proxy
-6. Bunch of other shits
+6. Bunch of other shit
 
-tl;dr 4get is the best way to browse for shit.
+tl;dr the best way to actually browse for shit.
 
 # Supported websites
 
-| web          | images       | videos       | news         | music      | autocomplete |
+| Web        | Images       | Videos     | News       | Music      | Autocompleter |
-|--------------|--------------|--------------|--------------|------------|--------------|
+|------------|--------------|------------|------------|------------|---------------|
-| DuckDuckGo   | DuckDuckGo   | YouTube      | DuckDuckGo   | SoundCloud | Brave        |
+| DuckDuckGo | DuckDuckGo   | YouTube    | DuckDuckGo | Soundcloud | Brave         |
-| Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
+| Brave      | Brave        | DuckDuckGo | Brave      |            | DuckDuckGo    |
-| Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
+| Yandex     | Yandex       | Brave      | Google     |            | Yandex        |
-| Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
+| Google     | Google       | Yandex     | Startpage  |            | Google        |
-| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
+| Startpage  | Startpage    | Google     | Qwant      |            | Startpage     |
-| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
+| Qwant      | Qwant        | Startpage  | Mojeek     |            | Kagi          |
-| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
+| Ghostery   | Yep          | Qwant      |            |            | Qwant         |
-| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
+| Yep        | Solofield    | Solofield  |            |            | Ghostery      |
-| Qwant        | Qwant        | Startpage    |              |            | Yep          |
+| Greppr     | Imgur        |            |            |            | Yep           |
-| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
+| Crowdview  | FindThatMeme |            |            |            | Marginalia    |
-| Yep          | Solofield    | Baidu        |              |            | YouTube      |
+| Mwmbl      |              |            |            |            | YouTube       |
-| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
+| Mojeek     |              |            |            |            | Soundcloud    |
-| Mojeek       | Cara         | Solofield    |              |            |              |
+| Solofield  |              |            |            |            |               |
-| Baidu        | Flickr       |              |              |            |              |
+| Marginalia |              |            |            |            |               |
-| Coc Coc      | Pexels       |              |              |            |              |
+| wiby       |              |            |            |            |               |
-| Solofield    | Pixabay      |              |              |            |              |
+| Curlie     |              |            |            |            |               |
-| Marginalia   | Unsplash     |              |              |            |              |
-| wiby         | 500px        |              |              |            |              |
-|              | VSCO         |              |              |            |              |
-|              | Imgur        |              |              |            |              |
-|              | FindThatMeme |              |              |            |              |
 
 # Installation
 Refer to the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/">documentation index</a>. I recommend following the <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">apache2 guide</a>.
 
 ## Contact
-Shit breaks all the time but I repair it all the time too. Email me here: <b>will (at) lolcat.ca</b> or create an issue.
+Shit breaks all the time but I repair it all the time too... Email me here: <b>will (at) lolcat.ca</b> or create an issue.
 
 ## License
-AGPLv3-only
+AGPL

api.txt

@@ -1,16 +1,9 @@
-                   44
+                        __ __             __
-                 4444444      44   
+                       / // / ____ ____  / /_
-                 44444444   44444       444
+                      / // /_/ __ `/ _ \/ __/
-                 44444444  444444    444444444
+                     /__  __/ /_/ /  __/ /_ 
-                  44444   44444444  444444444
+                       /_/  \__, /\___/\__/
-                         444444444   4444444
+                           /____/         
-                        4444444444    444444
-                      4444444444444
-                    444444444444444444
-                       444444444444444
-                          44444444
-                          4444
-                           44
 
            + Welcome to the 4get API documentation +
 

audio/linear.php

@@ -0,0 +1,20 @@
+<?php
+
+if(!isset($_GET["s"])){
+	
+	http_response_code(404);
+	header("X-Error: No SOUND(s) provided!");
+	die();
+}
+
+include "../data/config.php";
+include "../lib/curlproxy.php";
+$proxy = new proxy();
+
+try{
+	
+	$proxy->stream_linear_audio($_GET["s"]);
+}catch(Exception $error){
+	
+	header("X-Error: " . $error->getMessage());
+}

audio/sc.php

@@ -0,0 +1,224 @@
+<?php
+
+new sc_audio();
+
+class sc_audio{
+	
+	public function __construct(){
+		
+		include "../data/config.php";
+		include "../lib/curlproxy.php";
+		$this->proxy = new proxy();
+		
+		if(isset($_GET["u"])){
+			
+			/*
+				we're now proxying audio
+			*/
+			$viewkey = $_GET["u"];
+			
+			if(!isset($_GET["r"])){
+				
+				$this->do404("Ranges(r) are missing");
+			}
+			
+			$ranges = explode(",", $_GET["r"]);
+			
+			// sanitize ranges
+			foreach($ranges as &$range){
+				
+				if(!is_numeric($range)){
+					
+					$this->do404("Invalid range specified");
+				}
+				
+				$range = (int)$range;
+			}
+			
+			// sort ranges (just to make sure)
+			sort($ranges);
+			
+			// convert ranges to pairs
+			$last = -1;
+			foreach($ranges as &$r){
+				
+				$tmp = $r;
+				$r = [$last + 1, $r];
+				
+				$last = $tmp;
+			}
+			
+			$browser_headers = getallheaders();
+			
+			// get the requested range from client
+			$client_range = 0;
+			foreach($browser_headers as $key => $value){
+				
+				if(strtolower($key) == "range"){
+					
+					preg_match(
+						'/bytes=([0-9]+)/',
+						$value,
+						$client_regex
+					);
+					
+					if(isset($client_regex[1])){
+						
+						$client_range = (int)$client_regex[1];
+					}else{
+						
+						$client_range = 0;
+					}
+					break;
+				}
+			}
+			
+			if(
+				$client_range < 0 ||
+				$client_range > $ranges[count($ranges) - 1][1]
+			){
+				
+				// range is not satisfiable
+				http_response_code(416);
+				header("Content-Type: text/plain");
+				die();
+			}
+			
+			$rng = null;
+			for($i=0; $i<count($ranges); $i++){
+				
+				if($ranges[$i][0] <= $client_range){
+					
+					$rng = $ranges[$i];
+				}
+			}
+			
+			// proxy data!
+			http_response_code(206); // partial content
+			header("Accept-Ranges: bytes");
+			header("Content-Range: bytes {$rng[0]}-{$rng[1]}/" . ($ranges[count($ranges) - 1][1] + 1));
+			
+			$viewkey =
+				preg_replace(
+					'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
+					'/media/$1/' . $rng[0] . '/' . $rng[1],
+					$viewkey
+				);
+			
+			try{
+				
+				$this->proxy->stream_linear_audio(
+					$viewkey
+				);
+			}catch(Exception $error){
+				
+				$this->do404("Could not read stream");
+			}
+			
+			die();
+		}
+		
+		/*
+			redirect user to correct resource
+			we need to scrape and store the byte positions in the result URL
+		*/
+		if(!isset($_GET["s"])){
+			
+			$this->do404("The URL(s) parameter is missing");
+		}
+		
+		$viewkey = $_GET["s"];
+		
+		if(
+			preg_match(
+				'/soundcloud\.com$/',
+				parse_url($viewkey, PHP_URL_HOST)
+			) === false
+		){
+			
+			$this->do404("This endpoint can only be used for soundcloud streams");
+		}
+		
+		try{
+			
+			$json = $this->proxy->get($viewkey)["body"];
+		}catch(Exception $error){
+			
+			$this->do404("Curl error: " . $error->getMessage());
+		}
+		
+		$json = json_decode($json, true);
+		
+		if(!isset($json["url"])){
+			
+			$this->do404("Could not get URL from JSON");
+		}
+		
+		$viewkey = $json["url"];
+		
+		$m3u8 = $this->proxy->get($viewkey)["body"];
+		
+		$m3u8 = explode("\n", $m3u8);
+		
+		$lineout = null;
+		$streampos_arr = [];
+		foreach($m3u8 as $line){
+			
+			$line = trim($line);
+			if($line[0] == "#"){
+				
+				continue;
+			}
+			
+			if($lineout === null){
+				$lineout = $line;
+			}
+			
+			preg_match(
+				'/\/media\/[0-9]+\/([0-9]+)\/([0-9]+)/',
+				$line,
+				$matches
+			);
+			
+			if(isset($matches[0])){
+				
+				$streampos_arr[] = [
+					(int)$matches[1],
+					(int)$matches[2]
+				];
+			}
+		}
+		
+		if($lineout === null){
+			
+			$this->do404("Could not get stream URL");
+		}
+		
+		$lineout =
+			preg_replace(
+				'/\/media\/([0-9]+)\/[0-9]+\/[0-9]+/',
+				'/media/$1/0/0',
+				$lineout
+			);
+		
+		$streampos = [];
+		
+		foreach($streampos_arr as $pos){
+			
+			$streampos[] = $pos[1];
+		}
+		
+		$streampos = implode(",", $streampos);
+		
+		header("Location: /audio/sc?u=" . urlencode($lineout) . "&r=$streampos");
+		header("Accept-Ranges: bytes");
+	}
+	
+	private function do404($error){
+		
+		http_response_code(404);
+		header("Content-Type: text/plain");
+		header("X-Error: $error");
+		die();
+	}
+}

audio/seekable.php

@@ -0,0 +1,20 @@
+<?php
+
+if(!isset($_GET["s"])){
+	
+	http_response_code(404);
+	header("X-Error: No SOUND(s) provided!");
+	die();
+}
+
+include "../data/config.php";
+include "../lib/curlproxy.php";
+$proxy = new proxy();
+
+try{
+	
+	$proxy->stream_linear_audio($_GET["s"]);
+}catch(Exception $error){
+	
+	header("X-Error: " . $error->getMessage());
+}

audio/spotify.php

@@ -0,0 +1,214 @@
+<?php
+
+include "../data/config.php";
+new spotify();
+
+class spotify{
+	
+	public function __construct(){
+		
+		include "../lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+		
+		if(
+			!isset($_GET["s"]) ||
+			!preg_match(
+				'/^(track|episode)\.([A-Za-z0-9]{22})$/',
+				$_GET["s"],
+				$matches
+			)
+		){
+			
+			$this->do404("The track ID(s) parameter is missing or invalid");
+		}
+		
+		try{
+			
+			if($matches[1] == "episode"){
+				
+				$uri = "show";
+			}else{
+				
+				$uri = $matches[1];
+			}
+			
+			$embed =
+				$this->get("https://embed.spotify.com/{$uri}/" . $matches[2]);
+		}catch(Exception $error){
+			
+			$this->do404("Failed to fetch embed data");
+		}
+		
+		$this->fuckhtml->load($embed);
+		
+		$json =
+			$this->fuckhtml
+			->getElementById(
+				"__NEXT_DATA__",
+				"script"
+			);
+		
+		if($json === null){
+			
+			$this->do404("Failed to extract JSON");
+		}
+		
+		$json =
+			json_decode($json["innerHTML"], true);
+		
+		if($json === null){
+			
+			$this->do404("Failed to decode JSON");
+		}
+		
+		switch($matches[1]){
+			
+			case "track":
+				if(
+					isset(
+						$json
+						["props"]
+						["pageProps"]
+						["state"]
+						["data"]
+						["entity"]
+						["audioPreview"]
+						["url"]
+					)
+				){
+					
+					header("Content-type: audio/mpeg");
+					header(
+						"Location: /audio/linear?s=" .
+						urlencode(
+							$json
+							["props"]
+							["pageProps"]
+							["state"]
+							["data"]
+							["entity"]
+							["audioPreview"]
+							["url"]
+						)
+					);
+				}else{
+					
+					$this->do404("Could not extract playback URL");
+				}
+				break;
+			
+			case "episode":
+				if(
+					isset(
+						$json
+						["props"]
+						["pageProps"]
+						["state"]
+						["data"]
+						["entity"]
+						["id"]
+					)
+				){
+					
+					try{
+						$json =
+							$this->get(
+								"https://spclient.wg.spotify.com/soundfinder/v1/unauth/episode/" .
+								$json
+								["props"]
+								["pageProps"]
+								["state"]
+								["data"]
+								["entity"]
+								["id"] .
+								"/com.widevine.alpha"
+							);
+					}catch(Exception $error){
+						
+						$this->do404("Failed to fetch audio resource");
+					}
+					
+					$json = json_decode($json, true);
+					
+					if($json === null){
+						
+						$this->do404("Failed to decode audio resource JSON");
+					}
+					
+					if(
+						isset($json["passthrough"]) &&
+						$json["passthrough"] == "ALLOWED" &&
+						isset($json["passthroughUrl"])
+					){
+						
+						header(
+							"Location:" .
+							"/audio/linear.php?s=" .
+							urlencode(
+								str_replace(
+									"http://",
+									"https://",
+									$json["passthroughUrl"]
+								)
+							)
+						);
+					}else{
+						
+						$this->do404("Failed to find passthroughUrl");
+					}
+					
+				}else{
+					
+					$this->do404("Failed to find episode ID");
+				}
+				break;
+			}
+	}
+	
+	private function get($url){
+		
+		$headers = [
+			"User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1"
+		];
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	private function do404($error){
+		
+		http_response_code(404);
+		header("Content-Type: text/plain");
+		header("X-Error: $error");
+		die();
+	}
+}

data/api_keys/google_api.txt

@@ -1,8 +0,0 @@
-# Specify API keys for the Google API in the following format:
-# <key>
-#
-# Generate keys here:
-# https://developers.google.com/custom-search/v1/overview
-# Make sure to use a different Google account for each key, cause I'm
-# pretty sure the ratelimit is on a per-account basis :P
-#

data/api_keys/yep.txt

@@ -1 +0,0 @@
-# Paste Yep API keys here

data/config.php

@@ -23,22 +23,6 @@ class config{
 	// Enable the API?
 	const API_ENABLED = true;
 	
-	//
-	// 4play settings
-	//
-	// Enable 4get's /api/v2/provide_sesh endpoint
-	const FPLAY_ENABLE_API = false;
-	
-	// 4play password. Please set this to something secure if you use this extension.
-	// This password is used to POST sessions to /api/v2/provide_sesh
-	// It's also used as the external endpoint password.
-	const FPLAY_PASSWORD = "cnc";
-	
-	// Endpoint 4get hits to render webpages
-	// Eg: null=no endpoint provided
-	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
-	const FPLAY_EXTERNAL_ENDPOINT = null;
-	
 	//
 	// BOT PROTECTION
 	//
@@ -59,7 +43,7 @@ class config{
 	
 	// If this regex expression matches on the user agent, it blocks the request
 	// Not useful at all against a targetted attack
-	const HEADER_REGEX = '/bot|wget|curl|python-requests|scrapy|go-http-client|ruby|yahoo|spider|qwant|meta/i';
+	const HEADER_REGEX = '/bot|wget|curl|python-requests|scrapy|go-http-client|ruby|yahoo|spider|qwant/i';
 	
 	// Block clients who present any of the following headers in their request (SPECIFY IN !!lowercase!!)
 	// Eg: ["x-forwarded-for", "x-via", "forwarded-for", "via"];
@@ -116,6 +100,7 @@ class config{
 		"https://4get.sijh.net",
 		"https://4get.hbubli.cc",
 		"https://4get.plunked.party",
+		"https://4get.seitan-ayoub.lol",
 		"https://4get.etenie.pl",
 		"https://4get.lunar.icu",
 		"https://4get.dcs0.hu",
@@ -134,76 +119,50 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0";
-	
-	// User agent to use with 4get-friendly APIs
-	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
 	
 	// Proxy pool assignments for each scraper
 	// false = Use server's raw IP
 	// string = will load a proxy list from data/proxies
 	// Eg. "onion" will load data/proxies/onion.txt
 	const PROXY_DDG = false; // duckduckgo
-	const PROXY_YAHOO = false;
-	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
+	const PROXY_FB = false; // facebook
 	const PROXY_GOOGLE = false;
-	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
 	const PROXY_STARTPAGE = false;
 	const PROXY_QWANT = false;
-	const PROXY_BAIDU = false;
-	const PROXY_COCCOC = false;
 	const PROXY_GHOSTERY = false;
 	const PROXY_MARGINALIA = false;
 	const PROXY_MOJEEK = false;
 	const PROXY_SC = false; // soundcloud
-	const PROXY_SWISSCOWS = false;
 	const PROXY_SPOTIFY = false;
 	const PROXY_SOLOFIELD = false;
-	const PROXY_PURILI = false;
 	const PROXY_WIBY = false;
 	const PROXY_CURLIE = false;
 	const PROXY_YT = false; // youtube
-	const PROXY_ARCHIVEORG = false;
-	const PROXY_SEPIASEARCH = false;
-	const PROXY_ODYSEE = false;
-	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;
-	const PROXY_FLICKR = false;
-	const PROXY_PIXABAY = false;
-	const PROXY_UNSPLASH = false;
-	const PROXY_PEXELS = false;
-	const PROXY_FIVEHPX = false;
-	const PROXY_VSCO = false;
 	const PROXY_SEZNAM = false;
 	const PROXY_NAVER = false;
+	const PROXY_GREPPR = false;
+	const PROXY_CROWDVIEW = false;
 	const PROXY_MWMBL = false;
 	const PROXY_FTM = false; // findthatmeme
 	const PROXY_IMGUR = false;
-	const PROXY_CARA = false;
 	const PROXY_YANDEX_W = false; // yandex web
 	const PROXY_YANDEX_I = false; // yandex images
 	const PROXY_YANDEX_V = false; // yandex videos
-	const PROXY_SAFEBOORU = false;
-	const PROXY_KONACHAN = false;
-	const PROXY_YANDERE = false;
-	const PROXY_TBIB = false;
-	const PROXY_GELBOORU = false;
 	
 	//
 	// Scraper-specific parameters
 	//
 	
-	// GOOGLE CSE & GOOGLE API
+	// GOOGLE CSE
 	const GOOGLE_CX_ENDPOINT = "d4e68b99b876541f0";
 	
 	// MARGINALIA
 	// Use "null" to default out to HTML scraping OR specify a string to
 	// use the API (Eg: "public"). API has less filters.
 	const MARGINALIA_API_KEY = null;
-	
-	// Yep
-	const YEP_USE_API = false;
 }

docker-compose.yaml

@@ -6,15 +6,14 @@ services:
     image: luuul/4get:latest
     restart: unless-stopped
     environment:
-      - FOURGET_PROTO=http
       - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_INSTANCES=https://4get.ca
 
     ports:
       - "80:80"
       - "443:443"
 
-    # volumes:
+    volumes:
-    # - /etc/letsencrypt/live/domain.tld:/etc/4get/certs # mount ssl
+      - /etc/letsencrypt/live/domain.tld:/etc/4get/certs
-    # - ./banners:/var/www/html/4get/banner # mount custom banners
+      # mount custom banners and captcha
-    # - ./captcha:/var/www/html/4get/data/captcha # mount captcha images
+      - ./banners:/var/www/html/4get/banner
+      - ./captcha:/var/www/html/4get/data/captcha

docker/apache/http/conf.d/ssl.conf

@@ -1 +0,0 @@
-# intentionally blank

docker/docker-entrypoint.sh

@@ -8,27 +8,18 @@ FOURGET_PROTO="${FOURGET_PROTO#\"}"
 # make lowercase
 FOURGET_PROTO=`echo $FOURGET_PROTO | awk '{print tolower($0)}'`
 
-FOURGET_SRC='/var/www/html/4get'
-
-mkdir -p /etc/apache2
 
 if [ "$FOURGET_PROTO" = "https" ]; then
         echo "Using https configuration"
-        cp -r ${FOURGET_SRC}/docker/apache/https/httpd.conf /etc/apache2
+        cp /etc/apache2/https.conf /etc/apache2/httpd.conf
-        cp -r ${FOURGET_SRC}/docker/apache/https/conf.d/* /etc/apache2/conf.d
-
 else
         echo "Using http configuration"
-        cp -r ${FOURGET_SRC}/docker/apache/http/httpd.conf /etc/apache2
+        cp /etc/apache2/http.conf /etc/apache2/httpd.conf
-        cp -r ${FOURGET_SRC}/docker/apache/http/conf.d/* /etc/apache2/conf.d
 fi
 
 php ./docker/gen_config.php
 
-if [ "$@" = "start" ]; then
+
 echo "4get is running"
 exec httpd -DFOREGROUND
-else 
-        exec "$@"
-fi
 

docs/apache2-example.md

@@ -49,13 +49,9 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	</Directory>
 
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<Directory /var/www/4get/data/>
-		Require all denied
+		Order Deny,allow
-	</DirectoryMatch>
+		Deny from all
-
-	# remove directory listing from /icons
-	<Directory /var/www/4get/icons>
-		Options -Indexes
 	</Directory>
 </VirtualHost>
 

docs/apache2.md

@@ -7,35 +7,27 @@ Then, install the following dependencies:
 ```sh
 apt update
 apt upgrade
-apt install php-mbstring apache2 certbot php-imagick imagemagick php-curl curl php-apcu git libapache2-mod-fcgid php-fpm
+apt install php-mbstring apache2 certbot php-imagick imagemagick php-curl curl php-apcu git libapache2-mod-php
 ```
 
 Enable the required modules:
 ```sh
-a2dismod mpm_prefork
-a2enmod mpm_event
 a2enmod ssl
 a2enmod rewrite
-a2enmod proxy_fcgi setenvif actions alias
-a2enmod http2
-a2enmod headers
-a2enmod proxy
 ```
 
-Tune the performance of php-fpm. You will need to edit this file according to your server specs and number of users. Edit the file at `/etc/php/8.4/pool.d/www.conf`:
+And enable these optional ones, which might be useful to you later on. The `proxy` module is useful for setting up reverse proxies to services like gitea, and `headers` is useful to tweak global header values:
 ```sh
-pm = static
+a2enmod proxy
-pm.max_children = 50
+a2enmod headers
 ```
 
-These values are what I currently use on 4get.ca, but for personal use, you can set `pm` to `ondemand` and `pm.max_children` to `20` (if you want those thumbnails to load fast!)
-
 Now, restart apache2:
 ```sh
 service apache2 restart
 ```
 
-Just for good measure, please check if your webserver is running. Access it through HTTP, not HTTPS. You should see the apache2 default landing page. Just a note, http2 won't work just yet since you don't have SSL yet.
+Just for good measure, please check if your webserver is running. Access it through HTTP, not HTTPS. You should see the apache2 default landing page.
 
 ## 000-default.conf
 Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, remove everything and carefully add each rule specified here, while making sure to replace my domains with your own:
@@ -82,35 +74,16 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 
 	DocumentRoot /var/www/4get
 
-	<FilesMatch \.php$>
-		SetHandler "proxy:unix:/run/php/php8.1-fpm.sock|fcgi://localhost/"
-	</FilesMatch>
-	
 	Options -MultiViews
 	RewriteEngine On
 	RewriteCond %{REQUEST_FILENAME} !-d
 	RewriteCond %{REQUEST_FILENAME} !-f
 	RewriteRule ^([^\.]+)$ $1.php [NC,L]
 
-	<Directory /var/www/4get>
-		Options -MultiViews
-		AllowOverride All
-		Require all granted
-		
-		RewriteEngine On
-		RewriteCond %{REQUEST_FILENAME} !-d
-		RewriteCond %{REQUEST_FILENAME} !-f
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
-	</Directory>
-	
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<Directory /var/www/4get/data/>
-		Require all denied
+		Order Deny,allow
-	</DirectoryMatch>
+		Deny from all
-
-	# remove directory listing from /icons
-	<Directory /var/www/4get/icons>
-		Options -Indexes
 	</Directory>
 </VirtualHost>
 ```
@@ -143,7 +116,6 @@ Make sure to replace `4get.ca` with your own domain under the `SSLCertificate*`
 	ServerAdmin will@lolcat.ca
 	DocumentRoot /var/www/4get
 	
-	Protocols h2 http/1.1
 	SSLEngine On
 	SSLOptions +StdEnvVars
 	
@@ -156,10 +128,6 @@ Make sure to replace `4get.ca` with your own domain under the `SSLCertificate*`
 	AddOutputFilterByType DEFLATE text/plain
 	AddOutputFilterByType DEFLATE text/css
 	
-	<FilesMatch \.php$>
-		SetHandler "proxy:unix:/run/php/php8.1-fpm.sock|fcgi://localhost/"
-	</FilesMatch>
-	
 	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
 	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
 	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem

docs/configure.md

@@ -9,47 +9,34 @@ Welcome! This guide assumes that you have a working 4get instance. This will hel
 4. The captcha font is located in `data/fonts/captcha.ttf`
 
 # Cloudflare bypass (TLS check)
->These instructions have been updated to work with Debian 13 Trixie.
+**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed.**
 
-**Note: this only allows you to bypass the browser integrity checks. Captchas & javascript challenges will not be bypassed by this program!**
+Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** & the **Mwmbl** search engines. Please be aware that APT will fight against you and will re-install the openSSL-version of curl constantly when updating.
 
-Configuring this lets you fetch images sitting behind Cloudflare and allows you to scrape the **Yep** search engine.
+First, follow these instructions. Only install the Firefox modules:
 
-To come up with this set of instructions, I used [this guide](https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build) as a reference, but trust me you probably want to stick to what's written on this page.
+https://github.com/lwthiker/curl-impersonate/blob/main/INSTALL.md#native-build
 
-First, compile curl-impersonate (the firefox flavor).
+Once you did this, you should be able to run the following inside your terminal:
-```sh
-git clone https://github.com/lwthiker/curl-impersonate/
-cd curl-impersonate
-sudo apt install build-essential pkg-config cmake ninja-build curl autoconf automake libtool python3-pip libnss3 libnss3-dev
-mkdir build
-cd build
-../configure
-make firefox-build
-sudo make firefox-install
-sudo ldconfig
-```
-
-Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine is located at `/usr/local/lib/libcurl-impersonate-ff.so`. Patch your PHP install so that it loads the right library:
 
 ```sh
-sudo systemctl edit php8.4-fpm.service
+$ curl_ff117 --version
+curl 8.1.1 (x86_64-pc-linux-gnu) libcurl/8.1.1 NSS/3.92 zlib/1.2.13 brotli/1.0.9 zstd/1.5.4 libidn2/2.3.3 nghttp2/1.56.0
+Release-Date: 2023-05-23
+Protocols: dict file ftp ftps gopher gophers http https imap imaps mqtt pop3 pop3s rtsp smb smbs smtp smtps telnet tftp ws wss
+Features: alt-svc AsynchDNS brotli HSTS HTTP2 HTTPS-proxy IDN IPv6 Largefile libz NTLM NTLM_WB SSL threadsafe UnixSockets zstd
 ```
+Now, after compiling, you should have a `libcurl-impersonate-ff.so` sitting somewhere. Mine (on my debian install) is located at `/usr/local/lib/libcurl-impersonate-ff.so`.
 
-^This will open a text editor. Add the following shit in there, in between those 2 comments I pasted for ya just for reference:
+Find the `libcurl.so.4` file used by your current installation of curl. For me, this file is located at `/usr/lib/x86_64-linux-gnu/libcurl.so.4`
 
+Now comes the sketchy part: replace `libcurl.so.4` with `libcurl-impersonate-ff.so`. You can do this in the following way:
 ```sh
-### Editing /etc/systemd/system/php8.4-fpm.service.d/override.conf
+sudo rm /usr/lib/x86_64-linux-gnu/libcurl.so.4
-### Anything between here and the comment below will become the contents of the>
+sudo cp /usr/local/lib/libcurl-impersonate-ff.so /usr/lib/x86_64-linux-gnu/libcurl.so.4
-
-[Service]
-Environment="LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so"
-Environment="CURL_IMPERSONATE=firefox117"
-
-### Edits below this comment will be discarded
 ```
 
-Restart php8.4-fpm. (`sudo service php8.4-fpm restart`). To test things out, try making a search on "Yep", they check for SSL. If you get results (or a timeout, this piece of shit engine is slow as fuck) that means it works!
+Make sure to restart your webserver and/or PHP daemon, otherwise it will keep using the old library. You should now be able to bypass Cloudflare's shitty checks!!
 
 # Robots.txt
 Make sure you configure this right to optimize your search engine presence! Head over to `/robots.txt` and change the 4get.ca domain to your own domain.
@@ -79,87 +66,3 @@ Done! The scraper you chose should now be using the rotating proxies. When askin
 
 ## Important!
 If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life!
-
-# 4play setup
-4play is a Firefox extension I wrote that lets you control a browser from 4get. When you do a search, 4get sends a query to a nodeJS http server that communicates to the browser via websockets. The browser then reports back various payloads used for scraping. It's very similar to tools like puppeteer or playwright, except it uses websockets to communicate with the browser instead of the browser's debugger. This is needed to prevent `navigator.webdriver` from being set to `true` (also avoids a fuckton of other botchecks, that's just the tip of the iceberg).
-
-## Requirements
-- A dedicated laptop or computer running a full desktop environment (like Windows, Mac, or Linux DEs)
-- Something to make the computer think it's connected to a screen. You need it, otherwise you'll fallback to software rendering and sites can detect that. You can use:
-	- Your laptop's screen
-	- A turned-on screen connected via HDMI
-	- An EDID adapter to simulate a display (no affiliate links: [HDMI](https://www.aliexpress.com/item/1005011668390064.html), [DisplayPort](https://www.aliexpress.com/item/1005011659561729.html))
-- Firefox ESR or the latest version of Firefox
-- The [4play extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) installed
-
-## Installation
-Install the dependencies on the machine you wish to run 4play on. In a sane setup, you usually want 4play to run on a dedicated user-grade machine. For the sake of simplicity, this machine will be referred as the "render server".
-
-```sh
-git clone https://git.lolcat.ca/lolcat/4get
-cd 4get/extra/4play
-
-# install nodejs 26.x or later if it's not already installed
-
-npm install @lawlers/4play
-sudo npm install -g nodemon
-
-# edit line 7 where it says 'var password = "cnc";'. Use a strong password.
-nano page-render.js
-
-# test the server
-nodemon page-render.js
-```
-
-I recommend setting up a systemd service to auto-start the 4play server:
-```sh
-sudo nano /etc/systemd/system/fplay.service
-```
-
-Paste this in (make sure to update `User` & `WorkingDirectory`)
-```sh
-[Unit]
-Description=4play
-After=network.target
-
-[Service]
-Type=simple
-User=will
-WorkingDirectory=/home/will/Desktop/4get/extra/4play/
-ExecStart=nodemon /home/will/Desktop/4get/extra/4play/page-render.js
-Restart=always
-Environment=NODE_ENV=production
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Now, enable the service
-```sh
-sudo systemctl enable fplay
-sudo service fplay start
-sudo service fplay status # check the status. You should see it running
-```
-
-This should expose a webservice located at `http://localhost:3000`. I highly recommend you put this endpoint behind apache2 or nginx to add SSL, if the render server sits on a different network.
-
-## Setting up Firefox
-- Install Firefox. You can go for the ESR version or the latest version. You'll figure it out.
-- Head over to [this page](https://addons.mozilla.org/en-US/firefox/addon/4play/) or search "4play" on the Firefox extension store and install the extension.
-- Click the settings cog next to the extension name. Enable "Run in Private Windows" and "Allow automatic updates"
-- Click the extension in the top bar. Modify the `ws://localhost:3030/cnc` URL: replace the `cnc` path with the password you set earlier for the server.
-- Use a timeout value of `30000` to avoid getting throttled by Firefox during reconnections.
-
-If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
-
-## Setting up 4get
-Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
-
-```php
-const FPLAY_PASSWORD = "cnc"; # Change this to your password
-const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.com/api/example";, with the quotes and semicolon
-```
-
-**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
-
-If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.

docs/develop.md

@@ -1,7 +0,0 @@
-# Develop
-
-Start the PHP webserver like this, assuming you have curl-impersonate installed already
-
-```sh
-LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
-```

docs/docker.md

@@ -150,7 +150,3 @@ services:
 ##### Tor
 
 You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
-
-##### 4play
-
-You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)

docs/docker_4play.md

@@ -1,48 +0,0 @@
-### Running 4get and 4play via docker guide
-
-This setup will run both 4get and the 4play sidecar 
-
-# Docker compose 
-
-```
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"
-
-```
-
-There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
-
-# Steps
-
-1. start the services with
-
-```
-docker compose up -d
-
-```
-
-2. configure firefox
-
-folllow the steps in [configure](./configure.md) for firefox setup

extra/4play/Dockerfile

@@ -1,19 +0,0 @@
-FROM alpine:3.21
-WORKDIR /app
-
-RUN apk update && apk upgrade
-RUN apk add nodejs npm
-
-COPY package*.json ./
-
-RUN npm ci --only=production && npm cache clean --force
-
-COPY . .
-
-EXPOSE 3030
-
-ENTRYPOINT  ["node"]
-CMD ["page-render.js"]
-
-
-

extra/4play/docker-compose.yaml

@@ -1,24 +0,0 @@
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"

extra/4play/package-lock.json

@@ -1,47 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "4play",
-      "version": "1.0.0",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "@lawlers/4play": "^1.2.5",
-        "ws": "^8.21.2"
-      }
-    },
-    "node_modules/@lawlers/4play": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
-      "integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "ws": "^8.21.1"
-      }
-    },
-    "node_modules/ws": {
-      "version": "8.21.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
-      "integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    }
-  }
-}

extra/4play/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "description": "",
-  "repository": {
-    "type": "git",
-    "url": "https://git.lolcat.ca/lolcat/4get/"
-  },
-  "license": "AGPL-3.0-only",
-  "author": "lolcat",
-  "type": "commonjs",
-  "main": "page-render.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "dependencies": {
-    "@lawlers/4play": "^1.2.5",
-    "ws": "^8.21.2"
-  }
-}

extra/4play/page-render.js

@@ -1,337 +0,0 @@
-const fplay = require("@lawlers/4play");
-const http = require("http");
-
-// 4play settings
-var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
-var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
-var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
-
-// ==================================================
-
-// webserver
-
-var global_ws = null;
-const webserver_port = 3000;
-
-var callback_q = new Map();
-
-const server = http.createServer(async function(req, res){
-	
-	const parsed = new URL(req.url, "http://localhost");
-	
-	const url = parsed.searchParams.get("url");
-	const pw = parsed.searchParams.get("pw");
-	var proxy = parsed.searchParams.get("proxy");
-	const url_match = parsed.searchParams.get("url_match");
-	const content_match = parsed.searchParams.get("content_match");
-	const fail_url_match = parsed.searchParams.get("fail_url_match");
-	const fail_content_match = parsed.searchParams.get("fail_content_match");
-	
-	var container = parsed.searchParams.get("container");
-	
-	if(fplay.browser_connected === false){
-				
-		res.writeHead(
-			400,
-			{
-				"Content-Type": "text/plain"
-			}
-		);
-		
-		res.end("No browser available to render the motherfucking page");
-		return;
-	}
-	
-	var errors = [];
-	
-	if(
-		pw === null ||
-		pw != password
-	){
-		
-		errors.push("pw");
-	}
-	
-	if(url === null){ errors.push("url"); }
-	if(proxy === null){ errors.push("proxy"); }
-	
-	if(errors.length !== 0){
-		
-		res.writeHead(
-			400,
-			{
-				"Content-Type": "text/plain"
-			}
-		);
-		
-		res.end("The following parameters are missing or invalid: " + errors.join(", "));
-		return;
-	}
-	
-	console.log("Rendering " + url);
-	
-	if(
-		container === null ||
-		await fplay.container_exists(global_ws, container) === false
-	){
-		
-		container = await fplay.container_create(global_ws);
-		
-		proxy = get_proxy(proxy);
-		
-		// assign proxy
-		if(proxy.config !== false){
-			
-			await fplay.container_attach_proxy(
-				global_ws,
-				container,
-				proxy.config
-			);
-		}
-	}else{
-		
-		proxy = {raw: proxy};
-	}
-	
-	// open tab
-	const newtab = await fplay.tab_open(global_ws, url, false, container);
-	
-	if(newtab === false){
-		
-		// failed to open tab
-		res.writeHead(
-			400,
-			{
-				"Content-Type": "text/plain",
-				"X-Proxy": proxy.raw,
-				"X-Container-ID": typeof container.id == "undefined" ? container : container.id
-			}
-		);
-		
-		res.end("Failed to open tab");
-		return;
-	}
-		
-	// callback will be handled in "web_response"
-	callback_q.set(
-		newtab.id,
-		{
-			http_res: res,
-			proxy: proxy,
-			container: container,
-			url_match: url_match,
-			content_match: content_match,
-			fail_url_match: fail_url_match,
-			fail_content_match: fail_content_match
-		}
-	);
-	
-	// close tab in 30 seconds
-	await fplay.wait_random(15000, 35000);
-	await fplay.tab_close(global_ws, newtab);
-});
-
-server.listen(webserver_port, function(){
-	
-	console.log("http server running");
-});
-
-function get_proxy(proxy_line){
-	
-	parts = proxy_line.split(":");
-	// format: type:address:port:username:password
-	// PHP explode(":", $ip, 5) splits into max 5 parts
-	var type = parts[0];
-	const address = parts[1];
-	const port = parts[2];
-	const username = parts[3];
-	const password = parts[4];
-	
-	var proxy_dns = true;
-	
-	switch(type){
-		case "raw_ip":
-		case "ip":
-			// no proxy, use direct connection
-			return {
-				config: false,
-				raw: "raw_ip::::"
-			}
-			break;
-		
-		case "socks5":
-			type = "socks";
-			proxy_dns = false;
-			break;
-			
-		case "socks4a":
-			type = "socks4";
-			break;
-			
-		case "socks4":
-			proxy_dns = false;
-			break;
-			
-		case "socks5_hostname":
-		case "socks5h":
-		case "socks5a":
-			type = "socks";
-			break;
-		
-		case "http":
-		case "https":
-			// do nothing
-			break;
-		
-		default:
-			// invalid proxy provided
-			return {
-				config: false,
-				raw: "raw_ip::::"
-			}
-	}
-	
-	return {
-		config: {
-			type: type,
-			host: address,
-			port: port,
-			username: username,
-			password: password,
-			proxyDNS: proxy_dns
-		},
-		raw: proxy_line
-	};
-}
-
-fplay.event.on("server_ready", function(){
-	
-	console.log("listening on port " + port + " (timeout=" + timeout + ")");
-});
-
-fplay.event.on("browser_connect", async function(ws){
-	
-	global_ws = ws;
-	
-	const ua = await fplay.get_ua(ws);
-	console.log("New browser instance connected: " + ua);
-	
-	await fplay.close_all_tabs(ws);
-	await fplay.delete_all_containers(ws);
-	
-	fplay.web_response_whitelist(ws, ["main_frame"]);
-	
-	// clean up
-	const blanktab = await fplay.close_all_tabs(ws);
-	await fplay.delete_all_containers(ws);
-});
-
-fplay.event.on("web_response", async function(page){
-	
-	var callback_data = callback_q.get(page.id);
-	
-	if(typeof callback_data == "undefined"){ return; }
-	
-	var body = page.body.toString();
-	
-	var url_match = callback_data.url_match !== null ? page.url.match(new RegExp(callback_data.url_match, "i")) : null;
-	var content_match = callback_data.content_match !== null ? body.match(new RegExp(callback_data.content_match, "i")) : null;
-	var fail_url_match = callback_data.fail_url_match !== null ? page.url.match(new RegExp(callback_data.fail_url_match, "i")) : null;
-	var fail_content_match = callback_data.fail_content_match !== null ? body.match(new RegExp(callback_data.fail_content_match, "i")) : null;
-	
-	// handle negative response
-	if(
-		(
-			callback_data.fail_url_match !== null ||
-			callback_data.fail_content_match !== null
-		) &&
-		(
-			fail_url_match !== null ||
-			fail_content_match !== null
-		)
-	){
-		
-		callback_data.http_res.writeHead(
-			400,
-			{
-				"Content-Type": "text/plain",
-				"X-Proxy": callback_data.proxy.raw,
-				"X-Container-ID": callback_data.container.id
-			}
-		);
-		
-		callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)");
-		
-		// cleanup browser + callback
-		await fplay.tab_close(global_ws, page.id);
-		await fplay.container_delete(global_ws, callback_data.container);
-		callback_q.delete(page.id);
-		
-		return;
-	}
-	
-	// handle positive response
-	if(
-		(
-			callback_data.url_match === null &&
-			callback_data.content_match === null
-		) ||
-		(
-			callback_data.url_match !== null &&
-			callback_data.content_match !== null &&
-			url_match &&
-			content_match
-		) ||
-		(
-			callback_data.url_match === null &&
-			callback_data.content_match !== null &&
-			content_match
-		) ||
-		(
-			callback_data.url_match !== null &&
-			callback_data.content_match === null &&
-			url_match
-		)
-	){
-		
-		callback_data.http_res.writeHead(
-			200,
-			{
-				"Content-Type": "text/plain;charset=UTF-8",
-				"X-Proxy": callback_data.proxy.raw,
-				"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
-			}
-		);
-		
-		callback_data.http_res.end(body);
-		
-		callback_q.delete(page.id); // cleanup callback only
-		return;
-	}
-});
-
-fplay.event.on("dom_load_fail", async function(page){
-	
-	var callback_data = callback_q.get(page.id);
-	
-	if(typeof callback_data == "undefined"){ return; }
-	
-	callback_q.delete(page.id);
-	
-	callback_data.http_res.writeHead(
-		400,
-		{
-			"Content-Type": "text/plain",
-			"X-Proxy": callback_data.proxy.raw,
-			"X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id
-		}
-	);
-	
-	callback_data.http_res.end("Network error (" + page.error + ")");
-		
-	// cleanup browser
-	await fplay.tab_close(global_ws, page.id);
-	await fplay.container_delete(global_ws, callback_data.container);
-});
-
-fplay.init(port, password, timeout);

favicon.php

@@ -15,12 +15,7 @@ class favicon{
 		
 		header("Content-Type: image/png");
 		
-		if(
+		if(substr_count($url, "/") !== 2){
-			preg_match(
-				'/^https?:\/\/[A-Za-z0-9.-]+$/',
-				$url
-			) === 0
-		){
 			
 			header("X-Error: Only provide the protocol and domain");
 			$this->defaulticon();

lib/anubis.php

@@ -1,100 +0,0 @@
-<?php
-
-//
-// Reference
-// https://github.com/TecharoHQ/anubis/blob/ecc716940e34ebe7249974f2789a99a2c7115e4e/web/js/proof-of-work.mjs
-//
-
-class anubis{
-	
-	public function __construct(){
-		
-		include_once "fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function scrape($html){
-		
-		$this->fuckhtml->load($html);
-		
-		$script =
-			$this->fuckhtml
-			->getElementById(
-				"anubis_challenge",
-				"script"
-			);
-		
-		if($script === false){
-			
-			throw new Exception("Failed to scrape anubis challenge data");
-		}
-		
-		$script =
-			json_decode(
-				$this->fuckhtml
-				->getTextContent(
-					$script
-				),
-				true
-			);
-		
-		if($script === null){
-			
-			throw new Exception("Failed to decode anubis challenge data");
-		}
-		
-		if(
-			!isset($script["challenge"]) ||
-			!isset($script["rules"]["difficulty"]) ||
-			!is_int($script["rules"]["difficulty"]) ||
-			!is_string($script["challenge"])
-		){
-			
-			throw new Exception("Found invalid challenge data");
-		}
-		
-		return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
-	}
-	
-	private function is_valid_hash($hash, $difficulty){
-		
-		for ($i=0; $i<$difficulty; $i++) {
-			
-			$index = (int)floor($i / 2);
-			$nibble = $i % 2;
-			
-			$byte = ord($hash[$index]);
-			$nibble = ($byte >> ($nibble === 0 ? 4 : 0)) & 0x0f;
-			
-			if($nibble !== 0){
-				return false;
-			}
-		}
-		
-		return true;
-	}
-	
-	public function rape($data, $difficulty = 5){
-		
-		$nonce = 0;
-		
-		while(true){
-			
-			$hash_binary = hash("sha256", $data . $nonce, true);
-			
-			if($this->is_valid_hash($hash_binary, $difficulty)){
-				
-				$hash_hex = bin2hex($hash_binary);
-				
-				return [
-					"response" => $hash_hex,
-					//"data" => $data,
-					//"difficulty" => $difficulty,
-					"nonce" => $nonce
-				];
-			}
-			
-			$nonce++;
-		}
-	}
-}

lib/backend.php

@@ -4,15 +4,12 @@ class backend{
 	public function __construct($scraper){
 		
 		$this->scraper = $scraper;
-		$this->db = null;
-		$this->db_path_prefix = "data/sessions/";
-		$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
 	}
 	
 	/*
 		Proxy stuff
 	*/
-	public function get_ip($proxy_index_raw = null){
+	public function get_ip(){
 		
 		$pool = constant("config::PROXY_" . strtoupper($this->scraper));
 		if($pool === false){
@@ -22,10 +19,7 @@ class backend{
 		}
 		
 		// indent
-		if($proxy_index_raw === null){
-			
 		$proxy_index_raw = apcu_inc("p." . $this->scraper);
-		}
 		
 		$proxylist = file_get_contents("data/proxies/" . $pool . ".txt");
 		$proxylist = explode("\n", $proxylist);
@@ -38,12 +32,6 @@ class backend{
 		
 		$proxylist = array_values($proxylist);
 		
-		if(count($proxylist) === 0){
-			
-			throw new Exception("A proxy list was specified but it's empty!");
-		}
-		
-		//echo $proxylist[$proxy_index_raw % count($proxylist)];
 		return $proxylist[$proxy_index_raw % count($proxylist)];
 	}
 	
@@ -87,7 +75,6 @@ class backend{
 				break;
 			
 			case "socks5_hostname":
-			case "socks5h":
 			case "socks5a":
 				curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
 				curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
@@ -100,30 +87,6 @@ class backend{
 		}
 	}
 	
-	// API key rotation
-	public function get_key(){
-		
-		$keys = file_get_contents("data/api_keys/" . $this->scraper . ".txt");
-		$keys = explode("\n", $keys);
-		
-		$keys = array_filter($keys, function($entry){
-			$entry = ltrim($entry);
-			return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
-		});
-		
-		$keys = array_values($keys);
-		
-		if(count($keys) === 0){
-			
-			throw new Exception("Please specify API keys in data/api_keys/" . $this->scraper . ".txt");
-		}
-		
-		$increment = apcu_inc("s." . $this->scraper) % count($keys);
-		return [
-			"key" => $keys[$increment],
-			"increment" => $increment
-		];
-	}
 	
 	
 	/*
@@ -211,127 +174,4 @@ class backend{
 			$payload[1] // proxy
 		];
 	}
-	
-	//
-	// Sessions
-	//
-	public function session_init_db($force_init = false){
-		
-		if(
-			$force_init ||
-			$this->db === null
-		){
-			
-			$this->db = new SQLite3($this->db_path);
-			
-			// create table if it doesnt exist	
-			$this->db->exec(
-				"CREATE TABLE IF NOT EXISTS sessions ( " .
-					"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
-					"created_at INTEGER NOT NULL, " .
-					"proxy TEXT NOT NULL, " .
-					"headers_json TEXT NOT NULL" .
-				")"
-			);
-			
-			return true;
-		}
-		
-		return false;
-	}
-	
-	public function session_count(){
-		
-		$this->session_init_db();
-		
-		$stmt = $this->db->prepare(
-			"SELECT COUNT(*) as count FROM sessions"
-		);
-		
-		$result = $stmt->execute();
-		$row = $result->fetchArray(SQLITE3_ASSOC);
-		
-		return (int)($row["count"] ?? 0);
-	}
-	
-	public function session_write($proxy, $headers){
-		
-		$this->session_init_db();
-		
-		$stmt = $this->db->prepare(
-			"INSERT INTO sessions ( " .
-				"created_at, " .
-				"proxy, " .
-				"headers_json " .
-			") VALUES ( " .
-				":created_at, " .
-				":proxy, " .
-				":headers_json" .
-			")"
-		);
-		
-		$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
-		$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
-		$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
-		
-		$stmt->execute();
-		
-		return $this->db->lastInsertRowID();
-	}
-	
-	public function session_get(){
-		
-		$count = $this->session_count();
-		
-		if($count === 0){
-			
-			return false;
-		}
-		
-		$offset = apcu_inc("d." . $this->scraper) % $count;
-		
-		$stmt = $this->db->prepare(
-			"SELECT * " .
-			"FROM sessions " .
-			"ORDER BY id ASC " .
-			"LIMIT 1 OFFSET :offset"
-		);
-		
-		$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
-		
-		$result = $stmt->execute();
-		$row = $result->fetchArray(SQLITE3_ASSOC);
-		
-		return [
-			"id" => $row["id"],
-			"created_at" => $row["created_at"],
-			"proxy" => $row["proxy"],
-			"headers" => json_decode($row["headers_json"], true)
-		];
-	}
-	
-	public function session_destroy_list(){
-		
-		if(
-			!preg_match(
-				'/^[A-Za-z0-9-]+$/',
-				$this->scraper
-			)
-		){
-			
-			// path traversal
-			return false;
-		}
-		
-		if(
-			file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
-			is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
-		){
-			
-			unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
-			return true;
-		}
-		
-		return false;
-	}
 }

lib/bingcache-todo-fix.php

@@ -0,0 +1,144 @@
+<?php
+
+// https://www.bing.com/search?q=url%3Ahttps%3A%2F%2Flolcat.ca
+// https://cc.bingj.com/cache.aspx?q=url%3ahttps%3a%2f%2flolcat.ca&d=4769685974291356&mkt=en-CA&setlang=en-US&w=tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK-
+// <div class="b_attribution" u="0N|5119|4769685974291356|tEsWuE7HW3Z5AIPQMVkDH4WaotS4LrK-" tabindex="0">
+
+new bingcache();
+
+class bingcache{
+	
+	public function __construct(){
+		
+		if(
+			!isset($_GET["s"]) ||
+			$this->validate_url($_GET["s"]) === false
+		){
+			
+			var_dump($this->validate_url($_GET["s"]));
+			$this->do404("Please provide a valid URL.");
+		}
+		
+		$url = $_GET["s"];
+		
+		$curlproc = curl_init();
+		
+		curl_setopt(
+			$curlproc,
+			CURLOPT_URL,
+			"https://www.bing.com/search?q=url%3A" .
+			urlencode($url)
+		);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt(
+			$curlproc,
+			CURLOPT_HTTPHEADER,
+			["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/107.0",
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 5);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			$this->do404("Failed to connect to bing servers. Please try again later.");
+		}
+		
+		curl_close($curlproc);
+		
+		preg_match(
+			'/<div class="b_attribution" u="(.*)" tabindex="0">/',
+			$data,
+			$keys
+		);
+		
+		print_r($keys);
+		
+		if(count($keys) === 0){
+			
+			$this->do404("Bing has not archived this URL.");
+		}
+		
+		$keys = explode("|", $keys[1]);
+		$count = count($keys);
+		
+		//header("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]);
+		echo("Location: https://cc.bingj.com/cache.aspx?d=" . $keys[$count - 2] . "&w=" . $keys[$count - 1]);
+	}
+	
+	public function do404($text){
+				
+		include "lib/frontend.php";
+		$frontend = new frontend();
+		
+		echo
+			$frontend->load(
+				"error.html",
+				[
+					"title" => "Shit",
+					"text" => $text
+				]
+			);
+		
+		die();
+	}
+	
+	public function validate_url($url){
+		
+		$url_parts = parse_url($url);
+		
+		// check if required parts are there
+		if(
+			!isset($url_parts["scheme"]) ||
+			!(
+				$url_parts["scheme"] == "http" ||
+				$url_parts["scheme"] == "https"
+			) ||
+			!isset($url_parts["host"])
+		){
+			return false;
+		}
+		
+		if(
+			// if its not an RFC-valid URL
+			!filter_var($url, FILTER_VALIDATE_URL)
+		){
+			return false;
+		}
+		
+		$ip = 
+			str_replace(
+				["[", "]"], // handle ipv6
+				"",
+				$url_parts["host"]
+			);
+		
+		// if its not an IP
+		if(!filter_var($ip, FILTER_VALIDATE_IP)){
+			
+			// resolve domain's IP
+			$ip = gethostbyname($url_parts["host"] . ".");
+		}
+		
+		// check if its localhost
+		return filter_var(
+			$ip,
+			FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+		);
+	}
+}

lib/bot_protection.php

@@ -133,9 +133,6 @@ class bot_protection{
 			$answers[] = $regex;
 		}
 		
-		// dedup
-		$answers = array_unique($answers);
-		
 		if(
 			!$invalid &&
 			$key !== false // has captcha been gen'd?

lib/frontend.php

@@ -2,52 +2,6 @@
 
 class frontend{
 	
-	public function validateurl($url, $net_validate = false){
-		
-		$url_parts = parse_url($url);
-		
-		// check if required parts are there
-		if(
-			!isset($url_parts["scheme"]) ||
-			!(
-				$url_parts["scheme"] == "http" ||
-				$url_parts["scheme"] == "https"
-			) ||
-			!isset($url_parts["host"])
-		){
-			return false;
-		}
-		
-		if($net_validate){
-			$ip = 
-				str_replace(
-					["[", "]"], // handle ipv6
-					"",
-					$url_parts["host"]
-				);
-			
-			// if its not an IP
-			if(!filter_var($ip, FILTER_VALIDATE_IP)){
-				
-				// resolve domain's IP
-				$ip = gethostbyname($url_parts["host"] . ".");
-			}
-			
-			// check if its localhost
-			if(
-				filter_var(
-					$ip,
-					FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
-				) === false
-			){
-				
-				return false;
-			}
-		}
-		
-		return true;
-	}
-	
 	public function load($template, $replacements = []){
 		
 		$replacements["server_name"] = htmlspecialchars(config::SERVER_NAME);
@@ -449,28 +403,27 @@ class frontend{
 		$text =
 			trim(
 				preg_replace(
-					'/<code [^>]+>/',
+					'/<\/span>$/',
-					"",
+					"", // remove stray ending span because of the <?php stuff
 					str_replace(
 						[
-							"<br />",
+							'<br />',
-							"&nbsp;",
+							'&nbsp;'
-							"<pre>",
-							"</pre>",
-							"</code>"
 						],
 						[
-							"\n",
+							"\n", // replace <br> with newlines
-							" ",
+							" " // replace html entity to space
-							"",
-							"",
-							""
 						],
-						explode(
+						str_replace(
-							"&lt;?php",
+							[
-							highlight_string("<?php " . $text, true),
+								// leading <?php garbage
-							2
+								"<span style=\"color: c-default\">\n&lt;?php&nbsp;",
-						)[1]
+								"<code>",
+								"</code>"
+							],
+							"",
+							highlight_string("<?php " . $text, true)
+						)
 					)
 				)
 			);
@@ -492,17 +445,6 @@ class frontend{
 			Add favicon
 		*/
 		$host = parse_url($link);
-		
-		// special case for when we're not drawing a full url
-		if(!isset($host["host"])){
-			
-			$payload =
-				'<div class="url">' .
-					'<button class="favicon" tabindex="-1">' .
-						'<img src="/favicon?s=404" alt="xx">' .
-					'</button>';
-		}else{
-			
 		$esc =
 			explode(
 				".",
@@ -533,15 +475,376 @@ class frontend{
 				'</button>' .
 				'<div class="favicon-dropdown">';
 		
+		/*
+			Add archive links
+		*/
+		if(
+			$host["host"] == "boards.4chan.org" ||
+			$host["host"] == "boards.4channel.org"
+		){
+			
+			$archives = [];
+			$path = explode("/", $host["path"]);
+			$count = count($path);
+			// /pol/thread/417568063/post-shitty-memes-if-you-want-to
+			
+			if($count !== 0){
+				
+				$isboard = true;
+				
+				switch($path[1]){
+					
+					case "con":
+						break;
+					
+					case "q":
+						$archives[] = "desuarchive.org";
+						break;
+						
+					case "qa":
+						$archives[] = "desuarchive.org";
+						break;
+						
+					case "qb":
+						$archives[] = "arch.b4k.co";
+						break;
+						
+					case "trash":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "a":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "c":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "w":
+						break;
+					
+					case "m":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "cgl":
+						$archives[] = "desuarchive.org";
+						$archives[] = "warosu.org";
+						break;
+					
+					case "f":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "n":
+						break;
+					
+					case "jp":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "vt":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "v":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vg":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vm":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vmg":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vp":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vr":
+						$archives[] = "desuarchive.org";
+						$archives[] = "warosu.org";
+						break;
+					
+					case "vrpg":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "vst":
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "co":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "g":
+						$archives[] = "desuarchive.org";
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "tv":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "k":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "o":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "an":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "tg":
+						$archives[] = "desuarchive.org";
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "sp":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "xs":
+						$archives[] = "eientei.xyz";
+						break;
+					
+					case "pw":
+						break;
+					
+					case "sci":
+						$archives[] = "warosu.org";
+						$archives[] = "eientei.xyz";
+						break;
+					
+					case "his":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "int":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "out":
+						break;
+					
+					case "toy":
+						break;
+					
+					case "i":
+						$archives[] = "archiveofsins.com";
+						$archives[] = "eientei.xyz";
+						break;
+					
+					case "po":
+						break;
+					
+					case "p":
+						break;
+					
+					case "ck":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "ic":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "wg":
+						break;
+					
+					case "lit":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "mu":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "fa":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "3":
+						$archives[] = "warosu.org";
+						$archives[] = "eientei.xyz";
+						break;
+					
+					case "gd":
+						break;
+					
+					case "diy":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "wsg":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "qst":
+						break;
+					
+					case "biz":
+						$archives[] = "warosu.org";
+						break;
+					
+					case "trv":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "fit":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "x":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "adv":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "lgbt":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "mlp":
+						$archives[] = "desuarchive.org";
+						$archives[] = "arch.b4k.co";
+						break;
+					
+					case "news":
+						break;
+					
+					case "wsr":
+						break;
+					
+					case "vip":
+						break;
+					
+					case "b":
+						$archives[] = "thebarchive.com";
+						break;
+					
+					case "r9k":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "pol":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "bant":
+						$archives[] = "thebarchive.com";
+						break;
+					
+					case "soc":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "s4s":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "s":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "hc":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "hm":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "h":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "e":
+						break;
+					
+					case "u":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "d":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "t":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					case "hr":
+						$archives[] = "archive.4plebs.org";
+						break;
+					
+					case "gif":
+						break;
+					
+					case "aco":
+						$archives[] = "desuarchive.org";
+						break;
+					
+					case "r":
+						$archives[] = "archiveofsins.com";
+						break;
+					
+					default:
+						$isboard = false;
+						break;
+				}
+				
+				if($isboard === true){
+					
+					$archives[] = "archived.moe";
+				}
+				
+				$trail = "";
+				
+				if(
+					isset($path[2]) &&
+					isset($path[3]) &&
+					$path[2] == "thread"
+				){
+					
+					$trail .= "/" . $path[1] . "/thread/" . $path[3];
+				}elseif($isboard){
+					
+					$trail = "/" . $path[1] . "/";
+				}
+				
+				for($i=0; $i<count($archives); $i++){
+					
 					$payload .=
+						'<a href="https://' . $archives[$i] . $trail . '" class="list" target="_BLANK">' .
+							'<img src="/favicon?s=https://' . $archives[$i] . '" alt="' . $archives[$i][0] . $archives[$i][1] . '">' .
+							$archives[$i] .
+						'</a>';
+				}
+			}
+		}
+		
+		$payload .=
+				'<a href="https://webcache.googleusercontent.com/search?q=cache:' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://google.com" alt="go">Google cache</a>' .
 				'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
 				'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
 				'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
-					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
 				'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
 				'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
 			'</div>';
-		}
 		
 		/*
 			Draw link
@@ -558,12 +861,10 @@ class frontend{
 		}
 		
 		// merge https://site together
-		if(isset($host["host"])){
 		$parts = [
 			$parts[0] . $parts[1] . '//' . $parts[2],
 			...array_slice($parts, 3, count($parts) - 1)
 		];
-		}
 		
 		$c = count($parts);
 		for($i=0; $i<$c; $i++){
@@ -634,27 +935,23 @@ class frontend{
 				$filters["scraper"] = [
 					"display" => "Scraper",
 					"option" => [
-						//"fget" => "fget",
 						"ddg" => "DuckDuckGo",
-						//"yahoo" => "Yahoo!",
 						"brave" => "Brave",
 						"yandex" => "Yandex",
 						"google" => "Google",
-						"google_api" => "Google API",
 						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
+						"ghostery" => "Ghostery",
 						"yep" => "Yep",
+						"greppr" => "Greppr",
+						"crowdview" => "Crowdview",
 						"mwmbl" => "Mwmbl",
 						"mojeek" => "Mojeek",
-						"naver" => "Naver",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
 						"solofield" => "Solofield",
 						"marginalia" => "Marginalia",
-						"purili" => "Purili",
+						"wiby" => "wiby",
-						"wiby" => "wiby"
+						"curlie" => "Curlie"
 					]
 				];
 				break;
@@ -667,22 +964,12 @@ class frontend{
 						"yandex" => "Yandex",
 						"brave" => "Brave",
 						"google" => "Google",
-						"google_api" => "Google API",
 						"google_cse" => "Google CSE",
-						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"naver" => "Naver",
+						"yep" => "Yep",
-						"baidu" => "Baidu",
 						"solofield" => "Solofield",
-						"pinterest" => "Pinterest",
+						//"pinterest" => "Pinterest",
-						"cara" => "Cara",
-						"flickr" => "Flickr",
-						"pexels" => "Pexels",
-						"pixabay" => "Pixabay",
-						"unsplash" => "Unsplash",
-						"fivehpx" => "500px",
-						"vsco" => "VSCO",
 						"imgur" => "Imgur",
 						"ftm" => "FindThatMeme"
 					]
@@ -694,22 +981,13 @@ class frontend{
 					"display" => "Scraper",
 					"option" => [
 						"yt" => "YouTube",
-						//"archiveorg" => "Archive.org",
-						"vimeo" => "Vimeo",
-						//"odysee" => "Odysee",
-						"sepiasearch" => "Sepia Search",
 						//"fb" => "Facebook videos",
 						"ddg" => "DuckDuckGo",
 						"brave" => "Brave",
 						"yandex" => "Yandex",
 						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"naver" => "Naver",
-						"baidu" => "Baidu",
-						"coccoc" => "Cốc Cốc",
-						"purili" => "Purili",
 						"solofield" => "Solofield"
 					]
 				];
@@ -722,11 +1000,10 @@ class frontend{
 						"ddg" => "DuckDuckGo",
 						"brave" => "Brave",
 						"google" => "Google",
-						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
 						"qwant" => "Qwant",
-						"mojeek" => "Mojeek",
+						"yep" => "Yep",
-						"baidu" => "Baidu"
+						"mojeek" => "Mojeek"
 					]
 				];
 				break;
@@ -735,28 +1012,11 @@ class frontend{
 				$filters["scraper"] = [
 					"display" => "Scraper",
 					"option" => [
-						"sc" => "SoundCloud",
+						"sc" => "SoundCloud"
-						"swisscows" => "Swisscows (SoundCloud)"
 						//"spotify" => "Spotify"
 					]
 				];
 				break;
-			
-			case "booru":
-				$filters["scraper"] = [
-					"display" => "Scraper",
-					"option" => [
-						"safebooru" => "Safebooru",
-						"konachan" => "Konachan",
-						"tbib" => "The Big Imageboard",
-						"gelbooru" => "Gelbooru",
-						"yandere" => "Yande.re",
-						"tbib" => "The Big Imageboard",
-						"sankakucomplex" => "SankakuComplex",
-						"soybooru" => "SoyBooru"
-					]
-				];
-				break;
 		}
 		
 		// get scraper name from user input, or default out to preferred scraper
@@ -933,7 +1193,6 @@ class frontend{
 		
 		$html = null;
 		
-		//foreach(["web", "images", "videos", "news", "music", "booru"] as $type){
 		foreach(["web", "images", "videos", "news", "music"] as $type){
 			
 			$html .= '<a href="/' . $type . '?s=' . urlencode($query);
@@ -1069,7 +1328,6 @@ class frontend{
 			return htmlspecialchars($image);
 		}
 		
-		//return "https://4get.ca/proxy?i=" . urlencode($image) . "&s=" . $format;
 		return "/proxy?i=" . urlencode($image) . "&s=" . $format;
 	}
 	

lib/fuckhtml.php

@@ -13,7 +13,7 @@ class fuckhtml{
 		
 		if(is_array($html)){
 			
-			if(!array_key_exists("innerHTML", $html)){
+			if(!isset($html["innerHTML"])){
 				
 				throw new Exception("(load) Supplied array doesn't contain an innerHTML index");
 			}
@@ -240,13 +240,12 @@ class fuckhtml{
 	public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){
 		
 		$elems = $this->getElementsByAttributeName($name, $collection);
-		
 		$value =
 			explode(
 				" ",
 				trim(
 					preg_replace(
-						'/\s+/',
+						'/ +/',
 						" ",
 						$value
 					)
@@ -259,18 +258,7 @@ class fuckhtml{
 			
 			foreach($elem["attributes"] as $attrib_name => $attrib_value){
 				
-				$attrib_value =
+				$attrib_value = explode(" ", $attrib_value);
-					explode(
-						" ",
-						trim(
-							preg_replace(
-								'/\s+/',
-								" ",
-								$attrib_value
-							)
-						)
-					);
-				
 				$ac = count($attrib_value);
 				$nc = count($value);
 				$cr = 0;
@@ -339,7 +327,7 @@ class fuckhtml{
 		
 		if(is_array($html)){
 			
-			if(!array_key_exists("innerHTML", $html)){
+			if(!isset($html["innerHTML"])){
 				
 				throw new Exception("(getTextContent) Supplied array doesn't contain an innerHTML index");
 			}
@@ -538,91 +526,4 @@ class fuckhtml{
 				$string
 			);
 	}
-	
-	public function extract_json($json){
-		
-		$len = strlen($json);
-		$array_level = 0;
-		$object_level = 0;
-		$in_quote = null;
-		$start = null;
-		
-		for($i=0; $i<$len; $i++){
-			
-			switch($json[$i]){
-				
-				case "\"":
-				case "'":
-					// count preceding backslashes
-					$bsCount = 0;
-					$j = $i - 1;
-
-					while($j >= 0 && $json[$j] === "\\"){
-						$bsCount++;
-						$j--;
-					}
-
-					// quote is NOT escaped if even number of backslashes
-					if($bsCount % 2 === 0){
-						if($in_quote === null){
-							// open quote
-							$in_quote = $json[$i];
-						}elseif($in_quote === $json[$i]){
-							// close quote
-							$in_quote = null;
-						}
-					}
-					break;
-				
-				case "[":
-					if($in_quote === null){
-						
-						$array_level++;
-						if($start === null){
-							
-							$start = $i;
-						}
-					}
-					break;
-				
-				case "]":
-					if($in_quote === null){
-						
-						$array_level--;
-					}
-					break;
-				
-				case "{":
-					if($in_quote === null){
-						
-						$object_level++;
-						if($start === null){
-							
-							$start = $i;
-						}
-					}
-					break;
-				
-				case "}":
-					if($in_quote === null){
-						
-						$object_level--;
-					}
-					break;
-			}
-			
-			if(
-				$array_level === 0 &&
-				$object_level === 0 &&
-				$start !== null
-			){
-				
-				return substr($json, $start, $i - $start + 1);
-				break;
-			}
-		}
-		
-		// fallback
-		return "[]";
-	}
 }

proxy.php

@@ -34,46 +34,22 @@ try{
 		)
 	){
 		
-		if(!isset($image["path"])){
+		if(
+			!isset($image["query"]) ||
+			!isset($image["path"]) ||
+			$image["path"] != "/th"
+		){
 			
-			header("X-Error: Missing bing image path");
+			header("X-Error: Invalid bing image path");
 			$proxy->do404();
 			die();
 		}
 		
-		//
-		// get image ID
-		// formations:
-		// https://tse2.mm.bing.net/th/id/OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
-		// https://tse2.mm.bing.net/th?id=OIP.3yLBkUPn8EXA1wlhWP2BHwHaE3
-		//
-		$id = null;
-		if(isset($image["query"])){
-			
 		parse_str($image["query"], $str);
 		
-			if(isset($str["id"])){
+		if(!isset($str["id"])){
 			
-				$id = $str["id"];
+			header("X-Error: Missing bing ID");
-			}
-		}
-		
-		if($id === null){
-			
-			$id = explode("/th/id/", $image["path"], 2);
-			
-			if(count($id) !== 2){
-				
-				// malformed
-				return $url;
-			}
-			
-			$id = $id[1];
-		}
-		
-		if(is_array($id)){
-			
-			header("X-Error: Missing bing id parameter");
 			$proxy->do404();
 			die();
 		}
@@ -87,7 +63,7 @@ try{
 			case "cover": $req = "&w=207&h=270&p=0&qlt=90"; break;
 		}
 		
-		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . rawurlencode($id) . $req, "https://www.bing.com");
+		$proxy->stream_linear_image("https://" . $image["host"] . "/th?id=" . urlencode($str["id"]) . $req, "https://www.bing.com");
 		die();
 	}
 	

resolve/sc.php

@@ -1,117 +0,0 @@
-<?php
-
-class sc{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("sc");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		$headers =
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: none",
-			"Sec-Fetch-User: ?1",
-			"Priority: u=0, i"];
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	function resolve($id){
-		
-		if(
-			!preg_match(
-				'/^(t|p)[0-9]+$/',
-				$id
-			)
-		){
-			
-			throw new Exception("ID is invalid");
-		}
-		
-		$type = $id[0] == "t" ? "track" : "playlist";
-		$id = substr($id, 1);
-		
-		try{
-			$html =
-				$this->get(
-					$this->backend->get_ip(),
-					"https://w.soundcloud.com/player/",
-					[
-						"url" => "http://api.soundcloud.com/{$type}s/{$id}",
-						"show_artwork" => "true"
-					]
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch song embed page");
-		}
-		
-		$this->fuckhtml->load($html);
-		
-		$link =
-			$this->fuckhtml
-			->getElementsByAttributeValue(
-				"rel",
-				"canonical",
-				"link"
-			);
-		
-		if(count($link) === 0){
-			
-			throw new Exception("Soundcloud could not resolve the song ID to an URL");
-		}
-		
-		return
-			$this->fuckhtml
-			->getTextContent(
-				$link[0]
-				["attributes"]
-				["href"]
-			);
-	}
-}

resolver.php

@@ -1,66 +0,0 @@
-<?php
-
-$resolver = new resolver();
-
-class resolver{
-	
-	public const resolvers = [
-		"sc"
-	];
-	
-	public function __construct(){
-		
-		include "data/config.php";
-		
-		if(
-			!isset($_GET["scraper"]) ||
-			!is_string($_GET["scraper"]) ||
-			!in_array($_GET["scraper"], self::resolvers)
-		){
-			
-			$this->do400("Missing or invalid scraper");
-			return;
-		}
-		
-		if(
-			!isset($_GET["target"]) ||
-			!is_string($_GET["target"])
-		){
-			
-			$this->do400("Missing or invalid target");
-			return;
-		}
-		
-		$scraper = $_GET["scraper"];
-		$target = $_GET["target"];
-		
-		try{
-			
-			include "resolve/{$scraper}.php";
-			$resolver = new $scraper();
-			$link = $resolver->resolve($target);
-			
-			if(is_string($link)){
-				
-				header("Location: {$link}");
-			}
-		}catch(Exception $error){
-			
-			$this->do404("Fuck! Failed to resolve URL: " . $error->getMessage());
-		}
-	}
-	
-	public function do400($message){
-		
-		header("Content-Type: text/plain");
-		http_response_code(400);
-		echo $message;
-	}
-	
-	public function do404($message){
-		
-		header("Content-Type: text/plain");
-		http_response_code(404);
-		echo $message;
-	}
-}

scraper/brave.php

@@ -210,63 +210,6 @@ class brave{
 		return $data;
 	}
 	
-	private function get_js(){
-		
-		$script_disc =
-			$this->fuckhtml
-			->getElementsByTagName(
-				"script"
-			);
-		
-		$data = null;
-		foreach($script_disc as &$discs){
-			
-			if(
-				preg_match(
-					'/kit\.start\(/',
-					$discs["innerHTML"]
-				)
-			){
-				
-				$data =
-					explode(
-						"data:",
-						$discs["innerHTML"],
-						2
-					);
-				
-				if(count($data) !== 2){
-					
-					throw new Exception("Failed to split up data field");
-				}
-				
-				$data = $data[1];
-				break;
-			}
-		}
-		
-		if($data === null){
-			
-			throw new Exception("Could not grep JavaScript object");
-		}
-		
-		$data =
-			$this->fuckhtml
-			->parseJsObject(
-				$this->fuckhtml
-				->extract_json(
-					$data
-				)
-			);
-		
-		if($data === null){
-			
-			throw new Exception("Failed to decode JavaScript object");
-		}
-		
-		return $data;
-	}
-	
 	public function web($get){
 		
 		if($get["npt"]){
@@ -347,8 +290,12 @@ class brave{
 				$q["spellcheck"] = "0";
 			}
 		}
+		/*
+		$handle = fopen("scraper/brave.html", "r");
+		$html = fread($handle, filesize("scraper/brave.html"));
+		fclose($handle);*/
+		
 		
-		//$html = file_get_contents("scraper/brave.html");
 		try{
 			$html =
 				$this->get(
@@ -388,18 +335,18 @@ class brave{
 		*/
 		$nextpage =
 			$this->fuckhtml
-			->getElementsByClassName(
+			->getElementById(
 				"pagination",
 				"div"
 			);
 		
-		if(count($nextpage) !== 0){
+		if($nextpage){
 			
-			$this->fuckhtml->load($nextpage[0]);
+			$this->fuckhtml->load($nextpage);
 			
 			$nextpage =
 				$this->fuckhtml
-				->getElementsByClassName("button", "a");
+				->getElementsByClassName("btn", "a");
 			
 			if(count($nextpage) !== 0){
 				
@@ -435,9 +382,55 @@ class brave{
 			}
 		}
 		
-		// do some magic
 		$this->fuckhtml->load($html);
-		$data = $this->get_js();
+		
+		$script_disc =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"script"
+			);
+		
+		$grep = [];
+		foreach($script_disc as $discs){
+			
+			preg_match(
+				'/const data ?= ?(\[{.*}]);/',
+				$discs["innerHTML"],
+				$grep
+			);
+			
+			if(isset($grep[1])){
+				
+				break;
+			}
+		}
+		
+		if(!isset($grep[1])){
+			
+			throw new Exception("Could not grep JavaScript object");
+		}
+		
+		$data =
+			rtrim(
+				preg_replace(
+					'/\(Array\(0\)\)\).*$/',
+					"",
+					$grep[1]
+				),
+				" ]"
+			) . "]";
+		
+		$data =
+			$this->fuckhtml
+			->parseJsObject(
+				$data
+			);
+		unset($grep);
+		
+		if($data === null){
+			
+			throw new Exception("Failed to decode JavaScript object");
+		}
 		
 		if(
 			isset($data[2]["data"]["title"]) &&
@@ -1186,8 +1179,23 @@ class brave{
 				$proxy
 			);
 		
-		$this->fuckhtml->load($html);
+		preg_match(
-		$json = $this->get_js();
+			'/const data ?= ?(\[{.*}]);/',
+			$html,
+			$json
+		);
+		
+		if(!isset($json[1])){
+			
+			throw new Exception("Failed to grep javascript object");
+		}
+		
+		$json = $this->fuckhtml->parseJsObject($json[1], true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to parse javascript object");
+		}
 		
 		foreach(
 			$json[1]["data"]["body"]["response"]["news"]["results"]
@@ -1269,8 +1277,22 @@ class brave{
 		$html = fread($handle, filesize("scraper/brave-image.html"));
 		fclose($handle);*/
 		
-		$this->fuckhtml->load($html);
+		preg_match(
-		$json = $this->get_js();
+			'/const data = (\[{.*}\]);/',
+			$html,
+			$json
+		);
+		
+		if(!isset($json[1])){
+			
+			throw new Exception("Failed to get data object");
+		}
+		
+		$json =
+			$this->fuckhtml
+			->parseJsObject(
+				$json[1]
+			);
 		
 		foreach(
 			$json[1]
@@ -1286,13 +1308,13 @@ class brave{
 				"source" => [
 					[
 						"url" => $result["properties"]["url"],
-						"width" => (int)$result["properties"]["width"],
+						"width" => null,
-						"height" => (int)$result["properties"]["height"]
+						"height" => null
 					],
 					[
 						"url" => $result["thumbnail"]["src"],
-						"width" => (int)$result["thumbnail"]["width"],
+						"width" => null,
-						"height" => (int)$result["thumbnail"]["height"]
+						"height" => null
 					]
 				],
 				"url" => $result["url"]
@@ -1400,8 +1422,22 @@ class brave{
 		$html = fread($handle, filesize("scraper/brave-video.html"));
 		fclose($handle);*/
 		
-		$this->fuckhtml->load($html);
+		preg_match(
-		$json = $this->get_js();
+			'/const data = (\[{.*}\]);/',
+			$html,
+			$json
+		);
+		
+		if(!isset($json[1])){
+			
+			throw new Exception("Failed to get data object");
+		}
+		
+		$json =
+			$this->fuckhtml
+			->parseJsObject(
+				$json[1]
+			);
 		
 		foreach(
 			$json
@@ -1773,21 +1809,7 @@ class brave{
 		
 		$nextpage =
 			$this->fuckhtml
-			->getElementsByClassName(
+			->getElementsByClassName("btn", "a");
-				"pagination",
-				"div"
-			);
-		
-		if(count($nextpage) !== 0){
-			
-			$this->fuckhtml->load($nextpage[0]);
-			
-			$nextpage =
-				$this->fuckhtml
-				->getElementsByClassName(
-					"button",
-					"a"
-				);
 		
 		if(count($nextpage) !== 0){
 			
@@ -1825,7 +1847,6 @@ class brave{
 					);
 			}
 		}
-		}
 		
 		return null;
 	}

scraper/cara.php

@@ -1,847 +0,0 @@
-<?php
-
-class cara{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("cara");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"sort" => [
-				"display" => "Sort by",
-				"option" => [
-					"Top" => "Top",
-					"MostRecent" => "Most Recent"
-				]
-			],
-			"type" => [
-				"display" => "Post type",
-				"option" => [
-					"any" => "Any type",
-					"portfolio" => "Portfolio", // {"posts":["portfolio"]}
-					"timeline" => "Timeline" // {"posts":["timeline"]}
-				]
-			],
-			"fields" => [
-				"display" => "Field/Medium",
-				"option" => [
-					"any" => "Any field",
-					"2D" => "2D Work",
-					"3D" => "3D Work",
-					"3DPrinting" => "3D Printing",
-					"Acrylic" => "Acrylic",
-					"AlcoholMarkers" => "Alcohol Markers",
-					"Animation" => "Animation",
-					"Chalk" => "Chalk",
-					"Charcoal" => "Charcoal",
-					"Colored pencil" => "Colored pencil",
-					"Conte" => "Conte",
-					"Crayon" => "Crayon",
-					"Digital" => "Digital",
-					"Gouache" => "Gouache",
-					"Ink" => "Ink",
-					"MixedMedia" => "Mixed-Media",
-					"Oil" => "Oil",
-					"Oil-based Markers" => "Oil-based Markers",
-					"Other" => "Other",
-					"Pastels" => "Pastels",
-					"Photography" => "Photography",
-					"Sculpture" => "Sculpture",
-					"Sketches" => "Sketches",
-					"Tattoos" => "Tattoos",
-					"Traditional" => "Traditional",
-					"VFX" => "VFX",
-					"Watercolor" => "Watercolor"
-				]
-			],
-			"category" => [
-				"display" => "Category",
-				"option" => [
-					"any" => "Any category",
-					"3DScanning" => "3D Scanning",
-					"Abstract" => "Abstract",
-					"Adoptable" => "Adoptable",
-					"Anatomy" => "Anatomy",
-					"Animals" => "Animals",
-					"Anime" => "Anime",
-					"App" => "App",
-					"ArchitecturalConcepts" => "Architectural Concepts",
-					"ArchitecturalVisualization" => "Architectural Visualization",
-					"AugmentedReality" => "Augmented Reality",
-					"Automotive" => "Automotive",
-					"BoardGameArt" => "Board Game Art",
-					"BookIllustration" => "Book Illustration",
-					"CardGameArt" => "Card Game Art",
-					"CeramicsPottery" => "Ceramics/Pottery",
-					"CharacterAnimation" => "Character Animation",
-					"CharacterDesign" => "Character Design",
-					"CharacterModeling" => "Character Modeling",
-					"ChildrensArt" => "Children's Illustration",
-					"Collectibles" => "Collectibles",
-					"ColoringPage" => "Coloring Page",
-					"ComicArt" => "Comic Art",
-					"ConceptArt" => "Concept Art",
-					"Cosplay" => "Cosplay",
-					"CostumeDesign" => "Costume Design",
-					"CoverArt" => "Cover Art",
-					"Creatures" => "Creatures",
-					"Diorama" => "Diorama",
-					"EditorialIllustration" => "Editorial Illustration",
-					"EmbroiderySewing" => "Embroidery/Sewing",
-					"EnvironmentalConceptArt" => "Environmental Concept Art",
-					"EnvironmentalConceptDesign" => "Environmental Concept Design",
-					"FanArt" => "Fan Art",
-					"Fantasy" => "Fantasy",
-					"Fashion" => "Fashion",
-					"FashionStyling" => "Fashion Styling",
-					"FiberArts" => "Fiber Arts",
-					"Furry" => "Furry",
-					"GameArt" => "Game Art",
-					"GameplayDesign" => "Gameplay Design",
-					"GamesEnvironmentArt" => "Games Environment Art",
-					"Gem" => "Gem",
-					"GraphicDesign" => "Graphic Design",
-					"Handicraft" => "Handicraft",
-					"HairStyling" => "Hair Styling",
-					"HardSurface" => "Hard Surface",
-					"Horror" => "Horror",
-					"Illustration" => "Illustration",
-					"IllustrationVisualization" => "Illustration Visualization",
-					"IndustrialDesign" => "Industrial Design",
-					"Jewelry" => "Jewelry",
-					"KnittingCrochet" => "Knitting/Crochet",
-					"Landscape" => "Landscape",
-					"LevelDesign" => "Level Design",
-					"Lighting" => "Lighting",
-					"Makeup" => "Makeup",
-					"Manga" => "Manga",
-					"MapsCartography" => "Maps/Cartography",
-					"MattePainting" => "Matte Painting",
-					"Materials" => "Materials",
-					"MechanicalDesign" => "Mechanical Design",
-					"Medical" => "Medical",
-					"Mecha" => "Mecha",
-					"MiniatureArt" => "Miniature Art",
-					"MotionGraphics" => "Motion Graphics",
-					"FrescoMurals" => "Fresco/Murals",
-					"Natural" => "Natural",
-					"Original Character" => "Original Character",
-					"Overlay" => "Overlay",
-					"PleinAir" => "Plein Air",
-					"Photogrammetry" => "Photogrammetry",
-					"PixelArt" => "Pixel Art",
-					"Portraits" => "Portraits",
-					"Props" => "Props",
-					"ProductDesign" => "Product Design",
-					"PublicDomain" => "Public Domain or Royalty Free",
-					"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
-					"Realism" => "Realism",
-					"ScienceFiction" => "Science Fiction",
-					"ScientificVisualization" => "Scientific Visualization",
-					"Scripts" => "Scripts",
-					"StillLife" => "Still Life",
-					"Storyboards" => "Storyboards",
-					"Stylized" => "Stylized",
-					"Surreal" => "Surreal",
-					"TechnicalArt" => "Technical Art",
-					"Textures" => "Textures",
-					"Tools" => "Tools",
-					"Toys" => "Toys",
-					"ToyPackaging" => "Toy Packaging",
-					"Tutorials" => "Tutorials",
-					"UIArt" => "User Interface (UI) Art",
-					"UrbanSketch" => "Urban Sketch",
-					"VFXforAnimation" => "VFX for Animation",
-					"VFXforFilm" => "VFX for Film",
-					"VFXforGames" => "VFX for Games",
-					"VFXforRealTime" => "VFX for Real-Time",
-					"VFXforTV" => "VFX for TV",
-					"Vehicles" => "Vehicles",
-					"VirtualReality" => "Virtual Reality",
-					"VisualDevelopment" => "Visual Development",
-					"VoxelArt" => "Voxel Art",
-					"Vtubers" => "Vtubers",
-					"WIP" => "WIP (Work in Progress)",
-					"Web" => "Web",
-					"Weapons" => "Weapons",
-					"Wildlife" => "Wildlife",
-					"Woodcutting" => "Woodcutting"
-				]
-			],
-			"software" => [
-				"display" => "Software",
-				"option" => [
-					"any" => "Any software",
-					"123D" => "123D",
-					"123DCatch" => "123D Catch",
-					"3DBee" => "3DBee",
-					"3DCoat" => "3DCoat",
-					"3DCoatPrint" => "3DCoatPrint",
-					"3DCoatTextura" => "3DCoatTextura",
-					"3DEqualizer" => "3DEqualizer",
-					"3DFZephyr" => "3DF Zephyr",
-					"3Delight" => "3Delight",
-					"3dpeople" => "3dpeople",
-					"3dsMax" => "3ds Max",
-					"3DSPaint" => "3DS Paint",
-					"ACDSeeCanvas" => "ACDSee Canvas",
-					"AbletonLive" => "Ableton Live",
-					"Acrobat" => "Acrobat",
-					"AdobeDraw" => "Adobe Draw",
-					"AdobeFlash" => "Adobe Flash",
-					"AdobeFresco" => "Adobe Fresco",
-					"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
-					"AdobeXD" => "Adobe XD",
-					"AffinityDesigner" => "Affinity Designer",
-					"AffinityPhoto" => "Affinity Photo",
-					"AfterEffects" => "After Effects",
-					"Akeytsu" => "Akeytsu",
-					"Alchemy" => "Alchemy",
-					"AliasDesign" => "Alias Design",
-					"AlightMotion" => "Alight Motion",
-					"Amadine" => "Amadine",
-					"Amberlight" => "Amberlight",
-					"Animate" => "Animate",
-					"AnimationMaster" => "Animation:Master",
-					"AnimeStudio" => "Anime Studio",
-					"Apophysis" => "Apophysis",
-					"ArchiCAD" => "ArchiCAD",
-					"Arion" => "Arion",
-					"ArionFX" => "ArionFX",
-					"Arnold" => "Arnold",
-					"ArtEngine" => "ArtEngine",
-					"ArtFlow" => "ArtFlow",
-					"ArtRage" => "ArtRage",
-					"ArtstudioPro" => "Artstudio Pro",
-					"Artweaver" => "Artweaver",
-					"Aseprite" => "Aseprite",
-					"Audition" => "Audition",
-					"AutoCAD" => "AutoCAD",
-					"AutodeskSketchBook" => "Autodesk SketchBook",
-					"AvidMediaComposer" => "Avid Media Composer",
-					"AzPainter" => "AzPainter",
-					"babylonjs" => "babylon.js",
-					"BalsamiqMockup" => "Balsamiq Mockup",
-					"Bforartists" => "Bforartists",
-					"BlackInk" => "Black Ink",
-					"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
-					"Blender" => "Blender",
-					"Blender DeepPaint" => "Blender DeepPaint",
-					"BlenderGreasePencil" => "Blender Grease Pencil",
-					"Blockbench" => "Blockbench",
-					"BodyPaint" => "BodyPaint",
-					"Boxcutter" => "Boxcutter",
-					"BraidMaker" => "Braid Maker",
-					"BrickLinkStudio" => "BrickLink Studio",
-					"Bridge" => "Bridge",
-					"Brushifyio" => "Brushify.io",
-					"C" => "C",
-					"C#" => "C#",
-					"C++" => "C++",
-					"CACANi" => "CACANi",
-					"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
-					"CLO" => "CLO",
-					"CRYENGINE" => "CRYENGINE",
-					"Callipeg" => "Callipeg",
-					"Canva" => "Canva",
-					"CaptureOne" => "Capture One",
-					"CartoonAnimator" => "Cartoon Animator",
-					"Carveco" => "Carveco",
-					"Cavalry" => "Cavalry",
-					"Chaotica" => "Chaotica",
-					"CharacterAnimator" => "Character Animator",
-					"CharacterCreator" => "Character Creator",
-					"Cinema4D" => "Cinema 4D",
-					"ClarisseiFX" => "Clarisse iFX",
-					"Coiffure" => "Coiffure",
-					"ColorsLive" => "Colors Live",
-					"Combustion" => "Combustion",
-					"Construct2" => "Construct 2",
-					"Core" => "Core",
-					"CorelPainter" => "Corel Painter",
-					"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
-					"CoronaRenderer" => "Corona Renderer",
-					"ProMotionNG" => "Cosmigo Pro Motion NG",
-					"CrazyBump" => "CrazyBump",
-					"Crocotile3D" => "Crocotile 3D",
-					"Curvy3D" => "Curvy 3D",
-					"Cycles4D" => "Cycles 4D",
-					"Darkroom" => "Darkroom",
-					"DAZStudio" => "DAZ Studio",
-					"DDO" => "DDO",
-					"DECIMA" => "DECIMA",
-					"Darktable" => "Darktable",
-					"DaVinciResolve" => "DaVinci Resolve",
-					"Dimension" => "Dimension",
-					"DragonBones" => "DragonBones",
-					"Dragonframe" => "Dragonframe",
-					"Drawpile" => "Drawpile",
-					"Dreams" => "Dreams",
-					"Dreamweaver" => "Dreamweaver",
-					"DxOPhotoLab" => "DxO PhotoLab",
-					"ECycles" => "E-Cycles",
-					"EmberGen" => "EmberGen",
-					"Encore" => "Encore",
-					"Expresii" => "Expresii",
-					"FStorm" => "FStorm",
-					"FadeIn" => "FadeIn",
-					"Feather3D" => "Feather 3D",
-					"FiberShop" => "FiberShop",
-					"Figma" => "Figma",
-					"FilmoraWondershare" => "Filmora Wondershare",
-					"FilterForge" => "Filter Forge",
-					"FinalCutPro" => "Final Cut Pro",
-					"FinalDraft" => "Final Draft",
-					"finalRender" => "finalRender",
-					"FireAlpaca" => "FireAlpaca",
-					"Fireworks" => "Fireworks",
-					"FlamePainter" => "Flame Painter",
-					"Flash" => "Flash",
-					"FlipaClip" => "FlipaClip",
-					"FlipnoteStudio" => "Flipnote Studio",
-					"Fluent" => "Fluent",
-					"ForestPack" => "Forest Pack",
-					"FormZ" => "Form-Z",
-					"Fractorium" => "Fractorium",
-					"FreeCAD" => "FreeCAD",
-					"FreeHand" => "FreeHand",
-					"Forger" => "Forger",
-					"FrostbiteEngine" => "Frostbite Engine",
-					"fSpy" => "fSpy",
-					"FumeFX" => "FumeFX",
-					"Fusion360" => "Fusion 360",
-					"GIMP" => "GIMP",
-					"GSCurveTools" => "GS CurveTools",
-					"GSToolbox" => "GS Toolbox",
-					"Gaea" => "Gaea",
-					"GameTextures" => "Game Textures",
-					"GameMakerStudio" => "GameMaker: Studio",
-					"GarageFarmNET" => "GarageFarm.NET",
-					"GeoGlyph" => "GeoGlyph",
-					"GigapixelAl" => "Gigapixel Al",
-					"Glaxnimate" => "Glaxnimate",
-					"GnomePaint" => "Gnome Paint",
-					"Godot" => "Godot",
-					"Goxel" => "Goxel",
-					"Graphite" => "Graphite",
-					"Graswald" => "Graswald",
-					"GravitySketch" => "Gravity Sketch",
-					"GuerillaRender" => "GuerillaRender",
-					"HDRLightStudio" => "HDR Light Studio",
-					"HairStrandDesigner" => "Hair Strand Designer",
-					"HairTGHairFur" => "HairTG - Hair &amp; Fur",
-					"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
-					"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
-					"Handplane" => "Handplane",
-					"Hansoft" => "Hansoft",
-					"HardOps" => "Hard Ops",
-					"HardMesh" => "HardMesh",
-					"Harmony" => "Harmony",
-					"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
-					"HelloPaint" => "HelloPaint",
-					"HeliconFocus" => "Helicon Focus",
-					"Hexels" => "Hexels",
-					"HiPaint" => "HiPaint",
-					"Houdini" => "Houdini",
-					"HydraRenderer" => "Hydra Renderer",
-					"iArtbook" => "iArtbook",
-					"IbisPaint" => "ibisPaint",
-					"Ideas" => "Ideas",
-					"IllustStudio" => "Illust Studio",
-					"Illustrator" => "Illustrator",
-					"IllustratorDraw" => "Illustrator Draw",
-					"InDesign" => "InDesign",
-					"Inochi2D" => "Inochi2D",
-					"InVision" => "InVision",
-					"InVisionCraft" => "InVision Craft",
-					"InfinitePainter" => "Infinite Painter",
-					"Inkscape" => "Inkscape",
-					"Inspirit" => "Inspirit",
-					"InstaLOD" => "InstaLOD",
-					"InstaMAT" => "InstaMAT",
-					"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
-					"InstantMeshes" => "Instant Meshes",
-					"InstantTerra" => "Instant Terra",
-					"Inventor" => "Inventor",
-					"Iray" => "Iray",
-					"JWildfire" => "JWildfire",
-					"Java" => "Java",
-					"Jira" => "Jira",
-					"JumpPaint" => "Jump Paint by MediBang",
-					"JSPaint" => "JS Paint",
-					"Katana" => "Katana",
-					"Keyshot" => "Keyshot",
-					"KidPix" => "Kid Pix",
-					"KitBash3D" => "KitBash3D",
-					"Knald" => "Knald",
-					"Kodon" => "Kodon",
-					"KolourPaint" => "KolourPaint",
-					"Krakatoa" => "Krakatoa",
-					"KRESKA" => "KRESKA",
-					"Krita" => "Krita",
-					"LensStudio" => "Lens Studio",
-					"LibreSprite" => "LibreSprite",
-					"LightWave3D" => "LightWave 3D",
-					"Lightroom" => "Lightroom",
-					"Linearity" => "Linearity",
-					"LiquiGen" => "LiquiGen",
-					"Live2DCubism" => "Live2D Cubism",
-					"LookatmyHair" => "Look at my Hair",
-					"Lotpixel" => "Lotpixel",
-					"Lumion" => "Lumion",
-					"LuxRender" => "LuxRender",
-					"MacPaint" => "MacPaint",
-					"MagicaCSG" => "MagicaCSG",
-					"MagicaVoxel" => "MagicaVoxel",
-					"Magma" => "Magma",
-					"MakeHuman" => "MakeHuman",
-					"Malmal" => "Malmal",
-					"Mandelbulb3D" => "Mandelbulb 3D",
-					"Mandelbulber" => "Mandelbulber",
-					"MangaStudio" => "Manga Studio",
-					"Mari" => "Mari",
-					"MarmosetToolbag" => "Marmoset Toolbag",
-					"MarvelousDesigner" => "Marvelous Designer",
-					"MasterpieceStudioPro" => "Masterpiece Studio Pro",
-					"MasterpieceVR" => "MasterpieceVR",
-					"Maverick" => "Maverick",
-					"MaxwellRender" => "Maxwell Render",
-					"Maya" => "Maya",
-					"MediBangPaint" => "MediBang Paint",
-					"MediumbyAdobe" => "Medium by Adobe",
-					"Megascans" => "Megascans",
-					"mentalray" => "mental ray",
-					"MeshLab" => "MeshLab",
-					"Meshroom" => "Meshroom",
-					"MetaHumanCreator" => "MetaHuman Creator",
-					"Metashape" => "Metashape",
-					"MightyBake" => "MightyBake",
-					"MikuMikuDance" => "MikuMikuDance",
-					"Minecraft" => "Minecraft",
-					"Mischief" => "Mischief",
-					"Mixamo" => "Mixamo",
-					"Mixer" => "Mixer",
-					"MoI3D" => "MoI3D",
-					"Mocha" => "Mocha",
-					"Modo" => "Modo",
-					"Moho" => "Moho",
-					"MotionBuilder" => "MotionBuilder",
-					"Mudbox" => "Mudbox",
-					"Muse" => "Muse",
-					"MSPaint" => "MS Paint",
-					"MyPaint" => "MyPaint",
-					"NDO" => "NDO",
-					"NX" => "NX",
-					"NdotCAD" => "NdotCAD",
-					"NintendoNotes" => "Nintendo Notes",
-					"NomadSculpt" => "Nomad Sculpt",
-					"Notability" => "Notability",
-					"Nuke" => "Nuke",
-					"Nvil" => "Nvil",
-					"OctaneRender" => "Octane Render",
-					"Omniverse" => "Omniverse",
-					"OmniverseCreate" => "Omniverse Create",
-					"ON1PhotoRAW" => "ON1 Photo RAW",
-					"Open3DEngine" => "Open 3D Engine",
-					"OpenCanvas" => "OpenCanvas",
-					"OpenGL" => "OpenGL",
-					"OpenToonz" => "OpenToonz",
-					"Ornatrix" => "Ornatrix",
-					"OsciRender" => "Osci-Render",
-					"OurPaint" => "Our Paint",
-					"PBRMAX" => "PBRMAX",
-					"PFTrack" => "PFTrack",
-					"PTGui" => "PTGui",
-					"Paintbrush" => "Paintbrush",
-					"PaintNET" => "Paint.NET",
-					"PaintShopPro" => "PaintShop Pro",
-					"PaintToolSAI" => "Paint Tool SAI",
-					"PaintstormStudio" => "Paintstorm Studio",
-					"Paper" => "Paper",
-					"Pencil2D" => "Pencil2D",
-					"Penpot" => "Penpot",
-					"PhoenixFD" => "Phoenix FD",
-					"Phonto" => "Phonto",
-					"PhotoLab2" => "PhotoLab 2",
-					"Photopea" => "Photopea",
-					"Photoscan" => "Photoscan",
-					"Photoshop" => "Photoshop",
-					"PhotoshopElements" => "Photoshop Elements",
-					"PicoCAD" => "picoCAD",
-					"PicoCAD2" => "picoCAD 2",
-					"Pinta" => "Pinta",
-					"Piskel" => "Piskel",
-					"Pixilart" => "Pixilart",
-					"Pixelitor" => "Pixelitor",
-					"Pixelmator" => "Pixelmator",
-					"Pixelorama" => "Pixelorama",
-					"PixivSketch" => "pixiv Sketch",
-					"Pixquare" => "Pixquare",
-					"PlantCatalog" => "PlantCatalog",
-					"PlantFactory" => "PlantFactory",
-					"Plasticity" => "Plasticity",
-					"PNGtuberPlus" => "PNGtuber Plus",
-					"Poliigon" => "Poliigon",
-					"Polybrush" => "Polybrush",
-					"PopcornFx" => "PopcornFx",
-					"Poser" => "Poser",
-					"Premiere" => "Premiere",
-					"PremiereElements" => "Premiere Elements",
-					"PresagisCreator" => "Presagis Creator",
-					"ProTools" => "Pro Tools",
-					"Procreate" => "Procreate",
-					"ProcreateDreams" => "Procreate Dreams",
-					"Producer" => "Producer",
-					"PrometheanAI" => "Promethean AI",
-					"PureRef" => "PureRef",
-					"Python" => "Python",
-					"PyxelEdit" => "PyxelEdit",
-					"QuadRemesher" => "Quad Remesher",
-					"QuarkXPress" => "QuarkXPress",
-					"Qubicle" => "Qubicle",
-					"Quill" => "Quill",
-					"QuixelBridge" => "Quixel Bridge",
-					"QuixelMegascans" => "Quixel Megascans",
-					"QuixelMixer" => "Quixel Mixer",
-					"QuixelSuite" => "Quixel Suite",
-					"R3DSWrap" => "R3DS Wrap",
-					"R3DSZWRAP" => "R3DS ZWRAP",
-					"RDTextures" => "RD-Textures",
-					"RailClone" => "RailClone",
-					"RealFlow" => "RealFlow",
-					"RealisticPaintStudio" => "Realistic Paint Studio",
-					"RealityCapture" => "RealityCapture",
-					"RealityScan" => "RealityScan",
-					"RealtimeBoard" => "Realtime Board",
-					"Rebelle" => "Rebelle",
-					"Redshift" => "Redshift",
-					"RenderMan" => "RenderMan",
-					"RenderNetwork" => "Render Network",
-					"Revit" => "Revit",
-					"Rhino" => "Rhino",
-					"Rhinoceros" => "Rhinoceros",
-					"RizomUV" => "RizomUV",
-					"RoughAnimator" => "Rough Animator",
-					"SamsungNotes" => "Samsung Notes",
-					"SamsungPENUP" => "Samsung PENUP",
-					"ScansLibrary" => "ScansLibrary",
-					"Scrivener" => "Scrivener",
-					"Sculpt+" => "Sculpt+",
-					"Sculptris" => "Sculptris",
-					"ShaveandaHaircut" => "Shave and a Haircut",
-					"ShiVa3D" => "ShiVa3D",
-					"Shotgun" => "Shotgun",
-					"Silo" => "Silo",
-					"Silugen" => "Silugen",
-					"Sketch" => "Sketch",
-					"SketchApp" => "Sketch App",
-					"SketchBookPro" => "SketchBook Pro",
-					"SketchClub" => "SketchClub",
-					"SketchUp" => "SketchUp",
-					"Sketchable" => "Sketchable",
-					"Sketchfab" => "Sketchfab",
-					"Skyshop" => "Skyshop",
-					"Snapseed" => "Snapseed",
-					"Snowdrop" => "Snowdrop",
-					"Softimage" => "Softimage",
-					"SolidWorks" => "SolidWorks",
-					"SonySketch" => "Sony Sketch",
-					"Soundbooth" => "Soundbooth",
-					"Source2" => "Source 2",
-					"SourceControl" => "Source Control",
-					"SourceFilmmaker" => "Source Filmmaker",
-					"SpeedTree" => "SpeedTree",
-					"Speedgrade" => "Speedgrade",
-					"SpeedyPainter" => "SpeedyPainter",
-					"Spine2D" => "Spine 2D",
-					"Spriter" => "Spriter",
-					"Stingray" => "Stingray",
-					"Storyboarder" => "Storyboarder",
-					"StoryboardPro" => "Storyboard Pro",
-					"SublimeText" => "Sublime Text",
-					"Substance3DDesigner" => "Substance 3D Designer",
-					"Substance3DModeler" => "Substance 3D Modeler",
-					"Substance3DPainter" => "Substance 3D Painter",
-					"Substance3DSampler" => "Substance 3D Sampler",
-					"Substance3DStager" => "Substance 3D Stager",
-					"SubstanceB2M" => "Substance B2M",
-					"SweetHome3D" => "Sweet Home 3D",
-					"SynthEyes" => "SynthEyes",
-					"TTools" => "TTools",
-					"TVPaint" => "TVPaint",
-					"TVPaintAnimation" => "TVPaint Animation",
-					"TayasuiSketches" => "Tayasui Sketches",
-					"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
-					"TayasuiSketchesPro" => "Tayasui Sketches Pro",
-					"Terragen" => "Terragen",
-					"Texturescom" => "Textures.com",
-					"Texturingxyz" => "Texturingxyz",
-					"TeyaConceptor" => "Teya Conceptor",
-					"TheGrove3D" => "The Grove 3D",
-					"TheaRender" => "Thea Render",
-					"Threejs" => "Three.js",
-					"Tiled" => "Tiled",
-					"TiltBrush" => "Tilt Brush",
-					"Tooll3" => "Tooll3",
-					"ToonBoomHarmony" => "Toon Boom Harmony",
-					"ToonBoomStudio" => "Toon Boom Studio",
-					"ToonSquid" => "ToonSquid",
-					"TopoGun" => "TopoGun",
-					"TuxPaint" => "Tux Paint",
-					"Tvori" => "Tvori",
-					"Twinmotion" => "Twinmotion",
-					"UNIGINEEngine" => "UNIGINE Engine",
-					"UVLayout" => "UVLayout",
-					"UltraFractal" => "Ultra Fractal",
-					"uMake" => "uMake",
-					"Unfold3D" => "Unfold 3D",
-					"Unity" => "Unity",
-					"UnrealEngine" => "Unreal Engine",
-					"Vengi" => "vengi",
-					"VRay" => "V-Ray",
-					"VRED" => "VRED",
-					"VTubeStudio" => "VTube Studio",
-					"Vectary" => "Vectary",
-					"VectorayGen" => "VectorayGen",
-					"Vectorworks" => "Vectorworks",
-					"VegasPro" => "Vegas Pro",
-					"VisualDesigner3D" => "Visual Designer 3D",
-					"VisualStudio" => "Visual Studio",
-					"VRoidStudio" => "VRoid Studio",
-					"Vue" => "Vue",
-					"Vuforia" => "Vuforia",
-					"WebGL" => "WebGL",
-					"WhiteboardFox" => "Whiteboard Fox",
-					"WickEditor" => "Wick Editor",
-					"Wings3D" => "Wings 3D",
-					"Word" => "Word",
-					"WorldCreator" => "World Creator",
-					"WorldMachine" => "World Machine",
-					"XParticles" => "X-Particles",
-					"Xfrog" => "Xfrog",
-					"Xgen" => "Xgen",
-					"xNormal" => "xNormal",
-					"xTex" => "xTex",
-					"XoliulShader" => "Xoliul Shader",
-					"Yafaray" => "Yafaray",
-					"Yeti" => "Yeti",
-					"ZBrush" => "ZBrush",
-					"ZBrushCore" => "ZBrushCore",
-					"ZenBrush" => "Zen Brush"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $search){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
-			//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
-			"Referer: https://cara.app/search?q=" . urlencode($search),
-			//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$npt, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$npt = json_decode($npt, true);
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-						
-			$npt = [
-				"q" => $get["s"],
-				"sortBy" => $get["sort"],
-				"take" => 24,
-				"skip" => 0,
-				"filters" => []
-			];
-			
-			// parse filters
-			if($get["type"] != "any"){
-				
-				$npt["filters"]["posts"] = [$get["type"]];
-			}
-			
-			if($get["fields"] != "any"){
-				
-				$npt["filters"]["fields"] = [$get["fields"]];
-			}
-			
-			if($get["category"] != "any"){
-				
-				$npt["filters"]["categories"] = [$get["category"]];
-			}
-			
-			if($get["software"] != "any"){
-				
-				$npt["filters"]["softwares"] = [$get["software"]];
-			}
-			
-			if($npt["filters"] == []){
-				
-				$npt["filters"] = "{}";
-			}else{
-				
-				$npt["filters"] = json_encode($npt["filters"]);
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://cara.app/api/search/posts",
-					$npt,
-					$npt["q"]
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$imagecount = 0;
-		foreach($json as $image){
-			
-			if(count($image["images"]) === 0){
-				
-				// sometimes the api returns no images for an object
-				$imagecount++;
-				continue;
-			}
-			
-			$cover = null;
-			$sources = [];
-			
-			foreach($image["images"] as $source){
-				
-				if($source["isCoverImg"]){
-					
-					$cover = [
-						"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
-						"width" => 500,
-						"height" => 500
-					];
-				}else{
-					
-					$sources[] = [
-						"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
-						"width" => null,
-						"height" => null
-					];
-				}
-			}
-			
-			if($cover !== null){
-				
-				$sources[] = $cover;
-			}
-			
-			$out["image"][] = [
-				"title" => str_replace("\n", " ", $image["content"]),
-				"source" => $sources,
-				"url" => "https://cara.app/post/" . $image["id"]
-			];
-			
-			$imagecount++;
-		}
-		
-		if($imagecount === 24){
-			
-			$npt["skip"] += 24;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($npt),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function fix_url($url){
-		
-		return
-			str_replace(
-				[" "],
-				["%20"],
-				$url
-			);
-	}
-}

scraper/coccoc.php

@@ -1,680 +0,0 @@
-<?php
-
-class coccoc{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("coccoc");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
-			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			//"Cookie: _contentAB_15040_vi=V-06_01; split_test_search=new_search; uid=L_bauXyZBY1B; vid=uCVQJQSTgb9QGT3o; ls=1753742684; serp_version=29223843/7621a70; savedS=direct",
-			"Upgrade-Insecure-Requests: 1",
-			"Sec-Fetch-Dest: document",
-			"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: cross-site",
-			"Priority: u=0, i"
-		]);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function getfilters($pagetype){
-		
-		return [
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes", // nsfw by default????
-					"no" => "No" // &safe=1
-				]
-			],
-			"time" => [
-				"display" => "Time posted",
-				"option" => [
-					"any" => "Any time",
-					"1w" => "1 week ago",
-					"2w" => "2 weeks ago",
-					"1m" => "1 month ago",
-					"3m" => "3 months ago",
-					"6m" => "6 months ago",
-					"1Y" => "1 year ago"
-				]
-			],
-			"filter" => [
-				"display" => "Remove duplicates",
-				"option" => [
-					"no" => "No",
-					"yes" => "Yes" // &filter=0
-				]
-			]
-		];
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$query, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"web"
-				);
-			
-			$query = json_decode($query, true);
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			$query = [
-				"query" => $get["s"]
-			];
-			
-			// add filters
-			if($get["nsfw"] == "no"){
-				
-				$query["safe"] = 1;
-			}
-			
-			if($get["time"] != "any"){
-				
-				$query["tbs"] = $get["time"];
-			}
-			
-			if($get["filter"] == "yes"){
-				
-				$query["filter"] = 0;
-			}
-		}
-		
-		try{
-			
-			$html =
-				$this->get(
-					$proxy,
-					"https://coccoc.com/search",
-					$query
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to get search page");
-		}
-		//$html = file_get_contents("scraper/coccoc.html");
-		
-		
-		$html = explode("window.composerResponse", $html, 2);
-		
-		if(count($html) !== 2){
-			
-			throw new Exception("Failed to grep window.composerResponse");
-		}
-		
-		$html =
-			json_decode(
-				$this->fuckhtml
-				->extract_json(
-					ltrim($html[1], " =")
-				),
-				true
-			);
-		
-		if($html === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(
-			isset($html["captcha"]) &&
-			(int)$html["captcha"] === 1
-		){
-			
-			throw new Exception("Coc Coc returned a Captcha");
-		}
-		
-		if(!isset($html["search"]["search_results"])){
-			
-			throw new Exception("Coc Coc did not return a search_results object");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		// word correction
-		foreach($html["top"] as $element){
-			
-			if(isset($element["spellChecker"][0]["query"])){
-				
-				$out["spelling"] = [
-					"type" => "not_many",
-					"using" => $html["search"]["query"],
-					"correction" => $element["spellChecker"][0]["query"]
-				];
-			}
-		}
-		
-		foreach($html["search"]["search_results"] as $result){
-			
-			if(isset($result["type"])){
-				
-				switch($result["type"]){
-					
-					//
-					// Related searches
-					//
-					case "related_queries":
-						$out["related"] = $result["queries"];
-						continue 2;
-					
-					//
-					// Videos
-					//
-					case "video_hits":
-						foreach($result["results"] as $video){
-							
-							if(
-								isset($video["image_url"]) &&
-								!empty($video["image_url"])
-							){
-								
-								$thumb = [
-									"ratio" => "16:9",
-									"url" => $video["image_url"]
-								];
-							}else{
-								
-								$thumb = [
-									"ratio" => null,
-									"url" => null
-								];
-							}
-							
-							$out["video"][] = [
-								"title" =>
-									$this->titledots(
-										$this->fuckhtml
-										->getTextContent(
-											$video["title"]
-										)
-									),
-								"description" => null,
-								"author" => [
-									"name" => $video["uploader"],
-									"url" => null,
-									"avatar" => null
-								],
-								"date" => (int)$video["date"],
-								"duration" => (int)$video["duration"],
-								"views" => null,
-								"thumb" => $thumb,
-								"url" => $video["url"]
-							];
-						}
-						continue 2;
-				}
-			}
-			
-			if(
-				!isset($result["title"]) ||
-				!isset($result["url"])
-			){
-				
-				// should not happen
-				continue;
-			}
-			
-			if(isset($result["rich"]["data"]["image_url"])){
-				
-				$thumb = [
-					"url" => $result["rich"]["data"]["image_url"],
-					"ratio" => "16:9"
-				];
-			}else{
-				
-				$thumb = [
-					"url" => null,
-					"ratio" => null
-				];
-			}
-			
-			$sublinks = [];
-			
-			if(isset($result["rich"]["data"]["linked_docs"])){
-				
-				foreach($result["rich"]["data"]["linked_docs"] as $sub){
-					
-					$sublinks[] = [
-						"title" =>
-							$this->titledots(
-								$this->fuckhtml
-								->getTextContent(
-									$sub["title"]
-								)
-							),
-						"description" =>
-							$this->titledots(
-								$this->fuckhtml
-								->getTextContent(
-									$sub["content"]
-								)
-							),
-						"date" => null,
-						"url" => $sub["url"]
-					];
-				}
-			}
-			
-			// get date
-			if(isset($result["date"])){
-				
-				$date = (int)$result["date"];
-			}else{
-				
-				$date = null;
-			}
-			
-			// probe for metadata
-			$table = [];
-			
-			if(isset($result["rich"]["data"]["rating"])){
-				
-				$table["Rating"] = $result["rich"]["data"]["rating"];
-				
-				if(isset($result["rich"]["data"]["num_rating"])){
-					
-					$table["Rating"] .= " (" . number_format($result["rich"]["data"]["num_rating"]) . " ratings)";
-				}
-			}
-			
-			if(isset($result["rich"]["data"]["views"])){
-				
-				$table["Views"] = number_format($result["rich"]["data"]["views"]);
-			}
-			
-			if(isset($result["rich"]["data"]["duration"])){
-				
-				$table["Duration"] = $this->int2hms($result["rich"]["data"]["duration"]);
-			}
-			
-			if(isset($result["rich"]["data"]["channel_name"])){
-				
-				$table["Author"] = $result["rich"]["data"]["channel_name"];
-			}
-			
-			if(isset($result["rich"]["data"]["video_quality"])){
-				
-				$table["Quality"] = $result["rich"]["data"]["video_quality"];
-			}
-			
-			if(isset($result["rich"]["data"]["category"])){
-				
-				$table["Category"] = $result["rich"]["data"]["category"];
-			}
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$result["title"]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$result["content"]
-						)
-					),
-				"url" => $result["url"],
-				"date" => $date,
-				"type" => "web",
-				"thumb" => $thumb,
-				"sublink" => $sublinks,
-				"table" => $table
-			];
-		}
-		
-		//
-		// Get wikipedia head
-		//
-		if(isset($html["right"])){
-			
-			foreach($html["right"] as $wiki){
-				
-				$description = [];
-				
-				if(isset($wiki["short_intro"])){
-					
-					$description[] =
-						[
-							"type" => "quote",
-							"value" => $wiki["short_intro"],
-						];
-				}
-				
-				if(isset($wiki["intro"])){
-					
-					$description[] =
-						[
-							"type" => "text",
-							"value" => $wiki["intro"],
-						];
-				}
-				
-				// get table elements
-				$table = [];
-				
-				if(isset($wiki["fields"])){
-					
-					foreach($wiki["fields"] as $element){
-						
-						$table[$element["title"]] = implode(", ", $element["value"]);
-					}
-				}
-				
-				// get sublinks
-				$sublinks = [];
-				
-				if(isset($wiki["website"])){
-					
-					if(
-						preg_match(
-							'/^http/',
-							$wiki["website"]
-						) === 0
-					){
-						
-						$sublinks["Website"] = "https://" . $wiki["website"];
-					}else{
-						
-						$sublinks["Website"] = $wiki["website"];
-					}
-				}
-				
-				foreach($wiki["profiles"] as $sitename => $url){
-					
-					$sitename = explode("_", $sitename);
-					$sitename = ucfirst($sitename[count($sitename) - 1]);
-					
-					$sublinks[$sitename] = $url;
-				}
-				
-				$out["answer"][] = [
-					"title" =>
-						$this->titledots(
-							$wiki["title"]
-						),
-					"description" => $description,
-					"url" => null,
-					"thumb" => isset($wiki["image"]["contentUrl"]) ? $wiki["image"]["contentUrl"] : null,
-					"table" => $table,
-					"sublink" => $sublinks
-				];
-			}
-		}
-		
-		// get next page
-		if((int)$html["search"]["page"] < (int)$html["search"]["max_page"]){
-			
-			// https://coccoc.com/composer?_=1754021153532&p=0&q=zbabduiqwhduwqhdnwq&reqid=bwcAs00q&s=direct&apiV=1
-			// ^json endpoint, but we can just do &page=2 lol
-			
-			if(!isset($query["page"])){
-				
-				$query["page"] = 2;
-			}else{
-				
-				$query["page"]++;
-			}
-			
-			$out["npt"] =
-				$this->backend
-				->store(
-					json_encode($query),
-					"web",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	public function video($get){
-		
-		//$html = file_get_contents("scraper/coccoc.html");
-		if($get["npt"]){
-			
-			[$query, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"videos"
-				);
-			
-			$query = json_decode($query, true);
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-			
-			$query = [
-				"query" => $get["s"],
-				"tbm" => "vid"
-			];
-			
-			// add filters
-			if($get["nsfw"] == "no"){
-				
-				$query["safe"] = 1;
-			}
-			
-			if($get["time"] != "any"){
-				
-				$query["tbs"] = $get["time"];
-			}
-			
-			if($get["filter"] == "yes"){
-				
-				$query["filter"] = 0;
-			}
-		}
-		
-		try{
-			
-			$html =
-				$this->get(
-					$proxy,
-					"https://coccoc.com/search",
-					$query
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to get search page");
-		}
-		
-		$html = explode("window.composerResponse", $html, 2);
-		
-		if(count($html) !== 2){
-			
-			throw new Exception("Failed to grep window.composerResponse");
-		}
-		
-		$html =
-			json_decode(
-				$this->fuckhtml
-				->extract_json(
-					ltrim($html[1], " =")
-				),
-				true
-			);
-		
-		if($html === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"video" => [],
-			"author" => [],
-			"livestream" => [],
-			"playlist" => [],
-			"reel" => []
-		];
-		
-		if(!isset($html["search_video"]["search_results"])){
-			
-			if(isset($html["search_video"]["error"]["title"])){
-				
-				if($html["search_video"]["error"]["title"] == "Không tìm thấy kết quả nào"){
-					
-					return $out;
-				}
-				
-				throw new Exception("Coc Coc returned an error: " . $html["search_video"]["error"]["title"]);
-			}
-			
-			throw new Exception("Coc Coc did not supply a search_results object");
-		}
-		
-		foreach($html["search_video"]["search_results"] as $video){
-			
-			if(isset($video["rich"]["data"]["image_url"])){
-				
-				$thumb = [
-					"ratio" => "16:9",
-					"url" => $video["rich"]["data"]["image_url"]
-				];
-			}else{
-				
-				$thumb = [
-					"ratio" => null,
-					"url" => null
-				];
-			}
-			
-			$out["video"][] = [
-				"title" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$video["title"]
-						)
-					),
-				"description" =>
-					$this->titledots(
-						$this->fuckhtml
-						->getTextContent(
-							$video["content"]
-						)
-					),
-				"author" => [
-					"name" => 
-						isset($video["rich"]["data"]["channel_name"]) ?
-							$video["rich"]["data"]["channel_name"] : null,
-					"url" => null,
-					"avatar" => null
-				],
-				"date" =>
-					isset($video["date"]) ?
-						$video["date"] : null,
-				"duration" =>
-					isset($video["rich"]["data"]["duration"]) ?
-						(int)$video["rich"]["data"]["duration"] : null,
-				"views" => null,
-				"thumb" => $thumb,
-				"url" => $video["url"]
-			];
-		}
-		
-		// get next page
-		if((int)$html["search_video"]["page"] < (int)$html["search_video"]["max_page"]){
-			
-			if(!isset($query["page"])){
-				
-				$query["page"] = 2;
-			}else{
-				
-				$query["page"]++;
-			}
-			
-			$out["npt"] =
-				$this->backend
-				->store(
-					json_encode($query),
-					"videos",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-	
-	private function int2hms($seconds){
-		
-		$hours = floor($seconds / 3600);
-		$minutes = floor(($seconds % 3600) / 60);
-		$seconds = $seconds % 60;
-		
-		return sprintf("%02d:%02d:%02d", $hours, $minutes, $seconds);
-	}
-}

scraper/crowdview.php

@@ -0,0 +1,145 @@
+<?php
+
+class crowdview{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("crowdview");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+
+		return [];
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function web($get){
+		
+		$search = $get["s"];
+		if(strlen($search) === 0){
+			
+			throw new Exception("Search term is empty!");
+		}
+		
+		$proxy = $this->backend->get_ip();
+		
+		try{
+			$json = $this->get(
+				$proxy,
+				"https://crowdview-next-js.onrender.com/api/search-v3",
+				[
+					"query" => $search
+				]
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		$json = json_decode($json, true);
+		
+		if($json === NULL){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		foreach($json["results"] as $item){
+			
+			$description = explode("<b>", $item["snippet"], 2);
+			
+			$out["web"][] = [
+				"title" => $this->sanitize($item["title"]),
+				"description" => $this->sanitize($description[1]),
+				"url" => $item["link"],
+				"date" => strtotime($description[0]),
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		return $out;
+	}
+	
+	private function sanitize($html){
+		
+		return
+			trim(
+				$this->fuckhtml
+				->getTextContent(
+					html_entity_decode(
+						$html
+					)
+				),
+				". "
+			);
+	}
+}

scraper/curlie.php

@@ -0,0 +1,309 @@
+<?php
+
+class curlie{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("curlie");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		if($page != "web"){
+			
+			return [];
+		}
+		
+		return [
+			"lang" => [
+				"display" => "Language",
+				"option" => [
+					"any" => "Any language",
+					"en" => "English",
+					"de" => "German",
+					"fr" => "French",
+					"ja" => "Japanese",
+					"it" => "Italian",
+					"es" => "Spanish",
+					"ru" => "Russian",
+					"nl" => "Dutch",
+					"pl" => "Polish",
+					"tr" => "Turkish",
+					"da" => "Danish",
+					"sv" => "Swedish",
+					"no" => "Norwegian",
+					"is" => "Icelandic",
+					"fo" => "Faroese",
+					"fi" => "Finnish",
+					"et" => "Estonian",
+					"lt" => "Lithuanian",
+					"lv" => "Latvian",
+					"cy" => "Welsh",
+					"ga" => "Irish",
+					"gd" => "Scottish Gaelic",
+					"br" => "Breton",
+					"fy" => "Frisian",
+					"frr" => "North Frisian",
+					"gem" => "Saterland Frisian",
+					"lb" => "Luxembourgish",
+					"rm" => "Romansh",
+					"pt" => "Portuguese",
+					"ca" => "Catalan",
+					"gl" => "Galician",
+					"eu" => "Basque",
+					"ast" => "Asturian",
+					"an" => "Aragonese",
+					"fur" => "Friulan",
+					"sc" => "Sardinian",
+					"scn" => "Sicilian",
+					"oc" => "Occitan",
+					"be" => "Belarusian",
+					"cs" => "Czech",
+					"hu" => "Hungarian",
+					"sk" => "Slovak",
+					"uk" => "Ukrainian",
+					"csb" => "Kashubian",
+					"tt" => "Tatar",
+					"ba" => "Bashkir",
+					"os" => "Ossetian",
+					"sl" => "Slovene",
+					"sr" => "Serbian",
+					"hr" => "Croatian",
+					"bs" => "Bosnian",
+					"bg" => "Bulgarian",
+					"sq" => "Albanian",
+					"ro" => "Romanian",
+					"mk" => "Macedonian",
+					"el" => "Greek",
+					"iw" => "Hebrew",
+					"fa" => "Persian",
+					"ar" => "Arabic",
+					"ku" => "Kurdish",
+					"az" => "Azerbaijani",
+					"hy" => "Armenian",
+					"af" => "Afrikaans",
+					"sw" => "Kiswahili",
+					"uz" => "Uzbek",
+					"kk" => "Kazakh",
+					"ky" => "Kyrgyz",
+					"tg" => "Tajik",
+					"tk" => "Turkmen",
+					"ug" => "Uyghurche",
+					"hi" => "Hindi",
+					"si" => "Sinhalese",
+					"gu" => "Gujarati",
+					"ur" => "Urdu",
+					"mr" => "Marathi",
+					"pa" => "Punjabi",
+					"bn" => "Bengali",
+					"ta" => "Tamil",
+					"te" => "Telugu",
+					"kn" => "Kannada",
+					"zh_CN" => "Chinese Simplified",
+					"zh_TW" => "Chinese Traditional",
+					"ko" => "Korean",
+					"cfr" => "Taiwanese",
+					"th" => "Thai",
+					"vi" => "Vietnamese",
+					"in" => "Indonesian",
+					"ms" => "Malay",
+					"tl" => "Tagalog",
+					"eo" => "Esperanto",
+					"ia" => "Interlingua",
+					"la" => "Latin"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function web($get){
+		
+		if($get["npt"]){
+			
+			[$query, $proxy] = $this->backend->get($get["npt"], "web");
+			
+			try{
+				$html = $this->get(
+					$proxy,
+					"https://curlie.org/" . $query,
+					[]
+				);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search page");
+			}
+			
+		}else{
+			$proxy = $this->backend->get_ip();
+			
+			$query = [
+				"q" => $get["s"],
+				"start" => 0,
+				"stime" => 92452189 // ?
+			];
+			
+			if($get["lang"] !== "any"){
+				
+				$query["lang"] = $get["lang"];
+			}
+			
+			try{
+				$html = $this->get(
+					$proxy,
+					"https://curlie.org/search",
+					$query
+				);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search page");
+			}
+		}
+		
+		$this->fuckhtml->load($html);
+		
+		$nextpage =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"next-page",
+				"a"
+			);
+		
+		if(count($nextpage) !== 0){
+			
+			$nextpage =
+				$this->backend->store(
+					$nextpage[0]["attributes"]["href"],
+					"web",
+					$proxy
+				);
+		}else{
+			
+			$nextpage = null;
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => $nextpage,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		$items =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"site-item",
+				"div"
+			);
+		
+		foreach($items as $item){
+			
+			$this->fuckhtml->load($item);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByAttributeValue(
+					"target",
+					"_blank",
+					"a"
+				)[0];
+			
+			$description =
+				$this->fuckhtml
+				->getElementsByClassName("site-descr");
+			
+			if(count($description) !== 0){
+				
+				$description =
+					$this->fuckhtml
+					->getTextContent(
+						$description[0]
+					);
+			}else{
+				
+				$description = null;
+			}
+			
+			$out["web"][] = [
+				"title" =>
+					$this->fuckhtml
+					->getTextContent(
+						$a
+					),
+				"description" => $description,
+				"url" =>
+					$this->fuckhtml
+					->getTextContent(
+						$a["attributes"]["href"]
+					),
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		return $out;
+	}
+}

scraper/fivehpx.php

@@ -1,262 +0,0 @@
-<?php
-
-class fivehpx{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("fivehpx");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"sort" => [
-				"display" => "Sort",
-				"option" => [
-					"relevance" => "Relevance",
-					"pulse" => "Pulse",
-					"newest" => "Newest"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $post_data = null){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		if($post_data === null){
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
-				"TE: trailers"]
-			);
-		}else{
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"Referer: https://500px.com/",
-				"content-type: application/json",
-				//"x-csrf-token: undefined",
-				"x-500px-source: Search",
-				"Content-Length: " . strlen($post_data),
-				"Origin: https://500px.com",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				// "Cookie: _pin_unauth, _fbp, _sharedID, _sharedID_cst",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-site",
-				"Priority: u=4",
-				"TE: trailers"]
-			);
-						
-			// set post data
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$pagination, $proxy] =
-				$this->backend->get(
-					$get["npt"], "images"
-				);
-			
-			$pagination = json_decode($pagination, true);
-			$search = $pagination["search"];
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			$pagination = [
-				"sort" => strtoupper($get["sort"]),
-				"search" => $search,
-				"filters" => [],
-				"nlp" => false,
-			];
-		}
-		
-		try{
-			
-			$json =
-				$this->get(
-					$proxy,
-					"https://api.500px.com/graphql",
-					[],
-					json_encode([
-						"operationName" => "PhotoSearchPaginationContainerQuery",
-						"variables" => $pagination,
-						"query" =>
-							'query PhotoSearchPaginationContainerQuery(' .
-							(isset($pagination["cursor"]) ? '$cursor: String, ' : "") .
-							'$sort: PhotoSort, $search: String!, $filters: [PhotoSearchFilter!], $nlp: Boolean) {  ...PhotoSearchPaginationContainer_query_1vzAZD} fragment PhotoSearchPaginationContainer_query_1vzAZD on Query { photoSearch(sort: $sort, first: 100, ' .
-							(isset($pagination["cursor"]) ? 'after: $cursor, ' : "") .
-							'search: $search, filters: $filters, nlp: $nlp) { edges { node { id legacyId canonicalPath name description width height images(sizes: [33, 36]) { size url id } } } totalCount pageInfo { endCursor hasNextPage } }}'
-					])
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch graphQL object");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode graphQL object");
-		}
-		
-		if(isset($json["errors"][0]["message"])){
-			
-			throw new Exception("500px returned an API error: " . $json["errors"][0]["message"]);
-		}
-		
-		if(!isset($json["data"]["photoSearch"]["edges"])){
-			
-			throw new Exception("No edges returned by API");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		foreach($json["data"]["photoSearch"]["edges"] as $image){
-			
-			$image = $image["node"];
-			$title =
-				trim(
-					$this->fuckhtml
-					->getTextContent(
-						$image["name"]
-					) . ": " .
-					$this->fuckhtml
-					->getTextContent(
-						$image["description"]
-					)
-					, " :"
-				);
-			
-			$small = $this->image_ratio(600, $image["width"], $image["height"]);
-			$large = $this->image_ratio(2048, $image["width"], $image["height"]);
-			
-			$out["image"][] = [
-				"title" => $title,
-				"source" => [
-					[
-						"url" => $image["images"][1]["url"],
-						"width" => $large[0],
-						"height" => $large[1]
-					],
-					[
-						"url" => $image["images"][0]["url"],
-						"width" => $small[0],
-						"height" => $small[1]
-					]
-				],
-				"url" => "https://500px.com" . $image["canonicalPath"]
-			];
-		}
-		
-		// get NPT token
-		if($json["data"]["photoSearch"]["pageInfo"]["hasNextPage"] === true){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode([
-						"cursor" => $json["data"]["photoSearch"]["pageInfo"]["endCursor"],
-						"search" => $search,
-						"sort" => $pagination["sort"],
-						"filters" => [],
-						"nlp" => false
-					]),
-					"images",
-					$proxy
-				);
-		}
-			
-		return $out;
-	}
-	
-	private function image_ratio($longest_edge, $width, $height){
-		
-		$ratio = [
-			$longest_edge / $width,
-			$longest_edge / $height
-		];
-		
-		if($ratio[0] < $ratio[1]){
-			
-			$ratio = $ratio[0];
-		}else{
-			
-			$ratio = $ratio[1];
-		}
-		
-		return [
-			floor($width * $ratio),
-			floor($height * $ratio)
-		];
-	}
-}

scraper/flickr.php

@@ -1,415 +0,0 @@
-<?php
-
-class flickr{
-	
-	const req_web = 0;
-	const req_xhr = 1;
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("flickr");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes",
-					"maybe" => "Maybe",
-					"no" => "No",
-				]
-			],
-			"sort" => [
-				"display" => "Sort by",
-				"option" => [
-					"relevance" => "Relevance",
-					"date-posted-desc" => "Newest uploads",
-					"date-posted-asc" => "Oldest uploads",
-					"date-taken-desc" => "Newest taken",
-					"date-taken-asc" => "Oldest taken",
-					"interestingness-desc" => "Interesting"
-				]
-			],
-			"color" => [
-				"display" => "Color",
-				"option" => [
-					"any" => "Any color",
-					// color_codes=
-					"0" => "Red",
-					"1" => "Brown",
-					"2" => "Orange",
-					"b" => "Pink",
-					"4" => "Yellow",
-					"3" => "Golden",
-					"5" => "Lime",
-					"6" => "Green",
-					"7" => "Sky blue",
-					"8" => "Blue",
-					"9" => "Purple",
-					"a" => "Hot pink",
-					"c" => "White",
-					"d" => "Gray",
-					"e" => "Black",
-					// styles= override
-					"blackandwhite" => "Black & white",
-				]
-			],
-			"style" => [ // styles=
-				"display" => "Style",
-				"option" => [
-					"any" => "Any style",
-					"depthoffield" => "Depth of field",
-					"minimalism" => "Minimalism",
-					"pattern" => "Patterns"
-				]
-			],
-			"license" => [
-				"display" => "License",
-				"option" => [
-					"any" => "Any license",
-					"1,2,3,4,5,6,9,11,12,13,14,15,16" => "All creative commons",
-					"4,5,6,9,10,11,12,13" => "Commercial use allowed",
-					"1,2,4,5,9,10,11,12,14,15" => "Modifications allowed",
-					"4,5,9,10,11,12" => "Commercial use & mods allowed",
-					"7,9,10" => "No known copyright restrictions",
-					"8" => "U.S Government works"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $reqtype){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		if($reqtype === flickr::req_web){
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: same-origin",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i",
-				"TE: trailers"]
-			);
-		}else{
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"Origin: https://www.flickr.com",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Referer: https://www.flickr.com/",
-				// Cookie:
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-site",
-				"TE: trailers"]
-			);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$filters, $proxy] =
-				$this->backend->get(
-					$get["npt"], "images"
-				);
-			
-			$filters = json_decode($filters, true);
-			
-			// Workaround for the future, if flickr deprecates &page argument on html page
-			/*
-			try{
-				
-				$json =
-					$this->get(
-						$proxy,
-						"https://api.flickr.com/services/rest",
-						[
-							"sort" => $data["sort"],
-							"parse_tags" => 1,
-							// url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o
-							"extras" => "can_comment,can_print,count_comments,count_faves,description,isfavorite,license,media,needs_interstitial,owner_name,path_alias,realname,rotation,url_sq,url_q,url_t,url_s,url_n,url_w,url_m,url_z,url_c,url_l",
-							"per_page" => 100,
-							"page" => $data["page"],
-							"lang" => "en-US",
-							"text" => $data["search"],
-							"viewerNSID" => "",
-							"method" => "flickr.photos.search",
-							"csrf" => "",
-							"api_key" => $data["api_key"],
-							"format" => "json",
-							"hermes" => 1,
-							"hermesClient" => 1,
-							"reqId" => $data["reqId"],
-							"nojsoncallback" => 1
-						]
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JSON");
-			}*/
-			
-		}else{
-			
-			if(strlen($get["s"]) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-						
-			// compute filters
-			$filters = [
-				"page" => 1,
-				"sort" => $get["sort"]
-			];
-			
-			if($get["style"] != "any"){
-				
-				$filters["styles"] = $get["style"];
-			}
-			
-			if($get["color"] != "any"){
-				
-				if($get["color"] != "blackandwhite"){
-					
-					$filters["color_codes"] = $get["color"];
-				}else{
-					
-					$filters["styles"] = "blackandwhite";
-				}
-			}
-			
-			if($get["license"] != "any"){
-				
-				$filters["license"] = $get["license"];
-			}
-			
-			switch($get["nsfw"]){
-				
-				case "yes": $filters["safe_search"] = 0; break;
-				case "maybe": $filters["safe_search"] = 2; break;
-				case "no": $filters["safe_search"] = 1; break;
-			}
-		}
-		
-		$get_params = [
-			"text" => $get["s"],
-			"per_page" => 50,
-			// scrape highest resolution
-			"extras" => "url_s,url_n,url_w,url_m,url_z,url_c,url_l,url_h,url_k,url_3k,url_4k,url_5k,url_6k,url_o",
-			"view_all" => 1
-		];
-		
-		$get_params = array_merge($get_params, $filters);
-		
-		$html =
-			$this->get(
-				$proxy,
-				"https://www.flickr.com/search/",
-				$get_params,
-				flickr::req_web
-			);
-		
-		// @TODO
-		// get api_key and reqId, if flickr deprecates &page
-		
-		$this->fuckhtml->load($html);
-		
-		//
-		// get response JSON
-		//
-		$scripts =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"modelExport",
-				"script"
-			);
-		
-		$found = false;
-		foreach($scripts as $script){
-			
-			$json =	
-				preg_split(
-					'/modelExport: ?/',
-					$script["innerHTML"],
-					2
-				);
-			
-			if(count($json) !== 0){
-				
-				$found = true;
-				$json = $json[1];
-				break;
-			}
-		}
-		
-		if($found === false){
-			
-			throw new Exception("Failed to grep JSON");
-		}
-		
-		$json =
-			json_decode(
-				$this->fuckhtml
-				->extract_json(
-					$json
-				),
-				true
-			);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if(!isset($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"])){
-			
-			throw new Exception("Failed to access data object");
-		}
-		
-		foreach($json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["_data"] as $image){
-			
-			if(!isset($image["data"])){
-				
-				// flickr likes to gives us empty array objects
-				continue;
-			}
-			
-			$image = $image["data"];
-			
-			$title = [];
-			
-			if(isset($image["title"])){
-				
-				$title[] =
-					$this->fuckhtml
-					->getTextContent(
-						$image["title"]
-					);
-			}
-			
-			if(isset($image["description"])){
-				
-				$title[] =
-					$this->fuckhtml
-					->getTextContent(
-						str_replace(
-							"\n",
-							" ",
-							$image["description"]
-						)
-					);
-			}
-			
-			$title = implode(": ", $title);
-			
-			$sources = array_values($image["sizes"]["data"]);
-			
-			$suitable_sizes = ["n", "m", "w", "s"];
-			
-			$thumb = &$sources[0]["data"];
-			foreach($suitable_sizes as $testing_size){
-				
-				if(isset($image["sizes"]["data"][$testing_size])){
-					
-					$thumb = &$image["sizes"]["data"][$testing_size]["data"];
-					break;
-				}
-			}
-			
-			$og = &$sources[count($sources) - 1]["data"];
-			
-			$out["image"][] = [
-				"title" => $title,
-				"source" => [
-					[
-						"url" => "https:" . $og["displayUrl"],
-						"width" => (int)$og["width"],
-						"height" => (int)$og["height"]
-					],
-					[
-						"url" => "https:" . $thumb["displayUrl"],
-						"width" => (int)$thumb["width"],
-						"height" => (int)$thumb["height"]
-					]
-				],
-				"url" => "https://www.flickr.com/photos/" . $image["ownerNsid"] . "/" . $image["id"] . "/"
-			];
-		}
-		
-		$total_items = (int)$json["main"]["search-photos-lite-models"][0]["data"]["photos"]["data"]["totalItems"];
-		
-		if(($filters["page"]) * 50 < $total_items){
-			
-			$filters["page"]++;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($filters),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-}

scraper/ftm.php

@@ -136,7 +136,7 @@ class ftm{
 				"source" => [
 					[
 						"url" =>
-							"https://s3.thehackerblog.com/findthatmeme/" .
+							"https://findthatmeme.us-southeast-1.linodeobjects.com/" .
 							$thumb,
 						"width" => null,
 						"height" => null

scraper/ghostery.php

@@ -0,0 +1,320 @@
+<?php
+
+class ghostery{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("ghostery");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		if($page != "web"){
+			
+			return [];
+		}
+		
+		return [
+			"country" => [
+				"display" => "Country",
+				"option" => [
+					"any" => "All regions",
+					"AR" => "Argentina",
+					"AU" => "Australia",
+					"AT" => "Austria",
+					"BE" => "Belgium",
+					"BR" => "Brazil",
+					"CA" => "Canada",
+					"CL" => "Chile",
+					"DK" => "Denmark",
+					"FI" => "Finland",
+					"FR" => "France",
+					"DE" => "Germany",
+					"HK" => "Hong Kong",
+					"IN" => "India",
+					"ID" => "Indonesia",
+					"IT" => "Italy",
+					"JP" => "Japan",
+					"KR" => "Korea",
+					"MY" => "Malaysia",
+					"MX" => "Mexico",
+					"NL" => "Netherlands",
+					"NZ" => "New Zealand",
+					"NO" => "Norway",
+					"CN" => "People's Republic of China",
+					"PL" => "Poland",
+					"PT" => "Portugal",
+					"PH" => "Republic of the Philippines",
+					"RU" => "Russia",
+					"SA" => "Saudi Arabia",
+					"ZA" => "South Africa",
+					"ES" => "Spain",
+					"SE" => "Sweden",
+					"CH" => "Switzerland",
+					"TW" => "Taiwan",
+					"TR" => "Turkey",
+					"GB" => "United Kingdom",
+					"US" => "United States"
+				]
+			]
+		];
+	}
+	
+	private function get($proxy, $url, $get = [], $country){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"Referer: https://ghosterysearch.com",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Cookie: ctry=" . ($country == "any" ? "--" : $country) . "; noads=true",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: same-origin",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	public function web($get){
+		
+		if($get["npt"]){
+			
+			[$query, $proxy] = $this->backend->get($get["npt"], "web");
+			
+			parse_str($query, $query);
+			
+			// country
+			$country = $query["c"];
+			unset($query["c"]);
+			
+			$query = http_build_query($query);
+			
+			try{
+				
+				$html =
+					$this->get(
+						$proxy,
+						"https://ghosterysearch.com/search?" . $query,
+						[],
+						$country
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search page");
+			}
+		}else{
+			
+			$proxy = $this->backend->get_ip();
+			
+			try{
+				
+				$html =
+					$this->get(
+						$proxy,
+						"https://ghosterysearch.com/search",
+						[
+							"q" => $get["s"]
+						],
+						$get["country"]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search page");
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		$this->fuckhtml->load($html);
+		
+		$results_wrapper =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"results",
+				"section"
+			);
+		
+		if(count($results_wrapper) === 0){
+			
+			throw new Exception("Failed to grep result section");
+		}
+		
+		$this->fuckhtml->load($results_wrapper[0]);
+		
+		// get search results
+		$results =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"result",
+				"li"
+			);
+		
+		if(count($results) === 0){
+			
+			return $out;
+		}
+		
+		foreach($results as $result){
+			
+			$this->fuckhtml->load($result);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"url",
+					"a"
+				);
+			
+			if(count($a) === 0){
+				
+				continue;
+			}
+			
+			$a = $a[0];
+			
+			$out["web"][] = [
+				"title" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$this->fuckhtml
+							->getElementsByTagName(
+								"h2"
+							)[0]
+						)
+					),
+				"description" =>
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$this->fuckhtml
+							->getElementsByTagName(
+								"p"
+							)[0]
+						)
+					),
+				"url" =>
+					$this->fuckhtml
+					->getTextContent(
+						$a
+						["attributes"]
+						["href"]
+					),
+				"date" => null,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		$this->fuckhtml->load($html);
+		
+		// get pagination token
+		$pagination_wrapper =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"pagination",
+				"div"
+			);
+		
+		if(count($pagination_wrapper) !== 0){
+			
+			// found next page!
+			$this->fuckhtml->load($pagination_wrapper[0]);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"a"
+				);
+			
+			if(count($a) !== 0){
+				
+				$q =
+					parse_url(
+						$this->fuckhtml
+						->getTextContent(
+							$a[count($a) - 1]
+							["attributes"]
+							["href"]
+						),
+						PHP_URL_QUERY
+					);
+				
+				$out["npt"] =
+					$this->backend
+					->store(
+						$q . "&c=" . $get["country"],
+						"web",
+						$proxy
+					);
+			}
+		}
+		
+		return $out;
+	}
+	
+	private function titledots($title){
+		
+		return trim($title, " .\t\n\r\0\x0B…");
+	}
+}

scraper/google_api.php

@@ -1,870 +0,0 @@
-<?php
-
-// @TODO check for consent.google.com page, if need be
-
-class google_api{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("google_api");
-	}
-	
-	public function getfilters($page){
-		
-		$base = [
-			"country" => [ // gl=<country> (image: cr=countryAF)
-				"display" => "Country",
-				"option" => [
-					"any" => "Instance's country",
-					"af" => "Afghanistan",
-					"al" => "Albania",
-					"dz" => "Algeria",
-					"as" => "American Samoa",
-					"ad" => "Andorra",
-					"ao" => "Angola",
-					"ai" => "Anguilla",
-					"aq" => "Antarctica",
-					"ag" => "Antigua and Barbuda",
-					"ar" => "Argentina",
-					"am" => "Armenia",
-					"aw" => "Aruba",
-					"au" => "Australia",
-					"at" => "Austria",
-					"az" => "Azerbaijan",
-					"bs" => "Bahamas",
-					"bh" => "Bahrain",
-					"bd" => "Bangladesh",
-					"bb" => "Barbados",
-					"by" => "Belarus",
-					"be" => "Belgium",
-					"bz" => "Belize",
-					"bj" => "Benin",
-					"bm" => "Bermuda",
-					"bt" => "Bhutan",
-					"bo" => "Bolivia",
-					"ba" => "Bosnia and Herzegovina",
-					"bw" => "Botswana",
-					"bv" => "Bouvet Island",
-					"br" => "Brazil",
-					"io" => "British Indian Ocean Territory",
-					"bn" => "Brunei Darussalam",
-					"bg" => "Bulgaria",
-					"bf" => "Burkina Faso",
-					"bi" => "Burundi",
-					"kh" => "Cambodia",
-					"cm" => "Cameroon",
-					"ca" => "Canada",
-					"cv" => "Cape Verde",
-					"ky" => "Cayman Islands",
-					"cf" => "Central African Republic",
-					"td" => "Chad",
-					"cl" => "Chile",
-					"cn" => "China",
-					"cx" => "Christmas Island",
-					"cc" => "Cocos (Keeling) Islands",
-					"co" => "Colombia",
-					"km" => "Comoros",
-					"cg" => "Congo",
-					"cd" => "Congo, the Democratic Republic",
-					"ck" => "Cook Islands",
-					"cr" => "Costa Rica",
-					"ci" => "Cote D'ivoire",
-					"hr" => "Croatia",
-					"cu" => "Cuba",
-					"cy" => "Cyprus",
-					"cz" => "Czech Republic",
-					"dk" => "Denmark",
-					"dj" => "Djibouti",
-					"dm" => "Dominica",
-					"do" => "Dominican Republic",
-					"ec" => "Ecuador",
-					"eg" => "Egypt",
-					"sv" => "El Salvador",
-					"gq" => "Equatorial Guinea",
-					"er" => "Eritrea",
-					"ee" => "Estonia",
-					"et" => "Ethiopia",
-					"fk" => "Falkland Islands (Malvinas)",
-					"fo" => "Faroe Islands",
-					"fj" => "Fiji",
-					"fi" => "Finland",
-					"fr" => "France",
-					"gf" => "French Guiana",
-					"pf" => "French Polynesia",
-					"tf" => "French Southern Territories",
-					"ga" => "Gabon",
-					"gm" => "Gambia",
-					"ge" => "Georgia",
-					"de" => "Germany",
-					"gh" => "Ghana",
-					"gi" => "Gibraltar",
-					"gr" => "Greece",
-					"gl" => "Greenland",
-					"gd" => "Grenada",
-					"gp" => "Guadeloupe",
-					"gu" => "Guam",
-					"gt" => "Guatemala",
-					"gn" => "Guinea",
-					"gw" => "Guinea-Bissau",
-					"gy" => "Guyana",
-					"ht" => "Haiti",
-					"hm" => "Heard Island and Mcdonald Islands",
-					"va" => "Holy See (Vatican City State)",
-					"hn" => "Honduras",
-					"hk" => "Hong Kong",
-					"hu" => "Hungary",
-					"is" => "Iceland",
-					"in" => "India",
-					"id" => "Indonesia",
-					"ir" => "Iran, Islamic Republic",
-					"iq" => "Iraq",
-					"ie" => "Ireland",
-					"il" => "Israel",
-					"it" => "Italy",
-					"jm" => "Jamaica",
-					"jp" => "Japan",
-					"jo" => "Jordan",
-					"kz" => "Kazakhstan",
-					"ke" => "Kenya",
-					"ki" => "Kiribati",
-					"kp" => "Korea, Democratic People's Republic",
-					"kr" => "Korea, Republic",
-					"kw" => "Kuwait",
-					"kg" => "Kyrgyzstan",
-					"la" => "Lao People's Democratic Republic",
-					"lv" => "Latvia",
-					"lb" => "Lebanon",
-					"ls" => "Lesotho",
-					"lr" => "Liberia",
-					"ly" => "Libyan Arab Jamahiriya",
-					"li" => "Liechtenstein",
-					"lt" => "Lithuania",
-					"lu" => "Luxembourg",
-					"mo" => "Macao",
-					"mk" => "Macedonia, the Former Yugosalv Republic",
-					"mg" => "Madagascar",
-					"mw" => "Malawi",
-					"my" => "Malaysia",
-					"mv" => "Maldives",
-					"ml" => "Mali",
-					"mt" => "Malta",
-					"mh" => "Marshall Islands",
-					"mq" => "Martinique",
-					"mr" => "Mauritania",
-					"mu" => "Mauritius",
-					"yt" => "Mayotte",
-					"mx" => "Mexico",
-					"fm" => "Micronesia, Federated States",
-					"md" => "Moldova, Republic",
-					"mc" => "Monaco",
-					"mn" => "Mongolia",
-					"ms" => "Montserrat",
-					"ma" => "Morocco",
-					"mz" => "Mozambique",
-					"mm" => "Myanmar",
-					"na" => "Namibia",
-					"nr" => "Nauru",
-					"np" => "Nepal",
-					"nl" => "Netherlands",
-					"an" => "Netherlands Antilles",
-					"nc" => "New Caledonia",
-					"nz" => "New Zealand",
-					"ni" => "Nicaragua",
-					"ne" => "Niger",
-					"ng" => "Nigeria",
-					"nu" => "Niue",
-					"nf" => "Norfolk Island",
-					"mp" => "Northern Mariana Islands",
-					"no" => "Norway",
-					"om" => "Oman",
-					"pk" => "Pakistan",
-					"pw" => "Palau",
-					"ps" => "Palestinian Territory, Occupied",
-					"pa" => "Panama",
-					"pg" => "Papua New Guinea",
-					"py" => "Paraguay",
-					"pe" => "Peru",
-					"ph" => "Philippines",
-					"pn" => "Pitcairn",
-					"pl" => "Poland",
-					"pt" => "Portugal",
-					"pr" => "Puerto Rico",
-					"qa" => "Qatar",
-					"re" => "Reunion",
-					"ro" => "Romania",
-					"ru" => "Russian Federation",
-					"rw" => "Rwanda",
-					"sh" => "Saint Helena",
-					"kn" => "Saint Kitts and Nevis",
-					"lc" => "Saint Lucia",
-					"pm" => "Saint Pierre and Miquelon",
-					"vc" => "Saint Vincent and the Grenadines",
-					"ws" => "Samoa",
-					"sm" => "San Marino",
-					"st" => "Sao Tome and Principe",
-					"sa" => "Saudi Arabia",
-					"sn" => "Senegal",
-					"cs" => "Serbia and Montenegro",
-					"sc" => "Seychelles",
-					"sl" => "Sierra Leone",
-					"sg" => "Singapore",
-					"sk" => "Slovakia",
-					"si" => "Slovenia",
-					"sb" => "Solomon Islands",
-					"so" => "Somalia",
-					"za" => "South Africa",
-					"gs" => "South Georgia and the South Sandwich Islands",
-					"es" => "Spain",
-					"lk" => "Sri Lanka",
-					"sd" => "Sudan",
-					"sr" => "Suriname",
-					"sj" => "Svalbard and Jan Mayen",
-					"sz" => "Swaziland",
-					"se" => "Sweden",
-					"ch" => "Switzerland",
-					"sy" => "Syrian Arab Republic",
-					"tw" => "Taiwan, Province of China",
-					"tj" => "Tajikistan",
-					"tz" => "Tanzania, United Republic",
-					"th" => "Thailand",
-					"tl" => "Timor-Leste",
-					"tg" => "Togo",
-					"tk" => "Tokelau",
-					"to" => "Tonga",
-					"tt" => "Trinidad and Tobago",
-					"tn" => "Tunisia",
-					"tr" => "Turkey",
-					"tm" => "Turkmenistan",
-					"tc" => "Turks and Caicos Islands",
-					"tv" => "Tuvalu",
-					"ug" => "Uganda",
-					"ua" => "Ukraine",
-					"ae" => "United Arab Emirates",
-					"uk" => "United Kingdom",
-					"us" => "United States",
-					"um" => "United States Minor Outlying Islands",
-					"uy" => "Uruguay",
-					"uz" => "Uzbekistan",
-					"vu" => "Vanuatu",
-					"ve" => "Venezuela",
-					"vn" => "Viet Nam",
-					"vg" => "Virgin Islands, British",
-					"vi" => "Virgin Islands, U.S.",
-					"wf" => "Wallis and Futuna",
-					"eh" => "Western Sahara",
-					"ye" => "Yemen",
-					"zm" => "Zambia",
-					"zw" => "Zimbabwe"
-				]
-			],
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes", // safe=active
-					"no" => "No" // safe=off
-				]
-			],
-			"sort" => [ // sort
-				"display" => "Sort by",
-				"option" => [
-					"any" => "Any order",
-					"date:d" => "Oldest",
-					"date:a" => "Newest"
-				]
-			],
-			"newer" => [
-				"display" => "Newer than",
-				"option" => "_DATE"
-			],			
-			"rm_dupes" => [ // filter
-				"display" => "Remove duplicates",
-				"option" => [
-					"yes" => "Yes", // 1
-					"no" => "No" // 0
-				]
-			]
-		];
-		
-		switch($page){
-			
-			case "web":
-				return array_merge(
-					$base,
-					[
-						"lang" => [ // lr=<lang> (prefix lang with "lang_")
-							"display" => "Language",
-							"option" => [
-								"any" => "Any language",
-								"ar" => "Arabic",
-								"bg" => "Bulgarian",
-								"ca" => "Catalan",
-								"cs" => "Czech",
-								"da" => "Danish",
-								"de" => "German",
-								"el" => "Greek",
-								"en" => "English",
-								"es" => "Spanish",
-								"et" => "Estonian",
-								"fi" => "Finnish",
-								"fr" => "French",
-								"hr" => "Croatian",
-								"hu" => "Hungarian",
-								"id" => "Indonesian",
-								"is" => "Icelandic",
-								"it" => "Italian",
-								"iw" => "Hebrew",
-								"ja" => "Japanese",
-								"ko" => "Korean",
-								"lt" => "Lithuanian",
-								"lv" => "Latvian",
-								"nl" => "Dutch",
-								"no" => "Norwegian",
-								"pl" => "Polish",
-								"pt" => "Portuguese",
-								"ro" => "Romanian",
-								"ru" => "Russian",
-								"sk" => "Slovak",
-								"sl" => "Slovenian",
-								"sr" => "Serbian",
-								"sv" => "Swedish",
-								"tr" => "Turkish",
-								"zh-CN" => "Chinese (Simplified)",
-								"zh-TW" => "Chinese (Traditional)"
-							]
-						]
-					]
-				);
-				break;
-			
-			case "images":
-				return array_merge(
-					$base,
-					[
-						"size" => [ // imgSize
-							"display" => "Size",
-							"option" => [
-								"any" => "Any size",
-								"icon" => "Icon",
-								"small" => "Small",
-								"medium" => "Medium",
-								"large" => "Large",
-								"xlarge" => "X-Large",
-								"xxlarge" => "XX-Large",
-								"huge" => "Huge"
-							]
-						],
-						"format" => [ // fileType
-							"display" => "Format",
-							"option" => [
-								"any" => "Any format",
-								"jpg" => "JPG",
-								"gif" => "GIF",
-								"png" => "PNG",
-								"bmp" => "BMP",
-								"svg" => "SVG",
-								"webp" => "WEBP",
-								"ico" => "ICO",
-								"craw" => "RAW"
-							]
-						],
-						"color" => [
-							"display" => "Color",
-							"option" => [
-								"any" => "Any color",
-								
-								"color" => "Full color", // imgColorType
-								"mono" => "Black & White",
-								"trans" => "Transparent background",
-								
-								"red" => "Red", // imgDominantColor
-								"orange" => "Orange",
-								"yellow" => "Yellow",
-								"green" => "Green",
-								"teal" => "Teal",
-								"blue" => "Blue",
-								"purple" => "Purple",
-								"pink" => "Pink",
-								"white" => "White",
-								"gray" => "Gray",
-								"black" => "Black",
-								"brown" => "Brown"
-							]
-						],
-						"type" => [ // imgType
-							"display" => "Type",
-							"option" => [
-								"any" => "Any type",
-								"clipart" => "Clip Art",
-								"face" => "Faces",
-								"lineart" => "Line Drawing",
-								"stock" => "Stock photos",
-								"photo" => "Photos",
-								"animated" => "Animated",
-							]
-						],
-						"rights" => [ // rights
-							"display" => "Usage rights",
-							"option" => [
-								"any" => "Any license",
-								"cc_publicdomain" => "Public domain",
-								"cc_attribute" => "Attribution required",
-								"cc_sharealike" => "Sharealike",
-								"cc_noncommercial" => "Non-commercial use only",
-								"cc_nonderived" => "Original works"
-							]
-						]
-					]
-				);
-				break;
-		}
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-			
-		$headers = [
-			"Accept: application/json",
-			"Accept-Encoding: gzip"
-		];
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		// follow redirects
-		curl_setopt($curlproc, CURLOPT_FOLLOWLOCATION, true);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		
-		return $data;
-	}
-	
-	
-	public function web($get){
-		
-		// rotate proxy + key on EVERY request
-		$keydata = $this->backend->get_key();
-		$proxy = $this->backend->get_ip($keydata["increment"]);
-		
-		if($get["npt"]){
-			
-			// $p is never used
-			[$params, $p] = $this->backend->get(
-				$get["npt"],
-				"web"
-			);
-			
-			$params = json_decode($params, true);
-			
-			$params["key"] = $keydata["key"];
-			
-		}else{
-			
-			//$json = file_get_contents("scraper/google.json");
-			$params = [
-				"q" => $get["s"],
-				"cx" => config::GOOGLE_CX_ENDPOINT,
-				"num" => 10,
-				"start" => 1,
-				"key" => $keydata["key"]
-			];
-			
-			//
-			// parse filters
-			//
-			if($get["newer"] !== false){
-				
-				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
-			}
-			
-			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
-			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
-			if($get["lang"] != "any"){ $params["lr"] = "lang_" . $get["lang"]; }
-			
-			if($get["nsfw"] == "yes"){
-				
-				$params["safe"] = "off";
-			}else{
-				
-				$params["safe"] = "active";
-			}
-			
-			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
-		}
-		
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://www.googleapis.com/customsearch/v1",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		if(isset($json["error"]["message"])){
-			
-			throw new Exception(
-				"API returned an error: " .
-				$json["error"]["message"] .
-				" (key #" . $keydata["increment"] . ")"
-			);
-		}
-		
-		if(!isset($json["items"])){
-			
-			// google just doesnt return items when theres no results
-			return $out;
-		}
-		
-		foreach($json["items"] as $result){
-			
-			//
-			// probe for thumbnail
-			//
-			$probes = [
-				isset($result["pagemap"]["cse_thumbnail"][0]["src"]) ? $result["pagemap"]["cse_thumbnail"][0]["src"] : null,
-				isset($result["pagemap"]["cse_image"][0]["src"]) ? $result["pagemap"]["cse_image"][0]["src"] : null,
-				isset($result["pagemap"]["metatags"][0]["twitter:image"]) ? $result["pagemap"]["metatags"][0]["twitter:image"] : null,
-				isset($result["pagemap"]["metatags"][0]["og:image"]) ? $result["pagemap"]["metatags"][0]["og:image"] : null
-			];
-			
-			$thumb = [
-				"url" => null,
-				"ratio" => null
-			];
-			
-			foreach($probes as $probe){
-				
-				if($probe !== null){
-				
-					$thumb = [
-						"url" => $probe,
-						"ratio" => "16:9"
-					];
-					break;
-				}
-			}
-			
-			//
-			// probe for page format
-			//
-			$mime = "web";
-			if(isset($result["mime"])){
-				
-				$result["mime"] =
-					explode(
-						"/",
-						$result["mime"],
-						2
-					);
-				
-				if(count($result["mime"]) === 2){
-					
-					$mime = strtoupper($result["mime"][1]);
-				}
-			}
-			
-			$description = $result["snippet"];
-			
-			//
-			// Get date
-			//
-			$description_split =
-				explode(
-					"...", $description, 2
-				);
-			
-			if(count($description_split) === 1){
-				
-				$description = $result["snippet"];
-			}elseif(strlen($description_split[0]) < 17){
-				
-				$date = trim($description_split[0]);
-				$date_probe = strtotime($date);
-				
-				if($date_probe !== false){
-					
-					$description = $description_split[1];
-				}else{
-					
-					//
-					// fallback to getting date from meta tags
-					//
-					if(isset($result["pagemap"]["metatags"][0]["creationdate"])){
-						
-						$date = $result["pagemap"]["metatags"][0]["creationdate"];
-						
-					}elseif(isset($result["pagemap"]["metatags"][0]["moddate"])){
-						
-						$date = $result["pagemap"]["metatags"][0]["moddate"];
-					}else{
-						
-						$date = null;
-					}
-					
-					$description = $result["snippet"];
-				}
-			}
-			
-			if($date !== null){
-				
-				$date =
-					strtotime(
-						trim(
-							str_replace(
-								["D:", "'"],
-								"",
-								$date
-							)
-						)
-					);
-				
-				if($date === false){
-					
-					$date = null;
-				}
-			}
-			
-			$out["web"][] = [
-				"title" =>
-					$this->titledots(
-						$result["title"]
-					),
-				"description" =>
-					$this->titledots(
-						$description
-					),
-				"url" => $result["link"],
-				"date" => $date,
-				"type" => $mime,
-				"thumb" => $thumb,
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		// get npt
-		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
-			
-			unset($params["key"]);
-			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($params),
-					"web",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	
-	public function image($get){
-		
-		// rotate proxy + key on EVERY request
-		$keydata = $this->backend->get_key();
-		$proxy = $this->backend->get_ip($keydata["increment"]);
-		
-		if($get["npt"]){
-			
-			// $p is never used
-			[$params, $p] = $this->backend->get(
-				$get["npt"],
-				"web"
-			);
-			
-			$params = json_decode($params, true);
-			
-			$params["key"] = $keydata["key"];
-			
-		}else{
-			
-			//$json = file_get_contents("scraper/google.json");
-			$params = [
-				"q" => $get["s"],
-				"cx" => config::GOOGLE_CX_ENDPOINT,
-				"num" => 10,
-				"start" => 1,
-				"searchType" => "image",
-				"key" => $keydata["key"]
-			];
-			
-			//
-			// parse filters
-			//
-			if($get["newer"] !== false){
-				
-				$params["dateRestrict"] = "d" . (round((time() - $get["newer"]) / 100000));
-			}
-			
-			if($get["rm_dupes"] == "no"){ $params["filter"] = "0"; }
-			if($get["country"] != "any"){ $params["gl"] = $get["country"]; }
-			
-			if($get["nsfw"] == "yes"){
-				
-				$params["safe"] = "off";
-			}else{
-				
-				$params["safe"] = "active";
-			}
-			
-			if($get["sort"] != "any"){ $params["sort"] = $get["sort"]; }
-			
-			// image filters
-			if($get["size"] != "any"){ $params["imgSize"] = $get["size"]; }
-			if($get["format"] != "any"){ $params["fileType"] = $get["format"]; }
-			
-			switch($get["color"]){
-				
-				case "any":
-					break;
-				
-				case "color":
-				case "mono":
-				case "trans":
-					$params["imgColorType"] = $get["color"];
-					break;
-				
-				default:
-					$params["imgDominantColor"] = $get["color"];
-					break;
-			}
-			
-			if($get["type"] != "any"){ $params["imgType"] = $get["type"]; }
-			if($get["rights"] != "any"){ $params["rights"] = $get["rights"]; }
-		}
-		
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://www.googleapis.com/customsearch/v1",
-					$params
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if(isset($json["error"]["message"])){
-			
-			throw new Exception(
-				"API returned an error: " .
-				$json["error"]["message"] .
-				" (key #" . $keydata["increment"] . ")"
-			);
-		}
-		
-		if(!isset($json["items"])){
-			
-			// google just doesnt return items when theres no results
-			return $out;
-		}
-		
-		foreach($json["items"] as $image){
-			
-			$out["image"][] = [
-				"title" => $this->titledots($image["title"]),
-				"source" => [
-					[
-						"url" => $image["link"],
-						"width" => (int)$image["image"]["width"],
-						"height" => (int)$image["image"]["height"]
-					],
-					[
-						"url" => $image["image"]["thumbnailLink"],
-						"width" => (int)$image["image"]["thumbnailWidth"],
-						"height" => (int)$image["image"]["thumbnailHeight"]
-					]
-				],
-				"url" => $image["image"]["contextLink"]
-			];
-		}
-		
-		// get npt
-		if(isset($json["queries"]["nextPage"][0]["startIndex"])){
-			
-			unset($params["key"]);
-			$params["start"] = (int)$json["queries"]["nextPage"][0]["startIndex"];
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($params),
-					"web",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	
-	private function titledots($title){
-		
-		return trim($title, " .\t\n\r\0\x0B…");
-	}
-}

scraper/google_cse.php

@@ -910,10 +910,11 @@ class google_cse{
 			"image" => []
 		];
 		
-		// A response can contain a final page of image results while also marking
+		// detect next page
-		// the cursor as exact/finished.  Parse those results before deciding
+		if(
-		// whether another page token should be offered.
+			isset($json["cursor"]["isExactTotalResults"]) || // detects last page
-		if(!isset($json["results"]) || !is_array($json["results"])){
+			!isset($json["cursor"]["pages"]) // detects no results on page
+		){
 			
 			return $out;
 		}
@@ -938,16 +939,6 @@ class google_cse{
 			];
 		}
 		
-		// Only decide whether a following page exists after preserving every
-		// result returned on this page.
-		if(
-			isset($json["cursor"]["isExactTotalResults"]) || // detects last page
-			!isset($json["cursor"]["pages"]) // detects no results on page
-		){
-			
-			return $out;
-		}
-		
 		// get next page
 		$out["npt"] =			
 			$this->backend->store(

scraper/greppr.php

@@ -0,0 +1,435 @@
+<?php
+
+class greppr{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("greppr");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	public function getfilters($page){
+		
+		return [];
+	}
+	
+	private function get($proxy, $url, $get = [], $cookie = false){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		
+		if($cookie === false){
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1"]
+			);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+				["User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+				"Accept-Language: en-US,en;q=0.5",
+				"Accept-Encoding: gzip",
+				"Cookie: PHPSESSID=" . $cookie,
+				"DNT: 1",
+				"Connection: keep-alive",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1"]
+			);
+		}
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$this->backend->assign_proxy($curlproc, $proxy);
+		
+		$headers = [];
+		
+		curl_setopt(
+			$curlproc,
+			CURLOPT_HEADERFUNCTION,
+			function($curlproc, $header) use (&$headers){
+				
+				$len = strlen($header);
+				$header = explode(':', $header, 2);
+				
+				if(count($header) < 2){
+					
+					// ignore invalid headers
+					return $len;
+				}
+				
+				$headers[strtolower(trim($header[0]))] = trim($header[1]);
+
+				return $len;
+			}
+		);
+				
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		
+		return [
+			"headers" => $headers,
+			"data" => $data
+		];
+	}
+	
+	public function web($get, $first_attempt = true){
+		
+		if($get["npt"]){
+			
+			[$q, $proxy] = $this->backend->get($get["npt"], "web");
+			
+			$q = json_decode($q, true);
+			
+		}else{
+			
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+		}
+		
+		// get token
+		// token[0] = static token that changes once a day
+		// token[1] = dynamic token that changes on every request
+		// token[1] = PHPSESSID cookie
+		$tokens = apcu_fetch("greppr_token");
+		
+		if(
+			$tokens === false ||
+			$first_attempt === false // force token fetch
+		){
+			
+			// we haven't gotten the token yet, get it
+			try{
+				
+				$response =
+					$this->get(
+						$proxy,
+						"https://greppr.org",
+						[]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search tokens");
+			}
+			
+			$tokens = $this->parse_token($response);
+			
+			if($tokens === false){
+				
+				throw new Exception("Failed to grep search tokens");
+			}
+		}
+		
+		try{
+			
+			if($get["npt"]){
+				
+				$params = [
+					$tokens[0] => $q["q"],
+					"s" => $q["s"],
+					"l" => 30,
+					"n" => $tokens[1]
+				];
+			}else{
+				
+				$params = [
+					$tokens[0] => $search,
+					"n" => $tokens[1]
+				];
+			}
+			
+			$searchresults = $this->get(
+				$proxy,
+				"https://greppr.org/search",
+				$params,
+				$tokens[2]
+			);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch search page");
+		}
+		
+		if(strlen($searchresults["data"]) === 0){
+			
+			// redirected to main page, which means we got old token
+			// generate a new one
+			
+			// ... unless we just tried to do that
+			if($first_attempt === false){
+				
+				throw new Exception("Failed to get a new search token");
+			}
+			
+			return $this->web($get, false);
+		}
+		
+		// refresh the token with new data (this also triggers fuckhtml load)
+		$this->parse_token($searchresults, $tokens[2]);
+		
+		// response object
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		// get results for later
+		$results =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"result",
+				"div"
+			);
+		
+		// check for next page
+		$next_elem =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"pagination",
+				"ul"
+			);
+		
+		if(count($next_elem) !== 0){
+			
+			$this->fuckhtml->load($next_elem[0]);
+			
+			$as =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"page-link",
+					"a"
+				);
+			
+			$break = false;
+			foreach($as as $a){
+				
+				if($break === true){
+					
+					parse_str(
+						$this->fuckhtml
+						->getTextContent(
+							$a["attributes"]["href"]
+						),
+						$values
+					);
+					
+					$values = array_values($values);
+					
+					$out["npt"] =
+						$this->backend->store(
+							json_encode(
+								[
+									"q" => $values[0],
+									"s" => $values[1]
+								]
+							),
+							"web",
+							$proxy
+						);
+					break;
+				}
+				
+				if($a["attributes"]["href"] == "#"){
+					
+					$break = true;
+				}
+			}
+		}
+		
+		// scrape results
+		foreach($results as $result){
+			
+			$this->fuckhtml->load($result);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"a"
+				)[0];
+			
+			$description =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"highlightedDesc",
+					"p"
+				);
+			
+			if(count($description) === 0){
+				
+				$description = null;
+			}else{
+				
+				$description =
+					$this->limitstrlen(
+						$this->fuckhtml
+						->getTextContent(
+							$description[0]
+						)
+					);
+			}
+			
+			$date =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"p"
+				);
+			
+			$date =
+				strtotime(
+					explode(
+						":",
+						$this->fuckhtml
+						->getTextContent(
+							$date[count($date) - 1]["innerHTML"]
+						)
+					)[1]
+				);
+			
+			$out["web"][] = [
+				"title" =>
+					$this->fuckhtml
+					->getTextContent(
+						$a["innerHTML"]
+					),
+				"description" => $description,
+				"url" =>
+					$this->fuckhtml
+					->getTextContent(
+						$a["attributes"]["href"]
+					),
+				"date" => $date,
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		return $out;
+	}
+	
+	private function parse_token($response, $cookie = false){
+		
+		$this->fuckhtml->load($response["data"]);
+
+		$scripts =
+			$this->fuckhtml
+			->getElementsByTagName("script");
+		
+		$found = false;
+		foreach($scripts as $script){
+			
+			preg_match(
+				'/window\.location ?= ?\'\/search\?([^=]+).*&n=([0-9]+)/',
+				$script["innerHTML"],
+				$tokens
+			);
+			
+			if(isset($tokens[1])){
+				
+				$found = true;
+				break;
+			}
+		}
+		
+		if($found === false){
+			
+			return false;
+		}
+		
+		$tokens = [
+			$tokens[1],
+			$tokens[2]
+		];
+		
+		if($cookie !== false){
+			
+			// we already specified a cookie, so use the one we have already
+			$tokens[] = $cookie;
+			apcu_store("greppr_token", $tokens);
+			
+			return $tokens;
+		}
+		
+		if(!isset($response["headers"]["set-cookie"])){
+			
+			// server didn't send a cookie
+			return false;
+		}
+		
+		// get cookie
+		preg_match(
+			'/PHPSESSID=([^;]+)/',
+			$response["headers"]["set-cookie"],
+			$cookie
+		);
+		
+		if(!isset($cookie[1])){
+			
+			// server sent an unexpected cookie
+			return false;
+		}
+		
+		$tokens[] = $cookie[1];
+		apcu_store("greppr_token", $tokens);
+		
+		return $tokens;
+	}
+	
+	private function limitstrlen($text){
+		
+		return explode("\n", wordwrap($text, 300, "\n"))[0];
+	}
+}

scraper/imgur.php

@@ -182,23 +182,6 @@ class imgur{
 			throw new Exception("Failed to fetch HTML");
 		}
 		
-		$json = json_decode($html, true);
-		
-		if($json){
-			
-			// {"data":{"error":"Imgur is temporarily over capacity. Please try again later."},"success":false,"status":403}
-			
-			if(isset($json["data"]["error"])){
-				
-				if(stripos($json["data"]["error"], "capacity")){
-					
-					throw new Exception("Imgur IP blocked this 4get instance or request proxy. Try again");
-				}
-			}
-			
-			throw new Exception("Imgur returned an unknown error (IP ban?)");
-		}
-		
 		$this->fuckhtml->load($html);
 		
 		$posts =
@@ -214,14 +197,7 @@ class imgur{
 			
 			$image =
 				$this->fuckhtml
-				->getElementsByTagName("img");
+				->getElementsByTagName("img")[0];
-			
-			if(count($image) === 0){
-				
-				continue;
-			}
-			
-			$image = $image[0];
 			
 			$image_url = "https:" . substr($this->fuckhtml->getTextContent($image["attributes"]["src"]), 0, -5);
 			

scraper/marginalia.php

@@ -3,10 +3,7 @@
 class marginalia{
 	public function __construct(){
 		
-		include "lib/anubis.php";
+		include "lib/fuckhtml.php";
-		$this->anubis = new anubis();
-		
-		include_once "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
 		
 		include "lib/backend.php";
@@ -105,40 +102,7 @@ class marginalia{
 		);
 	}
 	
-	private function get($proxy, $url, $get = [], $get_cookies = 1){
+	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		switch($get_cookies){
-			
-			case 0:
-				$cookies = "";
-				$cookies_tmp = [];
-				curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
-					
-					$length = strlen($header);
-					
-					$header = explode(":", $header, 2);
-					
-					if(trim(strtolower($header[0])) == "set-cookie"){
-						
-						$cookie_tmp = explode("=", trim($header[1]), 2);
-						
-						$cookies_tmp[trim($cookie_tmp[0])] =
-							explode(";", $cookie_tmp[1], 2)[0];
-					}
-					
-					return $length;
-				});
-				break;
-			
-			case 1:
-				$cookies = "";
-				break;
-			
-			default:
-				$cookies = "Cookie: " . $get_cookies;
-		}
 		
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
@@ -146,7 +110,6 @@ class marginalia{
 			"Accept-Language: en-US,en;q=0.5",
 			"Accept-Encoding: gzip",
 			"DNT: 1",
-			$cookies,
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
 			"Sec-Fetch-Dest: document",
@@ -155,6 +118,8 @@ class marginalia{
 			"Sec-Fetch-User: ?1"
 		];
 		
+		$curlproc = curl_init();
+		
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
@@ -180,23 +145,11 @@ class marginalia{
 			throw new Exception(curl_error($curlproc));
 		}
 		
-		if($get_cookies === 0){
-			
-			$cookie = [];
-			
-			foreach($cookies_tmp as $key => $value){
-				
-				$cookie[] = $key . "=" . $value;
-			}
-			
 		curl_close($curlproc);
-			return implode(";", $cookie);
-		}
-		
 		return $data;
 	}
 	
-	public function web($get, $ss = ""){
+	public function web($get){
 		
 		$search = [$get["s"]];
 		if(strlen($get["s"]) === 0){
@@ -274,7 +227,7 @@ class marginalia{
 				$json =
 					$this->get(
 						$this->backend->get_ip(), // no nextpage
-						"https://api.marginalia-search.com/" . config::MARGINALIA_API_KEY . "/search/" . urlencode($search),
+						"https://api.marginalia.nu/" . config::MARGINALIA_API_KEY . "/search/" . urlencode($search),
 						[
 							"count" => 20
 						]
@@ -289,18 +242,8 @@ class marginalia{
 				throw new Exception("The API key used is rate limited. Please try again in a few minutes.");
 			}
 			
-			if(strpos($json, "<title>Bad Gateway (502)</title>")){
-				
-				throw new Exception("Marginalia returned a 502 error");
-			}
-			
 			$json = json_decode($json, true);
 			
-			if($json === null){
-				
-				throw new Exception("Failed to decode JSON");
-			}
-			
 			foreach($json["results"] as $result){
 				
 				$out["web"][] = [
@@ -324,60 +267,6 @@ class marginalia{
 		// HTML parser
 		$proxy = $this->backend->get_ip();
 		
-		//
-		// Bypass anubis check
-		//
-		/*
-		if(($anubis_key = apcu_fetch("marginalia_cookie")) === false){
-			
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://old-search.marginalia.nu/search",
-						[
-							"query" => $search
-						]
-					);
-					
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to get anubis challenge");
-			}
-			
-			try{
-				
-				$anubis_data = $this->anubis->scrape($html);
-			}catch(Exception $error){
-				
-				throw new Exception($error);
-			}
-			
-			// send anubis response & get cookies
-			// https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge?response=0000018966b086834f738bacba6031028adb5aa875974ead197a8b75778baf3a&nonce=39947&redir=https%3A%2F%2Fold-search.marginalia.nu%2F&elapsedTime=1164
-			
-			try{
-				
-				$anubis_key =
-					$this->get(
-						$proxy,
-						"https://old-search.marginalia.nu/.within.website/x/cmd/anubis/api/pass-challenge",
-						[
-							"response" => $anubis_data["response"],
-							"nonce" => $anubis_data["nonce"],
-							"redir" => "https://old-search.marginalia.nu/",
-							"elapsedTime" => random_int(1000, 2000)
-						],
-						0
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to submit anubis challenge");
-			}
-			
-			apcu_store("marginalia_cookie", $anubis_key);
-		}*/
-		
 		if($get["npt"]){
 			
 			[$params, $proxy] =
@@ -390,9 +279,7 @@ class marginalia{
 				$html =
 					$this->get(
 						$proxy,
-						"https://old-search.marginalia.nu/search?" . $params,
+						"https://search.marginalia.nu/search?" . $params
-						[],
-						//$anubis_key
 					);
 			}catch(Exception $error){
 				
@@ -400,9 +287,6 @@ class marginalia{
 			}
 			
 		}else{
-			
-			if($ss == ""){
-				
 			$params = [
 				"query" => $search
 			];
@@ -419,18 +303,13 @@ class marginalia{
 					}
 				}
 			}
-			}else{
-				
-				$params = [];
-			}
 			
 			try{
 				$html =
 					$this->get(
 						$proxy,
-						"https://old-search.marginalia.nu/search" . $ss,
+						"https://search.marginalia.nu/search",
-						$params,
+						$params
-						//$anubis_key
 					);
 			}catch(Exception $error){
 				
@@ -440,97 +319,6 @@ class marginalia{
 		
 		$this->fuckhtml->load($html);
 		
-		// detect meta redirect
-		$title =
-			$this->fuckhtml
-			->getElementsByTagName(
-				"title"
-			);
-		
-		if(
-			count($title) !== 0 &&
-			$this->fuckhtml
-			->getTextContent(
-				$title[0]
-			) == "Error"
-		){
-			
-			// redirect detected
-			
-			// get timeout
-			$timeout =
-				$this->fuckhtml
-				->getElementById(
-					"countdown",
-					"b"
-				);
-			
-			if(count($timeout) === null){
-				
-				throw new Exception("Failed to find timeout value");
-			}
-			
-			$timeout =
-				$this->fuckhtml
-				->getTextContent(
-					$timeout
-				);
-			
-			preg_match(
-				'/location\.replace\(\'([^\']+)\'\)/',
-				$html,
-				$redirect
-			);
-			
-			if(!isset($redirect[1])){
-				
-				throw new Exception("Failed to grep redirect value");
-			}
-			
-			$one = 1;
-			$redirect =
-				str_replace(
-					"/search",
-					"",
-					$redirect[1],
-					$one
-				);
-			
-			sleep((int)$timeout);
-			return $this->web($get, $redirect);
-		}
-		
-		// detect internal error
-		$infobox =
-			$this->fuckhtml
-			->getElementsByClassName("infobox");
-		
-		if(count($infobox) !== 0){
-			
-			foreach($infobox as $i){
-				
-				$this->fuckhtml->load($i);
-				
-				$h2 =
-					$this->fuckhtml
-					->getElementsByTagName("h2");
-				
-				if(
-					count($h2) !== 0 &&
-					$this->fuckhtml
-					->getTextContent(
-						$h2[0]
-					) == "Internal error"
-				){
-					
-					throw new Exception("Marginalia returned an internal server error");
-				}
-			}
-						
-			// reset
-			$this->fuckhtml->load($html);
-		}
-		
 		$sections =
 			$this->fuckhtml
 			->getElementsByClassName(

scraper/mojeek.php

@@ -340,7 +340,7 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],/*
+					],
 					"region" => [
 						"display" => "Region",
 						"option" => [
@@ -350,7 +350,7 @@ class mojeek{
 							"fr" => "France",
 							"uk" => "United Kingdom"
 						]
-					],*/
+					],
 					"domain" => [
 						"display" => "Results per domain",
 						"option" => [
@@ -376,7 +376,7 @@ class mojeek{
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.9",
+			"Accept-Language: en-US,en;q=0.5",
 			"Accept-Encoding: gzip",
 			"DNT: 1",
 			"Connection: keep-alive",
@@ -447,7 +447,7 @@ class mojeek{
 			$proxy = $this->backend->get_ip();
 			$lang = $get["lang"];
 			$country = $get["country"];
-			//$region = $get["region"];
+			$region = $get["region"];
 			$domain = $get["domain"];
 			$focus = $get["focus"];
 			
@@ -457,8 +457,8 @@ class mojeek{
 				"tn" => 7, // number of news results/page
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
-				//"dlen" => 511, // max length of description
+				"dlen" => 511, // max length of description
-				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
+				"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
 			switch($focus){
@@ -479,11 +479,11 @@ class mojeek{
 				
 				$params["lb"] = $lang;
 			}
-			/*
+			
 			if($region != "any"){
 				
 				$params["reg"] = $region;
-			}*/
+			}
 			
 			if($domain != "1"){
 				
@@ -501,15 +501,13 @@ class mojeek{
 				
 				throw new Exception("Failed to get HTML");
 			}
+			/*
+			$handle = fopen("scraper/mojeek.html", "r");
+			$html = fread($handle, filesize("scraper/mojeek.html"));
+			fclose($handle);*/
+			
 		}
 		
-		if(strlen($html) === 0){
-			
-			throw new Exception("Mojeek returned an empty page (probably a homepage redirect)");
-		}
-		
-		$this->fuckhtml->load($html);
-		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -526,7 +524,7 @@ class mojeek{
 			"related" => []
 		];
 		
-		$this->detect_block();
+		$this->fuckhtml->load($html);
 		
 		$results =
 			$this->fuckhtml
@@ -697,18 +695,17 @@ class mojeek{
 				preg_match(
 					'/\/image\?img=([^&]+)/i',
 					$thumb[0]["attributes"]["src"],
-					$matches
+					$thumb
 				);
 				
-				if(count($matches) === 2){
+				if(count($thumb) === 2){
 					
-					// for some reason, if we dont get the image from mojeek
-					// it sometimes fail to fetch the right image URL
 					$answer["thumb"] =
-						"https://mojeek.com" .
+						urldecode(
 							$this->fuckhtml
 							->getTextContent(
-							$thumb[0]["attributes"]["src"]
+								$thumb[1]
+							)
 						);
 				}
 			}
@@ -1037,8 +1034,6 @@ class mojeek{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_block();
-		
 		$articles =
 			$this->fuckhtml->getElementsByTagName("article");
 		
@@ -1171,38 +1166,6 @@ class mojeek{
 		return $out;
 	}
 	
-	private function detect_block(){
-		
-		$captcha =
-			$this->fuckhtml
-			->getElementsByClassName(
-				"captcha-box",
-				"div"
-			);
-		
-		if(count($captcha) !== 0){
-			
-			throw new Exception("Mojeek returned a captcha");
-		}
-		
-		$title =
-			$this->fuckhtml
-			->getElementsByTagName(
-				"title"
-			);
-		
-		if(
-			count($title) !== 0 &&
-			$this->fuckhtml
-			->getTextContent(
-				$title[0]["innerHTML"]
-			) == "403 - Forbidden"
-		){
-			
-			throw new Exception("Mojeek blocked this instance or request proxy.");
-		}
-	}
-	
 	private function titledots($title){
 		
 		return trim($title, ". \t\n\r\0\x0B");

scraper/pexels.php

@@ -1,285 +0,0 @@
-<?php
-
-class pexels{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("pexels");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"time" => [ // date_from
-				"display" => "Time posted",
-				"option" => [
-					"any" => "Any time",
-					"last_24_hours" => "Last 24 hours",
-					"last_week" => "Last week",
-					"last_month" => "Last month",
-					"last_year" => "Last year"
-				]
-			],
-			"orientation" => [
-				"display" => "Orientation",
-				"option" => [
-					"any" => "Any orientation",
-					"landscape" => "Horizontal",
-					"portrait" => "Vertical",
-					"square" => "Square"
-				]
-			],
-			"color" => [
-				"display" => "Color",
-				"option" => [
-					"any" => "Any color",
-					"795548" => "Brown",
-					"F44336" => "Red",
-					"E91E63" => "Hot pink",
-					"9C27B0" => "Magenta",
-					"673AB7" => "Purple",
-					"3F51B5" => "Indigo",
-					"2196F3" => "Blue",
-					"03A9F4" => "Light blue",
-					"00BCD4" => "Cyan",
-					"009688" => "Forest green",
-					"4CAF50" => "Green",
-					"8BC34A" => "Lime",
-					"CDDC39" => "Pear",
-					"FFEB3B" => "Yellow",
-					"FFC107" => "Gold",
-					"FF9800" => "Orange",
-					"FF5722" => "Tomato",
-					"9E9E9E" => "Gray",
-					"607D8B" => "Teal",
-					"000000" => "Black",
-					"FFFFFF" => "White"
-				]
-			],
-			"people_count" => [
-				"display" => "Head count",
-				"option" => [
-					"any" => "Any number",
-					"0" => "0",
-					"1" => "1",
-					"2" => "2",
-					"3_plus" => "3+",
-				]
-			],
-			"people_age" => [
-				"display" => "People's age",
-				"option" => [
-					"any" => "Any age",
-					"baby" => "Baby",
-					"child" => "Child",
-					"teenager" => "Teenager",
-					"adult" => "Adult",
-					"senior_adult" => "Senior adult"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		$search = $get["query"];
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.9",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: https://www.pexels.com/search/" . rawurlencode($search) . "/",
-			"Content-Type: application/json",
-			"secret-key: H2jk9uKnhRmL6WPwh89zBezWvr", // hardcoded but like, people on github have been using this shit since 23'
-			"X-Client-Type: react",
-			"X-Next-Forwarded-CF-Connecting-IP: ",
-			"X-Next-Forwarded-CF-IPCountry: ",
-			"X-Next-Forwarded-CF-IPRegionCode: ",
-			"X-Active-Experiment: ",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Alt-Used: www.pexels.com",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"Priority: u=4",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$filter, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$filter = json_decode($filter, true);
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			// ?query=blue%20footed%20booby&page=1&per_page=24&seo_tags=true
-			$filter = [
-				"query" => $search,
-				"page" => 1,
-				"per_page" => 24,
-				"seo_tags" => "true"
-			];
-			
-			// add filters
-			if($get["time"] != "any"){
-				
-				$filter["date_from"] = $get["time"];
-			}
-			
-			if($get["orientation"] != "any"){
-				
-				$filter["orientation"] = $get["orientation"];
-			}
-			
-			if($get["color"] != "any"){
-				
-				$filter["color"] = $get["color"];
-			}
-			
-			if($get["people_count"] != "any"){
-				
-				$filter["people_count"] = $get["people_count"];
-			}
-			
-			if($get["people_age"] != "any"){
-				
-				$filter["people_age"] = $get["people_age"];
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		try{
-			$html =
-				$this->get(
-					$proxy,
-					"https://www.pexels.com/en-us/api/v3/search/photos",
-					$filter
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($html, true);
-		
-		if(!isset($json["data"])){
-			
-			throw new Exception("Pexels did not return a data object");
-		}
-		
-		foreach($json["data"] as $image){
-			
-			$thumb_size =
-				$this->imgratio(
-					(int)$image["attributes"]["width"],
-					(int)$image["attributes"]["height"],
-					350
-				);
-			
-			$out["image"][] = [
-				"title" => $image["attributes"]["title"] . ": " . $image["attributes"]["description"],
-				"source" => [
-					[
-						"url" => $image["attributes"]["image"]["download_link"],
-						"width" => (int)$image["attributes"]["width"],
-						"height" => (int)$image["attributes"]["height"]
-					],
-					[
-						"url" =>
-							preg_replace(
-								'/(?:w|h)=[0-9]+$/',
-								"w=350",
-								$image["attributes"]["image"]["small"]
-							),
-						"width" => $thumb_size[0],
-						"height" => $thumb_size[1]
-					]
-				],
-				"url" =>
-					"https://pexels.com/photo/" .
-					$image["attributes"]["slug"] . "-" .
-					$image["attributes"]["id"] . "/"
-			];
-		}
-		
-		// get next page
-		if((int)$json["pagination"]["current_page"] < (int)$json["pagination"]["total_pages"]){
-			
-			$filter["page"]++;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($filter),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function imgratio($width, $height, $max_width){
-		
-		$ratio = $max_width / $width;
-		$new_height = floor($height * $ratio);
-		
-		return [
-			$max_width,
-			$new_height
-		];
-	}
-}

scraper/pinterest.php

@@ -13,104 +13,31 @@ class pinterest{
 		return [];
 	}
 	
-	private function get($proxy, $url, $get = [], &$cookies, $header_data_post = null){
+	private function get($proxy, $url, $get = []){
 		
 		$curlproc = curl_init();
 		
-		if($header_data_post === null){
-			
-			// handling GET
-						
-			// extract cookies
-			$cookies_tmp = [];
-			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
-				
-				$length = strlen($header);
-				
-				$header = explode(":", $header, 2);
-				
-				if(trim(strtolower($header[0])) == "set-cookie"){
-					
-					$cookie_tmp = explode("=", trim($header[1]), 2);
-					
-					$cookies_tmp[trim($cookie_tmp[0])] =
-						explode(";", $cookie_tmp[1], 2)[0];
-				}
-				
-				return $length;
-			});
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: application/json, text/javascript, */*, q=0.01",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"Referer: https://ca.pinterest.com/",
-				"X-Requested-With: XMLHttpRequest",
-				"X-APP-VERSION: 78f8764",
-				"X-Pinterest-AppState: active",
-				"X-Pinterest-Source-Url: /",
-				"X-Pinterest-PWS-Handler: www/index.js",
-				"screen-dpr: 1",
-				"is-preload-enabled: 1",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-origin",
-				"Connection: keep-alive",
-				"Alt-Used: ca.pinterest.com",
-				"Priority: u=0",
-				"TE: trailers"]
-			);
-			
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
 		}
-		}else{
-			
-			// handling POST (pagination)
-			$get = http_build_query($get);
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: application/json, text/javascript, */*, q=0.01",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
-				"Content-Type: application/x-www-form-urlencoded",
-				"Content-Length: " . strlen($get),
-				"Referer: https://ca.pinterest.com/",
-				"X-Requested-With: XMLHttpRequest",
-				"X-APP-VERSION: 78f8764",
-				"X-CSRFToken: " . $cookies["csrf"],
-				"X-Pinterest-AppState: active",
-				"X-Pinterest-Source-Url: /search/pins/?rs=ac&len=2&q=" . urlencode($header_data_post) . "&eq=" . urlencode($header_data_post),
-				"X-Pinterest-PWS-Handler: www/search/[scope].js",
-				"screen-dpr: 1",
-				"is-preload-enabled: 1",
-				"Origin: https://ca.pinterest.com",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-origin",
-				"Connection: keep-alive",
-				"Alt-Used: ca.pinterest.com",
-				"Cookie: " . $cookies["cookie"],
-				"TE: trailers"]
-			);
-			
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $get);
-		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-		// http2 bypass
+			["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0",
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip",
+			"DNT: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1"]
+		);
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -127,26 +54,6 @@ class pinterest{
 			throw new Exception(curl_error($curlproc));
 		}
 		
-		if($header_data_post === null){
-			
-			if(!isset($cookies_tmp["csrftoken"])){
-				
-				throw new Exception("Failed to grep CSRF token");
-			}
-			
-			$cookies = "";
-			
-			foreach($cookies_tmp as $cookie_name => $cookie_value){
-				
-				$cookies .= $cookie_name . "=" . $cookie_value . "; ";
-			}
-			
-			$cookies = [
-				"csrf" => $cookies_tmp["csrftoken"],
-				"cookie" => rtrim($cookies, " ;")
-			];
-		}
-		
 		curl_close($curlproc);
 		return $data;
 	}
@@ -155,68 +62,17 @@ class pinterest{
 		
 		if($get["npt"]){
 			
-			[$data, $proxy] =
+			// @TODO
-				$this->backend->get(
+			// post data for next page
-					$get["npt"], "images"
+			$data = [
-				);
-			
-			$data = json_decode($data, true);
-			
-			$search = $data["q"];
-			$cookies = $data["cookies"];
-			
-			try{
-				$json =
-					$this->get(
-						$proxy,
-						"https://ca.pinterest.com/resource/BaseSearchResource/get/",
-						[
 				"source_url" => "/search/pins/?q=" . urlencode($search) . "&rs=typed",
-							"data" => json_encode(
+				"data" =>
+					json_encode(
 						[
-									"options" => [
+							// {"options":{"applied_filters":null,"appliedProductFilters":"---","article":null,"auto_correction_disabled":false,"corpus":null,"customized_rerank_type":null,"domains":null,"filters":null,"journey_depth":null,"page_size":null,"price_max":null,"price_min":null,"query_pin_sigs":null,"query":"higurashi","redux_normalize_feed":true,"rs":"typed","scope":"pins","selected_one_bar_modules":null,"source_id":null,"source_module_id":null,"top_pin_id":null,"bookmarks":["Y2JVSG81V2sxcmNHRlpWM1J5VFVad1ZsWlVRbXhpVmtreVZsZHpOV0pIU2tkV2FscFhVbXhhVkZreU1WSmtNREZWVjIxR1RrMXNTbEJXYlhSaFVtMVdjMVZ1U2xaaWEzQnpXVlJPVTJWV1pISlhhM1JYVm10V05sVldVbE5XVjBwMVVXMUdWVll6VFhoVWJYaFhWMVp3Ums1V1RsTmlSbGt5Vm10YWFtVkdWbkpOU0dSUFZsZG9XRmxzWkc5VlZscHlWbGhrYkdKR1NubFdWelZQWVVaYWRHVkVRbFppUmtwVVZrUktWMlJIVWtWV2JHaHBVakZLU0Zkc1pEUmtNVnBZVW10b2FsSXdXbkJXYlRWRFpHeGFSMWRzVG1oaGVrWllXV3RvVTFVeFpFaFZiRUpoVm5wRk1GbHFSbXRYVjA1R1YyczFWMVpHV2pSWFZtaDNVakZrY2sxWVRsaGlhM0JXV1ZSR1MyRkdiRlZTYm1SVVVteHdXbGxWVlRGVk1VbDVWRmhrVjAxdVVuWlVhMXBTWlVaT2MxcEhSbE5TTWswMVdtdGFWMU5YU2paVmJYaFRUVmhDUjFZeU5YZFVNVkY0VjJ0b1ZXRnJOVlpVVmxwTFVURndXR042VmxOV2ExcGFXVlZWTlZVeFNYZE5WRTVYVWtWYVZGWkhNVTlXTVU1WllVWk9hR1ZyV2s1WFZ6QXhZakpPVjFWWWFHRlNWbkJRVm14U1IwMUdXWGxOVkVKVlRWWnNORll5TURWV1YwVjVWV3hDV21FeGNETmFSVnByVjFkS1IyTkhhR2xYUjJkM1ZtdGFhMlF4VVhsVGJGcE9Wa1p3YjFwWGVFdFZWbFp4VW14YWJGWnRVbHBaTUdoTFZHMUtTR1ZJYUZkV2VrWjJWMVphU21ReVJYcGpSbFpwVW10d1RGZHJVa0pPVms1SFZHNVNUbFl3V2xoVmJYUldaVVpaZUZremFGUk5hM0JYVkZaYVYyRkZNSGxWYkVKYVlrWlZlRnBGV210WFIwNUpVMnMxVTFaR1dscFdWekI0VFVaV1IxTllaR3BUUlhCb1dWUkdWbVZHVm5SbFJuQnNZbFpKTWxSVlVYaFBSVGxGV1hwR1QyVnJSVEZVVlZKT1RrVXhSVkpVUWs5bGJFVXhWRmhzZDFOR1ZsWmtNMFp0VWpGYWIxZFhjRXBsUlRGSVZWaHdUbFl4YTNoVVZWSnFUVVUxV0ZadGFFOVNSVnB6Vkd0a1drMUdiRFpUVkVaT1pXMWplRmRzVWxkaFJuQllWVlJTVDJWdFRqWlVNVkpTWlZad2NWcEhkRTlsYTFwMFZGVlNhMkpWTVZWVFZFcE9Wa1pzTmxkWE1WSk9WVEYwVlcweFVGWXdXVFJXUjNSWFYwZGFRbEJVTVRoUFJHTXhUbnBCTlUxRVRUUk5SRVV3VG5wUk5VMTVjRWhWVlhkeFprUlZlRTlFVVRKWlZHc3lUMWRSTWsxVVVUSk9iVnBvV1RKWmVrNTZXWGhPTWs1cFQwUkZNVTlFVm1sTlZGcHBUV3BTYTFsWFRtcE9SR015VG1wVk5GbHFaR2haVjFacldWUmFiVmxxWkdoYVZGWnFUa1JXT0ZSclZsaG1RVDA5fFVIbzVhRkpYZUc1WFYyUlpWVEpHYkdGNk1XWk5ha1ptVFZSR09FOUVZekZPZWtFMVRVUk5ORTFFUlRCT2VsRTFUWGx3U0ZWVmQzRm1SMWw1VFZSUk1WbDZUVEJhUjFGNVQxZFNhVnB0VlRGT1JFVXdXVlJuZVU1cVRUUk5hbU40VDBSSk1VNXFWVEZOYlZwcVdsUnJlRTFFVVhwWmVsVjNXbXBvYkU1dFJYbE9ha0Y2VDFSSk5VMTZWVEJaYWtJNFZHdFdXR1pCUFQwPXxOb25lfDg3NTcwOTAzODAxNDc0OTMqR1FMKnwzMjM3YjM3ZGNhMGU3YjYyYzYzYzAyZGJkNGU1MjdlNzMyMTExMTNlMmUyMzEyOWM2MDAzYmU1ZTlmZjkwYjAwfE5FV3w="]},"context":{}}
-										"applied_unified_filters" => null,
-										"appliedProductFilters" => "---",
-										"article" => null,
-										"auto_correction_disabled" => false,
-										"corpus" => null,
-										"customized_rerank_type" => null,
-										"domains" => null,
-										"dynamicPageSizeExpGroup" => null,
-										"filters" => null,
-										"journey_depth" => null,
-										"page_size" => null,
-										"price_max" => null,
-										"price_min" => null,
-										"query_pin_sigs" => null,
-										"query" => $data["q"],
-										"redux_normalize_feed" => true,
-										"request_params" => null,
-										"rs" => "typed",
-										"scope" => "pins",
-										"selected_one_bar_modules" => null,
-										"source_id" => null,
-										"source_module_id" => null,
-										"source_url" => "/search/pins/?q=" . urlencode($search) . "&rs=typed",
-										"top_pin_id" => null,
-										"top_pin_ids" => null,
-										"bookmarks" => [
-											$data["bookmark"]
 						]
-									],
-									"context" => []
-								],
-								JSON_UNESCAPED_SLASHES
-							)
-						],
-						$cookies,
-						$search
 					);
-				
+			];
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JSON");
-			}
 			
 		}else{
 			
@@ -226,44 +82,26 @@ class pinterest{
 				throw new Exception("Search term is empty!");
 			}
 			
-			// https://ca.pinterest.com/resource/BaseSearchResource/get/?source_url=%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac&data=%7B%22options%22%3A%7B%22applied_unified_filters%22%3Anull%2C%22appliedProductFilters%22%3A%22---%22%2C%22article%22%3Anull%2C%22auto_correction_disabled%22%3Afalse%2C%22corpus%22%3Anull%2C%22customized_rerank_type%22%3Anull%2C%22domains%22%3Anull%2C%22dynamicPageSizeExpGroup%22%3Anull%2C%22filters%22%3Anull%2C%22journey_depth%22%3Anull%2C%22page_size%22%3Anull%2C%22price_max%22%3Anull%2C%22price_min%22%3Anull%2C%22query_pin_sigs%22%3Anull%2C%22query%22%3A%22higurashi%20when%20they%20cry%22%2C%22redux_normalize_feed%22%3Atrue%2C%22request_params%22%3Anull%2C%22rs%22%3A%22ac%22%2C%22scope%22%3A%22pins%22%2C%22selected_one_bar_modules%22%3Anull%2C%22source_id%22%3Anull%2C%22source_module_id%22%3Anull%2C%22source_url%22%3A%22%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac%22%2C%22top_pin_id%22%3Anull%2C%22top_pin_ids%22%3Anull%7D%2C%22context%22%3A%7B%7D%7D&_=1736116313987
-			// source_url=%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac
-			// &data=%7B%22options%22%3A%7B%22applied_unified_filters%22%3Anull%2C%22appliedProductFilters%22%3A%22---%22%2C%22article%22%3Anull%2C%22auto_correction_disabled%22%3Afalse%2C%22corpus%22%3Anull%2C%22customized_rerank_type%22%3Anull%2C%22domains%22%3Anull%2C%22dynamicPageSizeExpGroup%22%3Anull%2C%22filters%22%3Anull%2C%22journey_depth%22%3Anull%2C%22page_size%22%3Anull%2C%22price_max%22%3Anull%2C%22price_min%22%3Anull%2C%22query_pin_sigs%22%3Anull%2C%22query%22%3A%22higurashi%20when%20they%20cry%22%2C%22redux_normalize_feed%22%3Atrue%2C%22request_params%22%3Anull%2C%22rs%22%3A%22ac%22%2C%22scope%22%3A%22pins%22%2C%22selected_one_bar_modules%22%3Anull%2C%22source_id%22%3Anull%2C%22source_module_id%22%3Anull%2C%22source_url%22%3A%22%2Fsearch%2Fpins%2F%3Feq%3Dhigurashi%26etslf%3D5966%26len%3D2%26q%3Dhigurashi%2520when%2520they%2520cry%26rs%3Dac%22%2C%22top_pin_id%22%3Anull%2C%22top_pin_ids%22%3Anull%7D%2C%22context%22%3A%7B%7D%7D
-			// &_=1736116313987
-			
-			$source_url = "/search/pins/?q=" . urlencode($search) . "&rs=" . urlencode($search);
-			
 			$filter = [
-				"source_url" => $source_url,
+				"source_url" => "/search/pins/?q=" . urlencode($search),
 				"rs" => "typed",
 				"data" =>
 					json_encode(
 						[
 							"options" => [
-								"applied_unified_filters" => null,
-								"appliedProductFilters" => "---",
 								"article" => null,
+								"applied_filters" => null,
+								"appliedProductFilters" => "---",
+								"auto_correction_disabled" => false,
 								"corpus" => null,
 								"customized_rerank_type" => null,
-								"domains" => null,
-								"dynamicPageSizeExpGroup" => null,
 								"filters" => null,
-								"journey_depth" => null,
-								"page_size" => null,
-								"price_max" => null,
-								"price_min" => null,
-								"query_pin_sigs" => null,
 								"query" => $search,
+								"query_pin_sigs" => null,
 								"redux_normalize_feed" => true,
-								"request_params" => null,
+								"rs" => "typed",
-								"rs" => "ac",
 								"scope" => "pins", // pins, boards, videos, 
-								"selected_one_bar_modules" => null,
+								"source_id" => null
-								"source_id" => null,
-								"source_module_id" => null,
-								"source_url" => $source_url,
-								"top_pin_id" => null,
-								"top_pin_ids" => null
 							],
 							"context" => []
 						]
@@ -272,99 +110,35 @@ class pinterest{
 			];
 			
 			$proxy = $this->backend->get_ip();
-			$cookies = [];
+		}
 		
 		try{
 			$json =
+				json_decode(
 					$this->get(
 						$proxy,
-						"https://ca.pinterest.com/resource/BaseSearchResource/get/",
+						"https://www.pinterest.ca/resource/BaseSearchResource/get/",
-						$filter,
+						$filter
-						$cookies,
+					),
-						null
+					true
 				);
 			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
 		}
-		}
-		
-		$json = json_decode($json, true);
 		
 		if($json === null){
 			
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		if(
-			isset($json["client_context"]["is_bad_bot"]) &&
-			(int)$json["client_context"]["is_bad_bot"] === 1
-		){
-			
-			throw new Exception("Pinterest blocked this instance or request proxy.");
-		}
-		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
 			"image" => []
 		];
 		
-		if(
-			!isset(
-				$json["resource_response"]
-				["status"]
-			)
-		){
-			
-			throw new Exception("Unknown API failure");
-		}
-		
-		if($json["resource_response"]["status"] != "success"){
-			
-			$status = "Got non-OK response: " . $json["resource_response"]["status"];
-			
-			if(
-				isset(
-					$json["resource_response"]["message"]
-				)
-			){
-				
-				$status .= " - " . $json["resource_response"]["message"];
-			}
-			
-			throw new Exception($status);
-		}
-		
-		if(
-			isset(
-				$json["resource_response"]["sensitivity"]
-				["notices"][0]["description"]["text"]
-			)
-		){
-			
-			throw new Exception(
-				"Pinterest returned a notice: " .
-				$json["resource_response"]["sensitivity"]["notices"][0]["description"]["text"]
-			);
-		}
-		
-		// get NPT
-		if(isset($json["resource_response"]["bookmark"])){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode([
-						"q" => $search,
-						"bookmark" => $json["resource_response"]["bookmark"],
-						"cookies" => $cookies
-					]),
-					"images",
-					$proxy
-				);
-		}
-		
 		foreach(
 			$json
 			["resource_response"]
@@ -376,7 +150,6 @@ class pinterest{
 			switch($item["type"]){
 				
 				case "pin":
-				case "board":
 					
 					/*
 						Handle image object
@@ -433,15 +206,42 @@ class pinterest{
 								"height" => (int)$thumb["height"]
 							]
 						],
-						"url" =>
+						"url" => "https://www.pinterest.com/pin/" . $item["id"]
-							!isset($item["link"]) ?
-							"https://ca.pinterest.com/pin/" . $item["id"] :
-							$item["link"]
 					];
 					break;
+				
+				case "board":
+					if(isset($item["cover_pin"]["image_url"])){
+						
+						$image = [
+							"url" => $item["cover_pin"]["image_url"],
+							"width" => (int)$item["cover_pin"]["size"][0],
+							"height" => (int)$item["cover_pin"]["size"][1]
+						];
+					}elseif(isset($item["image_cover_url_hd"])){
+						/*
+						$image = [
+							"url" => 
+							"width" => null,
+							"height" => null
+						];*/
+					}
+					break;
 			}
 		}
 		
 		return $out;
 	}
+	
+	private function getfullresimage($image, $has_og){
+		
+		$has_og = $has_og ? "1200x" : "originals";
+		
+		return
+			preg_replace(
+				'/https:\/\/i\.pinimg\.com\/[^\/]+\//',
+				"https://i.pinimg.com/" . $has_og . "/",
+				$image
+			);
+	}
 }

scraper/pixabay.php

@@ -1,396 +0,0 @@
-<?php
-
-class pixabay{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("pixabay");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes",
-					"no" => "No"
-				]
-			],
-			"time" => [
-				"display" => "Time posted",
-				"option" => [
-					"any" => "Any time",
-					"1d" => "< 24 hours",
-					"3d" => "< 72 hours",
-					"1w" => "< 7 days",
-					"6m" => "< 6 months",
-					"1y" => "< 12 months"
-				]
-			],
-			"category" => [
-				"display" => "Category",
-				"option" => [
-					"images" => "All images",
-					"photos" => "Photos",
-					"illustrations" => "Illustrations",
-					"vectors" => "Vectors",
-					"gifs" => "GIFs"
-				]
-			],
-			"content_type" => [
-				"display" => "Authenticity",
-				"option" => [
-					"authentic" => "Authentic only",
-					"show_all" => "Authentic + AI slop",
-					"ai" => "AI slop only"
-				]
-			],
-			"order" => [
-				"display" => "Sort by",
-				"option" => [
-					"relevance" => "Most relevant",
-					"latest" => "Latest",
-					"ec" => "Editor's choice",
-					"trending" => "Trending"
-				]
-			],
-			"orientation" => [
-				"display" => "Orientation",
-				"option" => [
-					"any" => "Any orientation",
-					"vertical" => "Vertical",
-					"horizontal" => "Horizontal"
-				]
-			],
-			"color" => [
-				"display" => "Color",
-				"option" => [
-					"any" => "Any color",
-					"transparent" => "Transparent",
-					"grayscale" => "Grayscale",
-					"red" => "Red",
-					"orange" => "Orange",
-					"yellow" => "Yellow",
-					"green" => "Green",
-					"turquoise" => "Turquoise",
-					"blue" => "Blue",
-					"brown" => "Brown",
-					"black" => "Black",
-					"gray" => "Gray",
-					"white" => "White",
-					"pink" => "Pink",
-					"lilac" => "Lilac"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $nsfw = true){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT . " Pixabay",
-			"Accept: application/json",
-			"Accept-Language: en-US,en;q=0.9",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: {$url}",
-			"Cookie: " . ($nsfw ? "g_rated=off" : "g_rated=1"),
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: no-cors",
-			"Sec-Fetch-Site: same-origin",
-			"x-fetch-bootstrap: 1",
-			"Alt-Used: pixabay.com",
-			"Priority: u=0",
-			"Pragma: no-cache",
-			"Cache-Control: no-cache"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$npt, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$npt = json_decode($npt, true);
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$npt = [
-				"s" => $search,
-				"pagi" => 1,
-				"category" => $get["category"]
-			];
-			
-			// filters
-			if($get["content_type"] != "show_all"){
-				
-				$npt["content_type"] = $get["content_type"];
-			}
-			
-			if($get["order"] != "relevance"){
-				
-				$npt["order"] = $get["order"];
-			}
-			
-			$npt["nsfw"] = $get["nsfw"] == "yes" ? true : false;
-			
-			if($get["orientation"] != "any"){
-				
-				$npt["orientation"] = $get["orientation"];
-			}
-			
-			if($get["color"] != "any"){
-				
-				$npt["colors"] = $get["color"];
-			}
-			
-			if($get["time"] != "any"){
-				
-				$npt["date"] = $get["time"];
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		// https://pixabay.com/images/search/japan/?pagi=1
-		
-		$npt_pass = $npt;
-		unset($npt_pass["s"]);
-		unset($npt_pass["category"]);
-		unset($npt_pass["nsfw"]);
-		
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://pixabay.com/{$npt["category"]}/search/" . rawurlencode($npt["s"]) . "/",
-					$npt_pass,
-					$npt["nsfw"]
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(!isset($json["page"]["results"])){
-			
-			throw new Exception("Pixabay API did not return a results object");
-		}
-		
-		//print_r($json);
-		
-		foreach($json["page"]["results"] as $image){
-			
-			if(
-				!(
-					$image["mediaType"] == "photo" ||
-					$image["mediaType"] == "animation"
-				)
-			){
-				
-				continue;
-			}
-			
-			// handle images that supply canvaRetouchUrl
-			if(
-				isset($image["canvaRetouchUrl"]) &&
-				$image["canvaRetouchUrl"] !== null
-			){
-				
-				parse_str(
-					parse_url(
-						$image["canvaRetouchUrl"],
-						PHP_URL_QUERY
-					),
-					$base
-				);
-				
-				if(!isset($base["image-url"])){
-					
-					// should not happen
-					continue;
-				}
-				
-				// get extension
-				$base = $base["image-url"];
-				
-				$parsed_base = parse_url($base);
-				
-				$ext =
-					explode(
-						".",
-						$parsed_base["path"]
-					);
-				
-				$ext = $ext[count($ext) - 1];
-				
-				// restructure url
-				$base =
-					explode(
-						"_",
-						$base
-					);
-				
-				unset($base[count($base) - 1]);
-				$base = implode("_", $base);
-				
-				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
-				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
-				
-				$source = [
-					[
-						"url" => "{$base}_1920.{$ext}",
-						"width" => $ratio_2x[0],
-						"height" => $ratio_2x[1]
-					],
-					[
-						"url" => "{$base}_180.{$ext}",
-						"width" => $ratio_1x[0],
-						"height" => $ratio_1x[1]
-					]
-				];
-			}else{
-				
-				// get intended sizes if canva is not set
-				// get 1x
-				$source_1x = null;
-				if(isset($image["sources"]["image_1x"])){ $source_1x = $image["sources"]["image_1x"]; }
-				elseif(isset($image["sources"]["1x"])){ $source_1x = $image["sources"]["1x"]; }
-				
-				if($source_1x === null){ continue; } // should not happen
-				
-				preg_match(
-					'/_([0-9]+)\./',
-					$image["sources"]["1x"],
-					$size_1x
-				);
-				
-				$size_1x = (int)$size_1x[1];
-				
-				// get 2x
-				$source_2x = null;
-				if(isset($image["sources"]["gif_2x"])){ $source_2x = $image["sources"]["gif_2x"]; }
-				elseif(isset($image["sources"]["2x"])){ $source_2x = $image["sources"]["2x"]; }
-				
-				if($source_2x === null){ continue; } // should not happen
-				
-				preg_match(
-					'/_([0-9]+)\./',
-					$source_2x,
-					$size_2x
-				);
-				
-				$size_2x = (int)$size_2x[1];
-				
-				// compute ratios
-				$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_1x);
-				$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], $size_2x);
-				
-				// handle images that only give normal thumbnails
-				$source = [
-					[
-						"url" => $source_2x,
-						"width" => $ratio_2x[0],
-						"height" => $ratio_2x[1]
-					],
-					[
-						"url" => $source_1x,
-						"width" => $ratio_1x[0],
-						"height" => $ratio_1x[1]
-					]
-				];
-			}
-			
-			$ratio_1x = $this->imgratio((int)$image["width"], (int)$image["height"], 180);
-			$ratio_2x = $this->imgratio((int)$image["width"], (int)$image["height"], 1920);
-			
-			$out["image"][] = [
-				"title" => $image["name"],
-				"source" => $source,
-				"url" => "https://pixabay.com" . $image["href"]
-			];
-		}
-		
-		// add next page
-		if((int)$json["page"]["page"] < (int)$json["page"]["pages"]){
-			
-			$npt["pagi"]++;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($npt),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function imgratio($width, $height, $max_width){
-		
-		$ratio = $max_width / $width;
-		$new_height = floor($height * $ratio);
-		
-		return [
-			$max_width,
-			$new_height
-		];
-	}
-}

scraper/purili.php

@@ -1,350 +0,0 @@
-<?php
-
-class purili{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("purili");
-	}
-	
-	public function getfilters($page){
-		
-		switch($page){
-			case "web":
-				return [
-					"nsfw" => [
-						"display" => "NSFW",
-						"option" => [
-							"yes" => "Yes",
-							"maybe" => "Maybe",
-							"no" => "No"
-						]
-					],
-					"language" => [
-						"display" => "Language",
-						"option" => [
-							"any" => "Any language",
-							"en" => "English only",
-							"nl" => "Dutch preferred",
-							"de" => "German preferred",
-							"fr" => "French preferred",
-							"es" => "Spanish preferred",
-							"it" => "Italian preferred",
-							"pt" => "Portuguese preferred"
-						]
-					]
-				];
-				break;
-			
-			case "videos":
-				return [];
-				break;
-		}
-	}
-	
-	private function get($proxy, $url, $get = [], $nsfw = null, $language = null){
-		
-		$curlproc = curl_init();
-		
-		switch($nsfw){
-			
-			case "yes":
-			case null:
-				$get["fv"] = "0";
-				$nsfw = "off";
-				break;
-				
-			case "maybe":
-				$get["fv"] = "1";
-				$nsfw = "moderate";
-				break;
-				
-			case "no":
-				$get["fv"] = "2";
-				$nsfw = "strict";
-				break;
-		}
-		
-		switch($language){
-			
-			case "any":
-			case null:
-				$language_cookie = "";
-				break;
-			
-			default:
-				$language_cookie = "; purili-language={$language}";
-				break;
-		}
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.9",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Cookie: purili-safesearch={$nsfw}{$language_cookie}",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"Priority: u=4"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function web($get){
-		
-		if($get["npt"]){
-			
-			[$q, $proxy] = $this->backend->get($get["npt"], "web");
-			$q = json_decode($q, true);
-			
-			$nsfw = $q["nsfw"];
-			unset($q["nsfw"]);
-			
-			$language = $q["language"];
-			unset($q["language"]);
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$q = [
-				"q" => $search,
-				"page" => 1,
-				// added at $this->get()
-				// "fv" => 0
-			];
-			
-			$nsfw = $get["nsfw"];
-			$language = $get["language"];
-		}
-		
-		// https://puri.li/api/search?q=4get&page=1&fv=0
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://puri.li/api/search",
-				$q,
-				$nsfw,
-				$language
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to hit API");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(!isset($json["results"])){
-			
-			throw new Exception("Failed to access results array");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
-			"npt" => null,
-			"answer" => [],
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
-		];
-		
-		foreach($json["results"] as $result){
-			
-			$out["web"][] = [
-				"title" => $result["title"],
-				"description" => $result["description"],
-				"url" => $result["url"],
-				"date" => null,
-				"type" => "web",
-				"thumb" => [
-					"url" => null,
-					"ratio" => null
-				],
-				"sublink" => [],
-				"table" => []
-			];
-		}
-		
-		// get next page
-		if($json["hasNext"]){
-			
-			$q["page"]++;
-			$q["nsfw"] = $nsfw;
-			$q["language"] = $language;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($q),
-					"web",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	public function video($get){
-		
-		
-		if($get["npt"]){
-			
-			[$q, $proxy] = $this->backend->get($get["npt"], "videos");
-			$q = json_decode($q, true);
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$q = [
-				"q" => $search,
-				"offset" => 0,
-				"limit" => 12
-				// added at $this->get()
-				// "fv" => 0
-			];
-		}
-		
-		// https://puri.li/api/videos/search?q=higurashi&limit=12&fv=1
-		// hidden &offset parameter only accessible through the API
-		try{
-			$json = $this->get(
-				$proxy,
-				"https://puri.li/api/videos/search",
-				$q,
-				null,
-				null
-			);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to hit API");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		if(!isset($json["results"])){
-			
-			throw new Exception("Failed to access results array");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"video" => [],
-			"author" => [],
-			"livestream" => [],
-			"playlist" => [],
-			"reel" => []
-		];
-		
-		foreach($json["results"] as $result){
-			
-			if(isset($result["thumbnail"])){
-				
-				$thumb = [
-					"ratio" => "16:9",
-					"url" => $result["thumbnail"]
-				];
-			}else{
-				
-				$thumb = [
-					"ratio" => null,
-					"url" => null
-				];
-			}
-			
-			$out["video"][] = [
-				"title" => $result["title"],
-				"description" =>
-					str_replace(
-						"\n", " ",
-						mb_substr(
-							$result["description"], 0, 255, "utf-8"
-						)
-					),
-				"author" => [
-					"name" => null,
-					"url" => isset($result["channel_id"]) ? "https://www.youtube.com/channel/" . $result["channel_id"] : null,
-					"avatar" => null
-				],
-				"date" => strtotime($result["upload_date"]),
-				"duration" => $result["duration"],
-				"views" => $result["view_count"],
-				"thumb" => $thumb,
-				"url" => $result["url"]
-			];
-		}
-		
-		// get next page
-		if($json["limit"] + $json["offset"] < $json["total"]){
-			
-			$q["offset"] = $q["offset"] + $json["limit"];
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($q),
-					"videos",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-}

scraper/qwant.php

@@ -212,59 +212,29 @@ class qwant{
 		return $base;
 	}
 	
-	private function get($proxy, $url, $get = [], $is_post = false, $cookie = null){
+	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($is_post){
-			
-			$headers = [
-				"User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US,en;q=0.9",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"Referer: https://www.qwant.com/",
-				"Content-type: application/x-www-form-urlencoded",
-				"Origin: https://www.qwant.com",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-site",
-				"Pragma: no-cache",
-				"Cache-Control: no-cache"
-			];
-			
-			curl_setopt($curlproc, CURLOPT_POST, true);
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, http_build_query($get));
-		}else{
 		
 		$headers = [
 			"User-Agent: " . config::USER_AGENT,
 			"Accept: application/json, text/plain, */*",
-				"Accept-Language: en-US,en;q=0.9",
+			"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip, deflate, br, zstd",
+			"Accept-Encoding: gzip",
-				"Referer: https://www.qwant.com/",
-				"Origin: https://www.qwant.com",
 			"DNT: 1",
-				($cookie !== null ? "Cookie: {$cookie}" : ""),
-				"Sec-GPC: 1",
 			"Connection: keep-alive",
+			"Origin: https://www.qwant.com",
+			"Referer: https://www.qwant.com/",
 			"Sec-Fetch-Dest: empty",
 			"Sec-Fetch-Mode: cors",
 			"Sec-Fetch-Site: same-site",
-				"Priority: u=4",
-				"Pragma: no-cache",
-				"Cache-Control: no-cache",
 			"TE: trailers"
 		];
 		
+		$curlproc = curl_init();
+		
 		if($get !== []){
 			$get = http_build_query($get);
 			$url .= "?" . $get;
 		}
-		}
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
@@ -301,8 +271,6 @@ class qwant{
 			[$params, $proxy] = $this->backend->get($get["npt"], "web");
 			
 			$params = json_decode($params, true);
-			$cookie = $params["cookie"];
-			unset($params["cookie"]);
 			
 		}else{
 			
@@ -329,8 +297,7 @@ class qwant{
 				"device" => "desktop",
 				"tgp" => 3,
 				"safesearch" => 0,
-				"displayed" => "true",
+				"displayed" => "true"
-				"llm" => "false"
 			];
 			
 			switch($get["nsfw"]){
@@ -339,24 +306,18 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
-			
-			$cookie = null;
 		}
 		/*
 		$handle = fopen("scraper/qwant_web.json", "r");
 		$json = fread($handle, filesize("scraper/qwant_web.json"));
 		fclose($handle);*/
 		
-		$cookie = $this->fuck_datadome($proxy, $search);
-		
 		try{
 			$json =
 				$this->get(
 					$proxy,
-					"https://api.qwant.com/v3/search/web",
+					"https://fdn.qwant.com/v3/search/web",
-					$params,
+					$params
-					false,
-					$cookie
 				);
 			
 		}catch(Exception $error){
@@ -392,8 +353,6 @@ class qwant{
 			"related" => []
 		];
 		
-		$this->detect_errors($json);
-		
 		if(
 			$json["status"] != "success" &&
 			$json["data"]["error_code"] === 5
@@ -403,6 +362,8 @@ class qwant{
 			return $out;
 		}
 		
+		$this->detect_errors($json);
+		
 		if(!isset($json["data"]["result"]["items"]["mainline"])){
 			
 			throw new Exception("Server did not return a result object");
@@ -449,7 +410,10 @@ class qwant{
 					"thumb" =>
 						$answer["data"]["result"]["thumbnail"]["landscape"] == null ?
 						null :
-						$this->unshitimage($answer["data"]["result"]["thumbnail"]["landscape"]),
+						$this->unshitimage(
+							$answer["data"]["result"]["thumbnail"]["landscape"],
+							false
+						),
 					"table" => [],
 					"sublink" => []
 				];
@@ -475,7 +439,6 @@ class qwant{
 		if($json["data"]["result"]["lastPage"] === false){
 			
 			$params["offset"] = $params["offset"] + 10;
-			$params["cookie"] = $cookie;
 			
 			$out["npt"] =
 				$this->backend->store(
@@ -617,9 +580,6 @@ class qwant{
 				);
 			
 			$params = json_decode($params, true);
-			$cookie = $params["cookie"];
-			unset($params["cookie"]);
-			
 		}else{
 			
 			$search = $get["s"];
@@ -660,19 +620,13 @@ class qwant{
 				case "maybe": $params["safesearch"] = 1; break;
 				case "no": $params["safesearch"] = 2; break;
 			}
-			
-			$cookie = null;
 		}
 		
-		$cookie = $this->fuck_datadome($proxy, $search);
-		
 		try{
 			$json = $this->get(
 				$proxy,
 				"https://api.qwant.com/v3/search/images",
 				$params,
-				false,
-				$cookie
 			);
 		}catch(Exception $err){
 			
@@ -816,7 +770,7 @@ class qwant{
 			}else{
 				
 				$thumb = [
-					"url" => $this->unshitimage($video["thumbnail"]),
+					"url" => $this->unshitimage($video["thumbnail"], false),
 					"ratio" => "16:9"
 				];
 			}
@@ -916,7 +870,7 @@ class qwant{
 			}else{
 				
 				$thumb = [
-					"url" => $this->unshitimage($news["media"][0]["pict_big"]["url"]),
+					"url" => $this->unshitimage($news["media"][0]["pict_big"]["url"], false),
 					"ratio" => "16:9"
 				];
 			}
@@ -954,17 +908,6 @@ class qwant{
 			
 			throw new Exception("Qwant returned an API error");
 		}
-		
-		if(
-			isset($json["url"]) &&
-			preg_match(
-				'/captcha/i',
-				$json["url"]
-			)
-		){
-			
-			throw new Exception("Qwant returned a captcha redirect");
-		}
 	}
 	
 	private function limitstrlen($text){
@@ -977,126 +920,18 @@ class qwant{
 		return trim($text, ". ");
 	}
 	
-	private function unshitimage($url){
+	private function unshitimage($url, $is_bing = true){
 		
 		// https://s1.qwant.com/thumbr/0x0/8/d/f6de4deb2c2b12f55d8bdcaae576f9f62fd58a05ec0feeac117b354d1bf5c2/th.jpg?u=https%3A%2F%2Fwww.bing.com%2Fth%3Fid%3DOIP.vvDWsagzxjoKKP_rOqhwrQAAAA%26w%3D160%26h%3D160%26c%3D7%26pid%3D5.1&q=0&b=1&p=0&a=0
-		// https://s2.qwant.com/thumbr/474x289/7/f/412d13b3fe3a03eb2b89633c8e88b609b7d0b93cdd9a5e52db3c663e41e65e/th.jpg?u=https%3A%2F%2Ftse.mm.bing.net%2Fth%3Fid%3DOIP.9Tm_Eo6m7V7ltN19mxduDgHaEh%26pid%3DApi&q=0&b=1&p=0&a=0
+		parse_str(parse_url($url)["query"], $parts);
 		
-		$image = parse_url($url);
+		if($is_bing){
+			$parse = parse_url($parts["u"]);
+			parse_str($parse["query"], $parts);
 			
-		if(
+			return "https://" . $parse["host"] . "/th?id=" . urlencode($parts["id"]);
-			!isset($image["host"]) ||
-			!isset($image["query"])
-		){
-			
-			// cant do anything
-			return $url;
 		}
 		
-		$id = null;
+		return $parts["u"];
-		
-		if(
-			preg_match(
-				'/s[0-9]+\.qwant\.com$/',
-				$image["host"]
-			)
-		){
-			
-			parse_str($image["query"], $str);
-			
-			// we're being served a proxy URL
-			if(isset($str["u"])){
-				
-				$bing_url = $str["u"];
-			}else{
-				
-				// give up
-				return $url;
-			}
-		}
-		
-		// parse bing URL
-		$id = null;
-		$image = parse_url($bing_url);
-		
-		if(isset($image["query"])){
-			
-			parse_str($image["query"], $str);
-			
-			if(isset($str["id"])){
-				
-				$id = $str["id"];
-			}
-		}
-		
-		if($id === null){
-			
-			$id = explode("/th/id/", $image["path"], 2);
-			
-			if(count($id) !== 2){
-				
-				// malformed
-				return $url;
-			}
-			
-			$id = $id[1];
-		}
-		
-		if(is_array($id)){
-			
-			// fuck off, let proxy.php deal with it
-			return $url;
-		}
-		
-		return "https://" . $image["host"] . "/th?id=" . rawurlencode($id);
-	}
-	
-	private function fuck_datadome($proxy, $search){
-		
-		if($cookie === null){
-			// get datadome bullshit
-			// yeah go ahead and patch this you code monkey
-			
-			// ... you know, instead of fixing this shit, can you make the second page of results return more than 1 result?
-			// i search shit like "higurashi" and i only get 1 result and then just a bunch of related searches ad finitum
-			// also, stop pretending you guys have your own index, this is clearly all bing results
-			try{
-				$datadome =
-					$this->get(
-						$proxy,
-						"https://dd.qwant.com/js/",
-						[
-							"jspl" => "-oirHDeP7G3G9LXY3rhxGDYf6t_ZlYPzaiL2dRhQ34uNUc4dO2Td0SmmKu_VJp7-jrAIoOMrTqKgCtV4cPzntMyNuOYN7QQ1q0Fi2c-679vYpJr9ub5dRFXMhCMQmjZsOWXUa8yafIXf85UA5Goct-n4hVT5dFfBGM-dE6gwZEFkHQ1j1195sLybZFTNA1IfX26MBMxlVHoyT-aHtkd6YjGXbvOPMMsObVHDa_Cf4BmApRGo-eehsAttsCqDRwkl0M2E1b0OXCIwMVnCK-qWZ81MGaJGjqxdj_LaxhAjE9JI_4oNedAg4Yj4m_vkQBKdmWJT_bTUsllAZXqX5IxYQHJ8WvlknOTr6sR3wgoSOa_JNVHkUQxqXBbQcoDdy6VciL5rIqwWl4Y2caVYzeXa8YNXxwsfrPDegH3dQ6vOiIiCDT3bexkcS_y7GAioPPv9qDu7M9y4sI8t0VkPgMbWY6MIu5u9WAv6lEiKIYrtlyU-iiXnNdkd9EnpC1TD8_s12PtCuLg5bGgLVmI1dw8VXVZxmVyFvfBtuqXkzM-kZOyv-YIRcf-wPY8DUtnHox1d6t64oYhGv-pYV5dmZ0rGs99LC6ThiifeF_oHz5ZK46Ms68WsiAjc1HLAbFPFxWw8X6lyLlpT5aHgFdDBeCTExKXyItlc9PBxkgK8UnXHQFiWX3HTen8rziwkI31AbkMU7xGP6cvuwuLG7MA7sH3yLF1uimQlhr6EuTrImWQNV7X81DqyZ0OE7I-uT9bp_xISa5it-6AJpFtGcwz04_OuO2zvZazBeiKJI5Fw0X2r3RHTHdA29SXqTIe4y2LRAAp0MSAZUIJAMIbbrz9BG-3D5BHfQUA1Bbuklxv-tNWiuanWbbTmAiuxTAo9P-D2ZMrWHwdN8sCb3nINLur38z1xviSs2qEEjKo9kzQIWoHgwm_amV-jjDtNBXwpCQnoaqt3FfL8e59ecYWVg61Gnl3YCwq6rubFdkIv6UStcWSlj4pRFlFZSjQxzznvGTAc5Nz1JJWVeBHJItei4wqDUYEw2BNTS_fU9keV3LFyKDH_x248AxHLiXt583DhZZwPAKlpmMo0E7odzfOgVQGM5b9l64NL3RQRq8cDxI8KgmOFnPHdI7Myq1A3nW1uoBB8m9_GR_9orboKl5tAQrT0VTMRd6JyNwOOQ71ZoQpMwsD5Xd9YDZd618TNE9dA38h9OyBco_-hwKCYayyEqib1atLLBO_gnO4InR471AklOokDOKx1oY_xgNhlFrZskE4nQQOqeCQ_47oysob9s3kblA8UqDrL2vM3i2KPLWwlQzMAPrQAIv7Amp6rO-wYwBQdbP7XghYympxx9-hNxc1Duq-nS4R3pjPtZg3LoCvG1N_0JLf9jfPBqw_rdPM6DnjEOvpCdvrfv4wBlEpgqbaRQCZBPEz-O5ZlY24OyPvaNX0krPeXkbi-KYuwZSAbfW0zsksXb8cbRmz6tvh25IRYP1XNDPVd8NFrYOdDH9GzhV5MYZziYMurQlVwgumGLz85Y6qTHZU2D5qkMH7o5C5bzptReO96889oxkReS_LDhzdxH0tAQ5a9GgETJTHAXJ9Lt7X037i0qQlatbPLizOl9fo4vUBGJ7w-uSUmIzg1yIwOo8xhLu39WghXfoJ_EX5ptNi7arpAo11UoJIGEgWgq7bS40q2dtnG5HYpp9ef37iFgztM7PV5dlJf_W-n9uXJyCk2nhxZeuGiIGhMTzjP_8oeyiMFelmoyLlqK8wOs6hvWqgDD4NwtZc-XBkEAKsxuPrirb_sszp0z6BWsh3D6fBxAW_lTrNVkxZ1EBtoEN72HvO1aeX8PVuJP3CKwkkUcAhJ5Zvajo8t4Ai8ave2MTZul4QhDIe7ZALJMo71u-hNSj8YG9JLPqF1jcnbkl8ykt_XMMxX8wSZ_AfTsfmAuOWBfEzxv8Fl2Ne-oxcEtfuYnYNJIjQDHXXdtYbWKBezE41wonQu7FWh3wSSF3DDm9EocWMsUZ610nrIviOFdtTqjrY7KnOZp0hS9Hm2V04z-06Db-To71L5O5poYKuOlDAP_ULfW0hgMgYaPnt4Nive9gtue6ZQ5yNFylKM11zCWcF8vmuEPGx4NIZjvNFB-QSkdxZtBTQOgO6r4BqHJNZIDqiD9A6PyMemVtQEo-JhooX6bwHFHx6C84r-kKq1lg0RaDelbu1h6347kocxOczdgHWlagKPFrUvHpPHvCZeFmzLLmPWbDEpUhhfkbEZlNe47svO5WN1TYZMCAlE0g01pe6b_NB9-twQ4FJB1SkUQRA4o5DxiIHSgXBBfw1X98XPEKcJgCU2zRbYuU69mvyPcsxoPsLo8Cl2OAXj2U1tcLUxVhR_vu3I0SG_Ee8hxBp5b6QbpIu2sE0bgpqfi88XqUXf3qvxGbkoZDZ8dPvEvCVmpDZOOGJY5Db4MnpKBByW79lUnpbLCbsVOoxYu8SVmVQ63eGY9yBlACuWxlXA_PhCpnKYwMppjxKvKqalNJGMMg5WQ8cvhsIftE5wgaj21n0A15GwRO1WgUcsO_dLe1MqO88hTHWHV0Q3gNut3sdsmVr_n-0SniLtVAaw4IEtDZWNUfhpaoC7iyKxoJ86t8r697EYPlWUu4nTQhb3TOXAjHF22rFsJJMldKHVxjxjd086ky3OY3L_7cbjVof60-y4gTJdumw5hch-mhaT93nVnaTuPLGCAC48dZSgOitW8Dw62mPS05fTPAfKUY1HpyIWr9ACk7U2KVAijpzFB-bXvqJeCyBeUHr4cB61K2XxTmkNA8Oo9RC32TeYDrThFzoUyYZ_D6zPe1lLtQaFsPRvDBSsYCcr5fcABlST4DyUgnkgqtx-0-D1QEwtWiH3xiHLQGa82ldjqNIXL9LqwxYoCRBKuljHyqEylZD26A9Jp6SL9rk_Ol6TJlYa3zx44PHyMoA3Coy-HIZW5NMwVKdV7egUDDOchvuFnc1Zv2QpSFKc0JY8lYFtSm1HZFwjq7hf2hBqkU0Xneh8AT4Ty9QVb4mJ8rpzc8m_NBZNrpZVE-pIzmv2yGjnRwKSrwsjat7b2iY34UPFqGBlDk5FX6EfRlN9bq-nztWdextLc47ndr1gxsjC-vr6tbdXl5FxMVhbsnLH80wOC6hHubxCysr5C7Ft59s3MLhVQCc3IIOPMBhLOjjjVsu6K8kAJvvamIpj-iYUQj9T8JztAqfobytzbiwPsQ3mywUGkJpZxTjXjqkKdT6yfFdW5ksR8p3VlKGAAKft9TKKcc5IgErmIRtiZF4hGhB45nwIO5XHA59V94QuV3b0oJrleuDCBj0iDFqWYpsUVHyKl3RjrO86Yq1IbNRwXIIMrlsVYos9CS1I75k9kW4yQC6ni3a2ZitDkiEB-SoQuTlBsvfTs892Wt5q3CWH6_7srZO6l2QEAlWqyQp_P4i8j8uiEbZksGsgghVIgOkriblDP9ml7PdkpRX7S3OpY7jjQgTNR8XHaZ-SEw0-zPiqIIt5VioFK8JljPn-nMIDx-ew_vTDKqmKEOF39ezQcc5oQNTQNUvTAe8qCbK4ghS3fagItezoHtB5zm3qwhx9Iu6JMJy0UhywUZBaInY6NokiVZGDgtoqlVjOk7K4299MVZgsOr2P9QHRDo41lYnCeWWFUf50d-ouEOUO_ngteDpgG1DfV64eycrUOWAnpfI8XOUZ7mxS89SqJriw3T3tcljowHiOFw9QjYsjJi0vuHL5jfoBZVdYAUH6KiLfIScO-KgqitdUTpFwks8Qk7xfd2HcEfArlzSiAt3Hmb2Ty-5mhvtQj2RbPehgoI7gqa60b6Qffth-5tbIjXoKlZCQsaKVMAggF9C3w0Rv5uJ3ExCTGsfKO9IVOf--AfxaA8cNoO0W3NTaMH2iWftY0O1-1x3VJEcerp2v9BcZ0iIL3VFJ9eyOPYNHNwTYrI3S6OrVZxuQXFbu2E_Us32wxYqXvnMx-DEOk7bUUFIanL69Cbn_mR_9Wm1sFixBQ15diJoe1Oyl3XWHWL6_YF4XnHzZlMWySt3Awdeynwu99Ter8WMGXl-6K30novKRWC9jDVOfAtxPvXBkkRtpm8nnJT8fwkmIQoWOEPT2MCg3VYzzHzVEH-AueuRiCZj_o7L-fRqbtFrWPyVnGuetUYS5u0D3n7Rr6kCVqWhLfwjqAwV_calzIQs8-J0-AJe6GGaaONPYfXwXjYan0Sl11CS64lXvx7i3MgeOZuH210T6g6hj54dSLw0upO1L8k0FGli3I5OhLMekeGMLT_N2akyHktjbz8Odpk2ms0pJP_PdqvUegjXgH81F5lr40HAXZs7jfC9ytgcWaEjwlmnoFHSCTdlCIgJqEBzItXiz6ilk5zP35VwxdQTNoms7Meqg-ZrgRuLRaqXNtpQT9qWBmiFP6zoYLwymqytZ_P1SV9DCM-KIZQ2Cq-ew8PsYSrwl8GmfuegoKEMBiON1I4mC9ChDgzJmbo5P9te60T0URS5g9ExiQuSo-JXs0MKC57LLiNo3MPZvqR1wRJOTg0_FqS4JFxzPwN5OqhBzfs25f6kK-axf0wWp-LetUx3U1CTNaxfbCTK5tb8KnLOIs3lDwJw5H5wIIRgXnfNLBJT8A2srfXkdpA",
-							"eventCounters" => [],
-							"jsType" => "ch",
-							"cid" => "3i4vPGHCl74uIBumWaxF2a0PjFkyCJxcSvFnn~heX7keA6sGHgRbnzSYcMr_RzJFlfeeBdg9HeufzFPxnvSHbyWowSvjm6mNCSPhkx6MwDqptANoj93ZgGVbrzxGtcYe",
-							"ddk" => "78B13B7513D180B7AB6D6FF9EB0A51",
-							"Referer" => "https://www.qwant.com/?q={$search}&t=web",
-							"request" => "/?q={$search}&t=web",
-							"responsePage" => "origin",
-							"ddv" => "5.8.0"
-						],
-						true
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JS challenge token");
-			}
-			
-			$datadome = json_decode($datadome, true);
-			if($datadome === null){
-				
-				throw new Exception("Failed to decode JS challenge JSON");
-			}
-			
-			if(!isset($datadome["cookie"])){
-				
-				throw new Exception("Failed to get cookie from JS challenge endpoint");
-			}
-			
-			$cookie = explode(";", $datadome["cookie"])[0];
-		}
-		
-		return $cookie;
 	}
 }

scraper/sc.php

@@ -135,8 +135,6 @@ class sc{
 			
 			$type = $get["type"];
 			$proxy = $this->backend->get_ip();
-			
-			// token is not tied to an IP address
 			$token = $this->get_token($proxy);
 			
 			switch($type){
@@ -151,7 +149,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -166,7 +164,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -181,7 +179,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -196,7 +194,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -211,7 +209,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -227,7 +225,7 @@ class sc{
 						"limit" => 20,
 						"offset" => 0,
 						"linked_partitioning" => 1,
-						"app_version" => 1769170554,
+						"app_version" => 1713542117,
 						"app_locale" => "en"
 					];
 					break;
@@ -249,34 +247,17 @@ class sc{
 		fclose($handle);
 		*/
 		
-		$this->fuckhtml->load($json);
-		
-		// detect for cloudfront error
-		$title =
-			$this->fuckhtml
-			->getElementsByTagName(
-				"title"
-			);
-		
-		if(
-			count($title) !== 0 &&
-			str_contains($title[0]["innerHTML"], "The request could not be satisfied")
-		){
-			
-			throw new Exception("Ratelimited by Cloudfront, add some working proxies");
-		}
-		
 		$json = json_decode($json, true);
 		
 		if($json === null){
 			
 			if($last_attempt === true){
 				
-				throw new Exception("Fetched an invalid token");
+				throw new Exception("Fetched an invalid token (please report!!)");
 			}
 			
 			// token might've expired, get a new one and re-try search
-			$token = $this->get_token($proxy, true);
+			$this->get_token($proxy);
 			return $this->music($get, true);
 		}
 		
@@ -397,12 +378,7 @@ class sc{
 					break;
 				
 				case "track":
-					$stream = [
+					if(stripos($item["monetization_model"], "TIER") === false){
-						"endpoint" => null,
-						"url" => null
-					];
-					
-					/*if(stripos($item["monetization_model"], "TIER") === false){
 						
 						$stream = [
 							"endpoint" => "sc",
@@ -418,7 +394,7 @@ class sc{
 							"endpoint" => null,
 							"url" => null
 						];
-					}*/
+					}
 					
 					// parse track
 					$out["song"][] = [
@@ -446,19 +422,16 @@ class sc{
 		return $out;
 	}
 	
-	public function get_token($proxy, $force_refresh = false){
+	public function get_token($proxy){
 		
 		$token = apcu_fetch("sc_token");
 		
-		if(
+		if($token !== false){
-			$token !== false &&
-			$force_refresh === false
-		){
 			
 			return $token;
 		}
 		
-		// get token from main page
+		// search through all javascript components on the main page
 		try{
 			$html =
 				$this->get(
@@ -474,44 +447,49 @@ class sc{
 		
 		$this->fuckhtml->load($html);
 		
-		$json =
+		$scripts =
-			preg_split(
-				'/window.__sc_hydration ?= ?/',
-				$html
-			);
-		
-		if(!isset($json[1])){
-			
-			throw new Exception("Failed to find JSON containing token");
-		}
-		
-		$json =
-			json_decode(
 			$this->fuckhtml
-				->extract_json(
+			->getElementsByTagName(
-					$json[1]
+				"script"
-				),
-				true
 			);
 		
-		if($json === null){
+		foreach($scripts as $script){
-			
-			throw new Exception("Failed to decode JSON containing token");
-		}
-		
-		foreach($json as $item){
 			
 			if(
-				isset($item["hydratable"]) &&
+				!isset($script["attributes"]["src"]) ||
-				isset($item["data"]["id"])
+				strpos($script["attributes"]["src"], "sndcdn.com") === false
 			){
 				
-				apcu_store("sc_token", $item["data"]["id"]);
+				continue;
-				return $item["data"]["id"];
+			}
+			
+			try{
+				$js =
+					$this->get(
+						$proxy,
+						$script["attributes"]["src"],
+						[]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to fetch search token");
+			}
+			
+			preg_match(
+				'/client_id=([^"]+)/',
+				$js,
+				$token
+			);
+			
+			if(isset($token[1])){
+				
+				apcu_store("sc_token", $token[1]);
+				return $token[1];
+				break;
 			}
 		}
 		
-		throw new Exception("Failed to find a valid token");
+		throw new Exception("Did not find a Soundcloud token in the Javascript blobs");
 	}
 	
 	private function limitstrlen($text){

scraper/sepiasearch.php

@@ -1,541 +0,0 @@
-<?php
-
-class sepiasearch{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("sepiasearch");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"nsfw" => [
-				"display" => "NSFW",
-				"option" => [
-					"yes" => "Yes", // &sensitiveContent=both
-					"no" => "No" // &sensitiveContent=false
-				]
-			],
-			"language" => [
-				"display" => "Language", // &language=
-				"option" => [
-					"any" => "Any language",
-					"en" => "English",
-					"fr" => "Français",
-					"ar" => "العربية",
-					"ca" => "Català",
-					"cs" => "Čeština",
-					"de" => "Deutsch",
-					"el" => "ελληνικά",
-					"eo" => "Esperanto",
-					"es" => "Español",
-					"eu" => "Euskara",
-					"fa" => "فارسی",
-					"fi" => "Suomi",
-					"gd" => "Gàidhlig",
-					"gl" => "Galego",
-					"hr" => "Hrvatski",
-					"hu" => "Magyar",
-					"is" => "Íslenska",
-					"it" => "Italiano",
-					"ja" => "日本語",
-					"kab" => "Taqbaylit",
-					"nl" => "Nederlands",
-					"no" => "Norsk",
-					"oc" => "Occitan",
-					"pl" => "Polski",
-					"pt" => "Português (Brasil)",
-					"pt-PT" => "Português (Portugal)",
-					"ru" => "Pусский",
-					"sk" => "Slovenčina",
-					"sq" => "Shqip",
-					"sv" => "Svenska",
-					"th" => "ไทย",
-					"tok" => "Toki Pona",
-					"tr" => "Türkçe",
-					"uk" => "украї́нська мо́ва",
-					"vi" => "Tiếng Việt",
-					"zh-Hans" => "简体中文（中国）",
-					"zh-Hant" => "繁體中文（台灣）"
-				]
-			],
-			"type" => [
-				"display" => "Result type", // i handle this
-				"option" => [
-					"videos" => "Videos",
-					"playlists" => "Playlists",
-					"channels" => "Channels"
-				]
-			],
-			"sort" => [
-				"display" => "Sort by",
-				"option" => [
-					"best" => "Best match", // no filter
-					"-publishedAt" => "Newest", // sort=-publishedAt
-					"publishedAt" => "Oldest" // sort=publishedAt
-				]
-			],
-			"newer" => [ // &startDate=2025-07-26T04:00:00.000Z
-				"display" => "Newer than",
-				"option" => "_DATE"
-			],
-			"duration" => [
-				"display" => "Duration",
-				"option" => [
-					"any" => "Any duration",
-					"short" => "Short (0-4mins)", // &durationRange=short
-					"medium" => "Medium (4-10 mins)",
-					"long" => "Long (10+ mins)",
-				]
-			],
-			"category" => [
-				"display" => "Category", // &categoryOneOf[]=
-				"option" => [
-					"any" => "Any category",
-					"1" => "Music",
-					"2" => "Films",
-					"3" => "Vehicles",
-					"4" => "Art",
-					"5" => "Sports",
-					"6" => "Travels",
-					"7" => "Gaming",
-					"8" => "People",
-					"9" => "Comedy",
-					"10" => "Entertainment",
-					"11" => "News & Politics",
-					"12" => "How To",
-					"13" => "Education",
-					"14" => "Activism",
-					"15" => "Science & Technology",
-					"16" => "Animals",
-					"17" => "Kids",
-					"18" => "Food"
-				]
-			],
-			"display" => [
-				"display" => "Display",
-				"option" => [
-					"any" => "Everything",
-					"true" => "Live videos", // &isLive=true
-					"false" => "VODs" // &isLive=false
-				]
-			],
-			"license" => [
-				"display" => "License", // &license=
-				"option" => [
-					"any" => "Any license",
-					"1" => "Attribution",
-					"2" => "Attribution - Share Alike",
-					"3" => "Attribution - No Derivatives",
-					"4" => "Attribution - Non Commercial",
-					"5" => "Attribution - Non Commercial - Share Alike",
-					"6" => "Attribution - Non Commercial - No Derivatives",
-					"7" => "Public Domain Dedication"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = []){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		curl_setopt(
-			$curlproc,
-			CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Referer: https://sepiasearch.org/search",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"Priority: u=0",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function video($get){
-		
-		if($get["npt"]){
-			
-			[$npt, $proxy] =
-				$this->backend
-				->get(
-					$get["npt"],
-					"videos"
-				);
-			
-			$npt = json_decode($npt, true);
-			$type = $npt["type"];
-			$npt = $npt["npt"];
-		}else{
-			
-			$proxy = $this->backend->get_ip();
-						
-			$npt = [
-				"search" => $get["s"],
-				"start" => 0,
-				"count" => 20
-			];
-			
-			if($get["type"] == "videos"){
-				
-				//
-				// Parse video filters
-				//
-				switch($get["nsfw"]){
-					
-					case "yes": $npt["nsfw"] = "both"; break;
-					case "no": $npt["nsfw"] = "false"; break;
-				}
-				
-				$npt["boostLanguages[]"] = "en";
-				if($get["language"] != "any"){
-					
-					$npt["languageOneOf[]"] = $get["language"];
-				}
-				
-				if($get["sort"] != "best"){
-					
-					$npt["sort"] = $get["sort"];
-				}
-				
-				if($get["newer"] !== false){
-					
-					$date = new DateTime("@{$get["newer"]}");
-					$date->setTimezone(new DateTimeZone("UTC"));
-					$formatted = $date->format("Y-m-d\TH:i:s.000\Z");
-					
-					$npt["startDate"] = $formatted;
-				}
-				
-				switch($get["duration"]){
-					
-					case "short":
-						$npt["durationMax"] = 240;
-						break;
-					
-					case "medium":
-						$npt["durationMin"] = 240;
-						$npt["durationMax"] = 600;
-						break;
-					
-					case "long":
-						$npt["durationMin"] = 600;
-						break;
-				}
-				
-				if($get["category"] != "any"){
-					
-					$npt["categoryOneOf[]"] = $get["category"];
-				}
-				
-				if($get["display"] != "any"){
-					
-					$npt["isLive"] = $get["display"];
-				}
-				
-				if($get["license"] != "any"){
-					
-					// typo in license, lol
-					$npt["licenceOneOf[]"] = $get["license"];
-				}
-			}
-			
-			$type = $get["type"];
-		}
-		
-		switch($type){
-			
-			case "videos":
-				$url = "https://sepiasearch.org/api/v1/search/videos";
-				break;
-			
-			case "channels":
-				$url = "https://sepiasearch.org/api/v1/search/video-channels";
-				break;
-			
-			case "playlists":
-				$url = "https://sepiasearch.org/api/v1/search/video-playlists";
-				break;
-		}
-		
-		//$json = file_get_contents("scraper/sepia.json");
-		try{
-			
-			$json =
-				$this->get(
-					$proxy,
-					$url,
-					$npt
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to parse JSON");
-		}
-		
-		if(isset($json["errors"])){
-			
-			$msg = [];
-			foreach($json["errors"] as $error){
-				
-				if(isset($error["msg"])){
-					
-					$msg[] = $error["msg"];
-				}
-			}
-			
-			throw new Exception("Sepia Search returned error(s): " . implode(", ", $msg));
-		}
-		
-		if(!isset($json["data"])){
-			
-			throw new Exception("Sepia Search did not return a data object");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"video" => [],
-			"author" => [],
-			"livestream" => [],
-			"playlist" => [],
-			"reel" => []
-		];
-		
-		
-		switch($get["type"]){
-			
-			case "videos":
-				foreach($json["data"] as $video){
-					
-					if(count($video["account"]["avatars"]) !== 0){
-						
-						$avatar =
-							$video["account"]["avatars"][count($video["account"]["avatars"]) - 1]["url"];
-					}else{
-						
-						$avatar = null;
-					}
-					
-					if($video["thumbnailUrl"] === null){
-						
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}else{
-						
-						$thumb  = [
-							"ratio" => "16:9",
-							"url" => $video["thumbnailUrl"]
-						];
-					}
-					
-					if($video["isLive"]){
-						
-						$append = "livestream";
-					}else{
-						
-						$append = "video";
-					}
-					
-					$out[$append][] = [
-						"title" => $video["name"],
-						"description" =>
-							$this->limitstrlen(
-								$this->titledots(
-									$video["description"]
-								)
-							),
-						"author" => [
-							"name" => $video["account"]["displayName"] . " ({$video["account"]["name"]})",
-							"url" => $video["account"]["url"],
-							"avatar" => $avatar
-						],
-						"date" => strtotime($video["publishedAt"]),
-						"duration" => $video["isLive"] ? "_LIVE" : $video["duration"],
-						"views" => $video["views"],
-						"thumb" => $thumb,
-						"url" => $video["url"]
-					];
-				}
-				break;
-			
-			case "playlists":
-				foreach($json["data"] as $playlist){
-					
-					if(count($playlist["ownerAccount"]["avatars"]) !== 0){
-						
-						$avatar =
-							$playlist["ownerAccount"]["avatars"][count($playlist["ownerAccount"]["avatars"]) - 1]["url"];
-					}else{
-						
-						$avatar = null;
-					}
-					
-					if($playlist["thumbnailUrl"] === null){
-						
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}else{
-						
-						$thumb  = [
-							"ratio" => "16:9",
-							"url" => $playlist["thumbnailUrl"]
-						];
-					}
-					
-					$out["playlist"][] = [
-						"title" => $playlist["displayName"],
-						"description" =>
-							$this->limitstrlen(
-								$this->titledots(
-									$playlist["description"]
-								)
-							),
-						"author" => [
-							"name" => $playlist["ownerAccount"]["displayName"] . " ({$playlist["ownerAccount"]["name"]})",
-							"url" => $playlist["ownerAccount"]["url"],
-							"avatar" => $avatar
-						],
-						"date" => strtotime($playlist["createdAt"]),
-						"duration" => $playlist["videosLength"],
-						"views" => null,
-						"thumb" => $thumb,
-						"url" => $playlist["url"]
-					];
-				}
-				break;
-			
-			case "channels":
-				foreach($json["data"] as $channel){
-					
-					if(count($channel["avatars"]) !== 0){
-						
-						$thumb = [
-							"ratio" => "1:1",
-							"url" => $channel["avatars"][count($channel["avatars"]) - 1]["url"]
-						];
-					}else{
-						
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["author"][] = [
-						"title" => $channel["displayName"] . " ({$channel["name"]})",
-						"followers" => $channel["followersCount"],
-						"description" =>
-							$channel["videosCount"] . " videos. " .
-							$this->limitstrlen(
-								$this->titledots(
-									$channel["description"]
-								)
-							),
-						"thumb" => $thumb,
-						"url" => $channel["url"]
-					];
-				}
-				break;
-		}
-		
-		// get next page
-		if($json["total"] - 20 > $npt["start"]){
-			
-			$npt["start"] += 20;
-			
-			$npt = [
-				"type" => $get["type"],
-				"npt" => $npt
-			];
-			
-			$out["npt"] =
-				$this->backend
-				->store(
-					json_encode($npt),
-					"videos",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function titledots($title){
-		
-		$substr = substr($title, -3);
-		
-		if(
-			$substr == "..." ||
-			$substr == "…"
-		){
-			
-			return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0");
-		}
-		
-		return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0");
-	}
-	
-	private function limitstrlen($text){
-		
-		return
-			explode(
-				"\n",
-				wordwrap(
-					str_replace(
-						["\n\r", "\r\n", "\n", "\r"],
-						" ",
-						$text
-					),
-					300,
-					"\n"
-				),
-				2
-			)[0];
-	}
-}

scraper/startpage.php

@@ -408,8 +408,6 @@ class startpage{
 			//$html = file_get_contents("scraper/startpage.html");
 		}
 		
-		//echo htmlspecialchars($html);
-		
 		$this->detect_captcha($html);
 		
 		if(
@@ -566,16 +564,12 @@ class startpage{
 					break;
 				
 				case "spellsuggest-google":
-					
-					if(isset($category["results"][0]["query"])){
-						
 					$out["spelling"] =
 						[
 							"type" => "including",
-								"using" => rawurldecode($json["render"]["query"]),
+							"using" => $json["render"]["query"],
-								"correction" => html_entity_decode($category["results"][0]["query"])
+							"correction" => $category["results"][0]["query"]
 						];
-					}
 					break;
 				
 				case "dictionary-qi":
@@ -651,6 +645,318 @@ class startpage{
 			}
 		}
 		
+		// parse instant answers
+		if(
+			$get["extendedsearch"] == "yes" &&
+			$get_instant_answer === true
+		){
+			
+			// https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=BqZ3inqrAgF701&sr=1
+			try{
+				$post = [
+					"se" => "n0vze2y9dqwy",
+					"q" => $json["render"]["query"],
+					"results" => [], // populate
+					"enableKnowledgePanel" => true,
+					"enableMediaThumbBar" => false,
+					"enableSearchSuggestions" => false,
+					"enableTripadvisorProperties" => [],
+					"enableTripadvisorPlaces" => [],
+					"enableTripadvisorPlacesForLocations" => [],
+					"enableWebProducts" => false,
+					"tripadvisorPartnerId" => null,
+					"tripadvisorMapColorMode" => "light",
+					"tripadvisorDisablesKnowledgePanel" => false,
+					"instantAnswers" => [
+						"smartAnswers",
+						"youtube",
+						"tripadvisor"
+					],
+					"iaType" => null,
+					"forceEnhancedKnowledgePanel" => false,
+					"shoppingOnly" => false,
+					"allowAdultProducts" => true,
+					"lang" => "en",
+					"browserLang" => "en-US",
+					"browserTimezone" => "America/New_York",
+					"market" => null,
+					"userLocation" => null,
+					"userDate" => date("Y-m-d"),
+					"userAgentType" => "unknown"
+				];
+				
+				foreach($out["web"] as $result){
+					
+					$post["results"][] = [
+						"url" => $result["url"],
+						"title" => $result["title"]
+					];
+				}
+				
+				$post = json_encode($post, JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE);
+				
+				$additional_data =
+					$this->get(
+						$proxy,
+						"https://www.startpage.com/sp/qi?qimsn=ex&sxap=%2Fv1%2Fquery&sc=" . $json["render"]["callback_sc"] . "&sr=1",
+						$post,
+						true,
+						true
+					);
+				
+				$additional_data = json_decode($additional_data, true);
+				
+				if($additional_data === null){
+					
+					throw new Exception("Failed to decode JSON"); // just break out, dont fail completely
+				}
+				
+				if(!isset($additional_data["knowledgePanel"])){
+					
+					throw new Exception("Response has missing data (knowledgePanel)");
+				}
+				
+				$additional_data = $additional_data["knowledgePanel"];
+				
+				$answer = [
+					"title" => $additional_data["meta"]["title"],
+					"description" => [
+						[
+							"type" => "quote",
+							"value" => $additional_data["meta"]["description"]
+						]
+					],
+					"url" => $additional_data["meta"]["origWikiUrl"],
+					"thumb" => $additional_data["meta"]["image"],
+					"table" => [],
+					"sublink" => []
+				];
+				
+				// parse html for instant answer
+				$this->fuckhtml->load($additional_data["html"]);
+				
+				$div =
+					$this->fuckhtml
+					->getElementsByTagName(
+						"div"
+					);
+				
+				// get description
+				$description =
+					$this->fuckhtml
+					->getElementsByClassName(
+						"sx-kp-short-extract sx-kp-short-extract-complete",
+						$div
+					);
+				
+				if(count($description) !== 0){
+					
+					$answer["description"][] = [
+						"type" => "text",
+						"value" =>
+							html_entity_decode(
+								$this->fuckhtml
+								->getTextContent(
+									$description[0]
+								)
+							)
+					];
+				}
+				
+				// get socials
+				$socials =
+					$this->fuckhtml
+					->getElementsByClassName(
+						"sx-wiki-social-link",
+						"a"
+					);
+				
+				foreach($socials as $social){
+					
+					$title =
+						$this->fuckhtml
+						->getTextContent(
+							$social["attributes"]["title"]
+						);
+					
+					$url =
+						$this->fuckhtml
+						->getTextContent(
+							$social["attributes"]["href"]
+						);
+					
+					switch($title){
+						
+						case "Official Website":
+							$title = "Website";
+							break;
+					}
+					
+					$answer["sublink"][$title] = $url;
+				}
+				
+				// get videos
+				$videos =
+					$this->fuckhtml
+					->getElementsByClassName(
+						"sx-kp-video-grid-item",
+						$div
+					);
+				
+				foreach($videos as $video){
+					
+					$this->fuckhtml->load($video);
+					
+					$as =
+						$this->fuckhtml
+						->getElementsByTagName(
+							"a"
+						);
+					
+					if(count($as) === 0){
+						
+						// ?? invalid
+						continue;
+					}
+					
+					$image =
+						$this->fuckhtml
+						->getElementsByAttributeName(
+							"data-sx-src",
+							"img"
+						);
+					
+					if(count($image) !== 0){
+						
+						$thumb = [
+							"ratio" => "16:9",
+							"url" =>
+								$this->fuckhtml
+								->getTextContent(
+									$image[0]["attributes"]["data-sx-src"]
+								)
+						];
+					}else{
+
+						$thumb = [
+							"ratio" => null,
+							"url" => null
+						];
+					}
+					
+					$out["video"][] = [
+						"title" =>
+							$this->fuckhtml
+							->getTextContent(
+								$as[0]["attributes"]["title"]
+							),
+						"description" => null,
+						"date" => null,
+						"duration" => null,
+						"views" => null,
+						"thumb" => $thumb,
+						"url" =>
+							$this->fuckhtml
+							->getTextContent(
+								$as[0]["attributes"]["href"]
+							)
+					];
+				}
+				
+				// reset
+				$this->fuckhtml->load($additional_data["html"]);
+				
+				// get table elements
+				$table =
+					$this->fuckhtml
+					->getElementsByClassName(
+						"sx-infobox",
+						"table"
+					);
+				
+				if(count($table) !== 0){
+					
+					$trs =
+						$this->fuckhtml
+						->getElementsByTagName(
+							"tr"
+						);
+					
+					foreach($trs as $tr){
+						
+						$this->fuckhtml->load($tr);
+						
+						// ok so startpage devs cant fucking code a table
+						// td = content
+						// th (AAAHH) = title
+						$tds =
+							$this->fuckhtml
+							->getElementsByTagName(
+								"td"	
+							);
+						
+						$ths =
+							$this->fuckhtml
+							->getElementsByTagName(
+								"th"
+							);
+						
+						if(
+							count($ths) === 1 &&
+							count($tds) === 1
+						){
+							
+							$title =
+								$this->fuckhtml
+								->getTextContent(
+									$ths[0]
+								);
+							
+							$description = [];
+							
+							$this->fuckhtml->load($tds[0]);
+							
+							$lis =
+								$this->fuckhtml
+								->getElementsByTagName(
+									"li"
+								);
+							
+							if(count($lis) !== 0){
+								
+								foreach($lis as $li){
+									
+									$description[] =
+										$this->fuckhtml
+										->getTextContent(
+											$li
+										);
+								}
+								
+								$description = implode(", ", $description);
+							}else{
+								
+								$description =
+									$this->fuckhtml
+									->getTextContent(
+										$tds[0]
+									);
+							}
+							
+							$answer["table"][$title] = $description;
+						}
+					}
+				}
+				
+				$out["answer"][] = $answer;
+				
+			}catch(Exception $error){
+				
+				// do nothing
+				//echo "error!";
+			}
+		}
+		
 		return $out;
 	}
 	
@@ -920,12 +1226,7 @@ class startpage{
 		// get results
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
-			if(
+			if($category["display_type"] == "video-youtube"){
-				preg_match(
-					'/^video-/i',
-					$category["display_type"]
-				)
-			){
 				
 				foreach($category["results"] as $video){
 					
@@ -947,7 +1248,7 @@ class startpage{
 					}
 					
 					$out["video"][] = [
-						"title" => str_replace(["", ""], "", $video["title"]),
+						"title" => $video["title"],
 						"description" => $this->limitstrlen($video["description"]),
 						"author" => [
 							"name" => $video["channelTitle"],
@@ -955,7 +1256,7 @@ class startpage{
 							"avatar" => null
 						],
 						"date" => strtotime($video["publishDate"]),
-						"duration" => $this->hms2int($category["display_type"] == "video-youtube" ? $video["duration"] : $video["duration"] / 1000),
+						"duration" => $this->hms2int($video["duration"]),
 						"views" => (int)$video["viewCount"],
 						"thumb" => $thumb,
 						"url" => $video["clickUrl"]
@@ -1107,22 +1408,27 @@ class startpage{
 			
 			if($page["name"] == "Next"){
 				
+				parse_str(
+					explode(
+						"?",
+						$page["url"],
+						2
+					)[1],
+					$str
+				);
+				
 				return
 					$this->backend->store(
 						http_build_query(
 							[
 								"lui" => "english",
 								"language" => "english",
-								"sc" => $json["render"]["search_sc"],
+								"query" => $str["q"],
-								"t" => "device",
 								"cat" => $pagetype,
-								"segment" => "organic",
+								"sc" => $str["sc"],
-								"abd" => "0",
+								"t" => "device",
-								"abe" => "0",
+								"segment" => "startpage.udog",
-								"query" => rawurldecode($json["render"]["query"]),
+								"page" => $str["page"]
-								"page" => $page["number"],
-								"qsr" => "all",
-								"qadf" => "none" // @ todo fix (??)
 							]
 						),
 						$pagetype,

scraper/swisscows.php

@@ -1,252 +0,0 @@
-<?php
-
-class swisscows{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("swisscows");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"type" => [
-				"display" => "Type",
-				"option" => [
-					"track" => "Tracks",
-					"playlist" => "Playlists"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $web_req = false){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
-		
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Access-Control-Request-Method: GET",
-			"Access-Control-Request-Headers: cache-control",
-			"Referer: https://swisscows.com/",
-			"Origin: https://swisscows.com",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-site",
-			"Priority: u=4",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function music($get, $last_attempt = false){
-		
-		if($get["npt"]){
-			
-			[$params, $proxy] = $this->backend->get($get["npt"], "music");
-			$params = json_decode($params, true);
-			$type = $params["type"];
-			$search = $params["s"];
-			$offset = $params["offset"];
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$type = $get["type"];
-			$offset = 0;
-			$proxy = $this->backend->get_ip();
-		}
-		
-		try{
-			
-			$json =
-				$this->get(
-					$proxy,
-					"https://api.swisscows.com/audio/search/{$type}s",
-					[
-						"query" => $search,
-						"offset" => $offset,
-						"itemsCount" => 20,
-						"region" => "en-US"
-					]
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"song" => [],
-			"playlist" => [],
-			"album" => [],
-			"podcast" => [],
-			"author" => [],
-			"user" => []
-		];
-		
-		if(!isset($json["items"])){
-			
-			throw new Exception("Swisscows did not return an items object");
-		}
-		
-		if($type == "track"){
-			foreach($json["items"] as $item){
-				
-				$tags = $item["tags"];
-				if(!empty($item["genre"])){
-					
-					$tags[] = $item["genre"];
-				}
-				
-				$out["song"][] = [
-					"title" => $item["title"],
-					"description" => implode(", ", $tags),
-					"url" => "/resolver?scraper=sc&target=t{$item["id"]}",
-					"views" => null,
-					"author" => [
-						"name" => null,
-						"url" => null,
-						"avatar" => null
-					],
-					"thumb" => [
-						"ratio" => "1:1",
-						"url" => $item["artworkUrl"]
-					],
-					"date" => null,
-					"duration" => $this->convert_time($item["duration"]),
-					"stream" => [					
-						"endpoint" => null,
-						"url" => null
-					]
-				];
-			}
-		}else{
-			
-			foreach($json["items"] as $item){
-				
-				$out["playlist"][] = [
-					"title" => $item["title"],
-					"description" => $this->limitstrlen($item["description"]),
-					"author" => [
-						"name" => null,
-						"url" => null,
-						"avatar" => null
-					],
-					"thumb" => [
-						"ratio" => "1:1",
-						"url" => $item["artworkUrl"]
-					],
-					"date" => null,
-					"duration" => $this->convert_time($item["duration"]),
-					"url" => "/resolver?scraper=sc&target=p{$item["id"]}",
-				];
-			}
-		}
-		
-		//
-		// get NPT
-		//
-		if(
-			isset($json["nextOffset"]) &&
-			$json["nextOffset"] !== null
-		){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode(
-						[
-							"type" => $type,
-							"s" => $search,
-							"offset" => $json["nextOffset"]
-						]
-					),
-					"music",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function limitstrlen($text){
-		
-		return
-			explode(
-				"\n",
-				wordwrap(
-					str_replace(
-						["\n\r", "\r\n", "\n", "\r"],
-						" ",
-						$text
-					),
-					300,
-					"\n"
-				),
-				2
-			)[0];
-	}
-	
-	private function convert_time($time){
-		
-		list($hours, $minutes, $seconds) = explode(':', $time);
-		
-		return
-			((int)$hours * 3600) +
-			((int)$minutes * 60) +
-			(float)$seconds;
-	}
-}

scraper/unsplash.php

@@ -1,267 +0,0 @@
-<?php
-
-class unsplash{
-	
-	public function __construct(){
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-		
-		include "lib/backend.php";
-		$this->backend = new backend("unsplash");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"order_by" => [
-				"display" => "Order by",
-				"option" => [
-					"relevance" => "Relevance",
-					"latest" => "Newest",
-					"editorial" => "Curated"
-				]
-			],
-			"orientation" => [
-				"display" => "Order by",
-				"option" => [
-					"any" => "Any orientation",
-					"landscape" => "Landscape",
-					"portrait" => "Portrait",
-					"squarish" => "Square"
-				]
-			],
-			"license" => [
-				"display" => "License",
-				"option" => [
-					"any" => "Any license",
-					"only" => "Unsplash+",
-					"none" => "Free"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $referer){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: */*",
-			"Accept-Language: en-US",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			"Referer: {$referer}",
-			"client-geo-region: global",
-			"x-client-version: 8999df28be3f138bf2c646df5d656e4dc6970ba0",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"Priority: u=0",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$filter, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$filter = json_decode($filter, true);
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			$filter = [
-				"page" => 1,
-				"per_page" => 20,
-				"query" => $search
-			];
-			
-			// add filters
-			if($get["order_by"] != "relevance"){
-				
-				$filter["order_by"] = $get["order_by"];
-			}
-			
-			if($get["orientation"] != "any"){
-				
-				$filter["orientation"] = $get["orientation"];
-			}
-			
-			if($get["license"] != "any"){
-				
-				$filter["plus"] = $get["license"];
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-				
-		// https://unsplash.com/s/photos/shibuya-night?order_by=latest&orientation=landscape&license=free
-		// https://unsplash.com/s/photos/%3Ctest-%3F!-haha == <test ?! haha
-		
-		// ?page=1&per_page=20&query=japan
-		
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://unsplash.com/napi/search/photos",
-					$filter,
-					"https://unsplash.com/s/photos/" . rawurlencode(str_replace(" ", "-", $filter["query"])),
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Could not decode the JSON payload");
-		}
-		
-		foreach($json["results"] as $image){
-			
-			$base = explode("?", $image["urls"]["raw"])[0];
-			
-			if(
-				(bool)$image["premium"] ||
-				(bool)$image["plus"]
-			){
-				
-				// when we get "premium" images, give
-				// 1. highest resolution with watermarks
-				// 2. highest resolution without watermarks
-				// (if width of image is above 900px, it has watermarks)
-				// 3. thumbnail without watermark
-				$x900 = $this->imgratio((int)$image["width"], (int)$image["height"], 900);
-				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
-				
-				$source = [
-					[
-						"url" => $base,
-						"width" => (int)$image["width"],
-						"height" => (int)$image["height"]
-					],
-					[
-						"url" => $base . "?w=900",
-						"width" => $x900[0],
-						"height" => $x900[1]
-					],
-					[
-						"url" => $base . "?w=500",
-						"width" => $x500[0],
-						"height" => $x500[1]
-					]
-				];
-			}else{
-				
-				$x500 = $this->imgratio((int)$image["width"], (int)$image["height"], 500);
-				
-				// image is free as in freedom(tm)
-				$source = [
-					[
-						"url" => $base,
-						"width" => (int)$image["width"],
-						"height" => (int)$image["height"]
-					],
-					[
-						"url" => $base . "?w=500",
-						"width" => $x500[0],
-						"height" => $x500[1]
-					]
-				];
-			}
-			
-			$title = [];
-			
-			$image["description"] = trim($image["description"]);
-			$image["alt_description"] = trim($image["alt_description"]);
-			
-			if(!empty($image["description"])){ $title[] = $image["description"]; }
-			if(!empty($image["alt_description"])){ $title[] = $image["alt_description"]; }
-			
-			$title = implode(": ", $title);
-			
-			$out["image"][] = [
-				"title" => $title,
-				"source" => $source,
-				"url" => "https://unsplash.com/photos/" . $image["slug"]
-			];
-		}
-		
-		// next page stuff
-		if($filter["page"] < (int)$json["total_pages"]){
-			
-			$filter["page"]++;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($filter),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function imgratio($width, $height, $max_width){
-		
-		$ratio = $max_width / $width;
-		$new_height = floor($height * $ratio);
-		
-		return [
-			$max_width,
-			$new_height
-		];
-	}
-}

scraper/vimeo.php

@@ -1,754 +0,0 @@
-<?php
-
-class vimeo{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("vimeo");
-		
-		include "lib/fuckhtml.php";
-		$this->fuckhtml = new fuckhtml();
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"time" => [
-				"display" => "Date uploaded", // &filter_uploaded=
-				"option" => [
-					"any" => "Any time",
-					"today" => "Last 24 hours",
-					"this-week" => "Last 7 days",
-					"this-month" => "Last 30 days",
-					"this-year" => "Last 365 days",
-				]
-			], 
-			"display" => [
-				"display" => "Display",
-				"option" => [
-					"video" => "Videos",
-					"ondemand" => "On-Demand ($$)",
-					"people" => "People",
-					"channel" => "Channels",
-					"group" => "Groups"
-				]
-			],
-			"sort" => [
-				"display" => "Sort by",
-				"option" => [
-					"relevance" => "Relevance", // no param
-					"recent" => "Newest", // &sort=latest&direction=desc
-					"popular" => "Most popular", // &sort=popularity&direction=desc
-					"a_z" => "Title, A to Z", // &sort=alphabetical&direction=asc
-					"z_a" => "Title, Z to A", // &sort=alphabetical&direction=desc
-					"longest" => "Longest", // &sort=duration&direction=desc
-					"shortest" => "Shortest", // &sort=duration&direction=asc
-				]
-			],
-			"duration" => [
-				"display" => "Duration", // &filter_duration=
-				"option" => [
-					"any" => "Any duration",
-					"short" => "Short (less than 4 minutes)",
-					"medium" => "Medium (4-10 minutes)",
-					"long" => "Long (over 10 minutes)"
-				]
-			],
-			"resolution" => [
-				"display" => "Resolution",
-				"option" => [
-					"any" => "Any resolution",
-					"4k" => "4K" // &filter_resolution=4k
-				]
-			],
-			"category" => [
-				"display" => "Category", // &filter_category=
-				"option" => [
-					"any" => "Any category",
-					"animation" => "Animation",
-					"comedy" => "Comedy",
-					"music" => "Music",
-					"experimental" => "Experimental",
-					"documentary" => "Documentary",
-					"identsandanimatedlogos" => "Idents and Animated Logos",
-					"industry" => "Industry",
-					"instructionals" => "Instructionals",
-					"narrative" => "Narrative",
-					"personal" => "Personal"
-				]
-			],
-			"live" => [
-				"display" => "Live events",
-				"option" => [
-					"any" => "Any",
-					"yes" => "Live now" // &filter_live=now
-				]
-			],
-			"hdr" => [
-				"display" => "HDR", // &filter_hdr=
-				"option" => [
-					"any" => "Any",
-					"hdr" => "Any HDR",
-					"dolby_vision" => "Dolby Vision",
-					"hdr10" => "HDR10",
-					"hdr10+" => "HDR10+"
-				]
-			],
-			"vimeo_360" => [
-				"display" => "Vimeo 360°", // &filter_vimeo_360
-				"option" => [
-					"any" => "Any",
-					"spatial" => "Spatial",
-					"360" => "360°"
-				]
-			],
-			"price" => [ // &filter_price=
-				"display" => "Price",
-				"option" => [
-					"any" => "Any price",
-					"free" => "Free",
-					"paid" => "Paid"
-				]
-			],
-			"collection" => [
-				"display" => "Vimeo collections",
-				"option" => [
-					"any" => "Any collection",
-					"staff_pick" => "Staff picks" // &filter_staffpicked=true
-				]
-			],
-			"license" => [ // &filter_license=
-				"display" => "License",
-				"option" => [
-					"any" => "Any license",
-					"by-nc-nd" => "CC BY-NC-ND",
-					"by" => "CC BY",
-					"by-nc" => "CC BY-NC",
-					"by-nc-sa" => "CC BY-NC-SA",
-					"by-nd" => "CC BY-ND",
-					"by-sa" => "CC BY-SA",
-					"cc0" => "CC0"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $jwt = false){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		if($jwt === false){
-			
-			curl_setopt(
-				$curlproc,
-				CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"Referer: https://vimeo.com/search",
-				"X-Requested-With: XMLHttpRequest",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-origin",
-				"Priority: u=4"]
-			);
-			
-		}else{
-			
-			curl_setopt(
-				$curlproc,
-				CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: application/vnd.vimeo.*+json;version=3.3",
-				"Accept-Language: en",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"Referer: https://vimeo.com/",
-				"Content-Type: application/json",
-				"Authorization: jwt $jwt",
-				"Vimeo-Page: /search/[[...slug]]",
-				"Origin: https://vimeo.com",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-site",
-				"Priority: u=4"]
-			);
-		}
-		
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function video($get){
-		
-		// parse shit
-		if($get["npt"]){
-			
-			[$npt, $proxy] =
-				$this->backend
-				->get(
-					$get["npt"],
-					"videos"
-				);
-			
-			$npt = json_decode($npt, true);
-			$pagetype = $npt["pagetype"];
-			$npt = $npt["npt"];
-			
-			$jwt = $this->get_jwt($proxy);
-			
-			try{
-				
-				$json =
-					$this->get(
-						$proxy,
-						"https://api.vimeo.com" . $npt,
-						[],
-						$jwt
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JSON");
-			}
-			
-		}else{
-			
-			$proxy = null;
-			$jwt = $this->get_jwt($proxy); // this gives us a proxy by reference
-			
-			// parse filters
-			$npt = [
-				"query" => $get["s"],
-				"page" => 1,
-				"per_page" => 24,
-				"facets" => "type"
-			];
-			
-			switch($get["display"]){
-				
-				case "video":
-					$npt["filter_type"] = "clip";
-					$npt["fields"] = "clip.name,stats.plays,clip.pictures,clip.user.name,clip.user.link,clip.user.pictures.sizes,clip.uri,clip.stats.plays,clip.duration,clip.created_time,clip.link,clip.description";
-					break;
-				
-				case "ondemand":
-					$npt["filter_type"] = "ondemand";
-					$npt["sizes"] = "296x744";
-					$npt["fields"] = "ondemand.link,ondemand.name,ondemand.pictures.sizes,ondemand.metadata.interactions.buy,ondemand.metadata.interactions.rent,ondemand.uri";
-					break;
-				
-				case "people":
-					$npt["filter_type"] = "people";
-					$npt["fetch_user_profile"] = "1";
-					$npt["fields"] = "people.name,people.location_details.formatted_address,people.metadata.public_videos.total,people.pictures.sizes,people.link,people.metadata.connections.followers.total,people.skills.name,people.skills.uri,people.background_video,people.uri";
-					break;
-				
-				case "channel":
-					$npt["filter_type"] = "channel";
-					$npt["fields"] = "channel.name,channel.metadata.connections.users.total,channel.metadata.connections.videos.total,channel.pictures.sizes,channel.link,channel.uri";
-					break;
-				
-				case "group":
-					$npt["filter_type"] = "group";
-					$npt["fields"] = "group.name,group.metadata.connections.users.total,group.metadata.connections.videos.total,group.pictures.sizes,group.link,group.uri";
-					break;
-			}
-			
-			// only apply filters if we're searching for videos
-			if($get["display"] == "video"){
-				
-				switch($get["sort"]){
-					
-					case "relevance": break; // do nothing
-					
-					case "recent":
-						$npt["sort"] = "latest";
-						$npt["direction"] = "desc";
-						break;
-					
-					case "popular":
-						$npt["sort"] = "popularity";
-						$npt["direction"] = "desc";
-						break;
-					
-					case "a_z":
-						$npt["sort"] = "alphabetical";
-						$npt["direction"] = "asc";
-						break;
-					
-					case "z_a":
-						$npt["sort"] = "alphabetical";
-						$npt["direction"] = "desc";
-						break;
-					
-					case "longest":
-						$npt["sort"] = "duration";
-						$npt["direction"] = "desc";
-						break;
-					
-					case "shortest":
-						$npt["sort"] = "duration";
-						$npt["direction"] = "asc";
-						break;
-				}
-				
-				if($get["time"] != "any"){
-					
-					$npt["filter_uploaded"] = $get["time"];
-				}
-				
-				if($get["duration"] != "any"){
-					
-					$npt["filter_duration"] = $get["duration"];
-				}
-				
-				if($get["resolution"] != "any"){
-					
-					$npt["filter_resolution"] = $get["resolution"];
-				}
-				
-				if($get["category"] != "any"){
-					
-					$npt["filter_category"] = $get["category"];
-				}
-				
-				if($get["live"] != "any"){
-					
-					$npt["filter_live"] = "now";
-				}
-				
-				if($get["hdr"] != "any"){
-					
-					$npt["filter_hdr"] = $get["hdr"];
-				}
-				
-				if($get["vimeo_360"] != "any"){
-					
-					$npt["filter_vimeo_360"] = $get["vimeo_360"];
-				}
-				
-				if($get["price"] != "any"){
-					
-					$npt["filter_price"] = $get["price"];
-				}
-				
-				if($get["collection"] == "staff_pick"){
-					
-					$npt["filter_staffpicked"] = "true";
-				}
-				
-				if($get["license"] != "any"){
-					
-					$npt["filter_license"] = $get["license"];
-				}
-			}
-			
-			$pagetype = $npt["filter_type"];
-			
-			try{
-				
-				$json =
-					$this->get(
-						$proxy,
-						"https://api.vimeo.com/search",
-						$npt,
-						$jwt
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JSON");
-			}
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to parse JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"video" => [],
-			"author" => [],
-			"livestream" => [],
-			"playlist" => [],
-			"reel" => []
-		];
-		
-		if(isset($json["error"])){
-			
-			$error = $json["error"];
-			if(isset($json["developer_message"])){
-				
-				$error .= " ({$json["developer_message"]})";
-			}
-			
-			throw new Exception("Vimeo returned an error: " . $error);
-		}
-		
-		if(!isset($json["data"])){
-			
-			throw new Exception("Vimeo did not return a data object");
-		}
-		
-		switch($pagetype){
-			
-			case "clip":
-				foreach($json["data"] as $video){
-					
-					$video = $video["clip"];
-					
-					if(isset($video["user"]["pictures"]["sizes"])){
-						
-						$avatar = $video["user"]["pictures"]["sizes"][count($video["user"]["pictures"]["sizes"]) - 1]["link"];
-					}else{
-						
-						$avatar = null;
-					}
-					
-					$out["video"][] = [
-						"title" => $video["name"],
-						"description" =>
-							$this->limitstrlen(
-								$video["description"]
-							),
-						"author" => [
-							"name" => $video["user"]["name"],
-							"url" => $video["user"]["link"],
-							"avatar" => $avatar
-						],
-						"date" => strtotime($video["created_time"]),
-						"duration" => (int)$video["duration"],
-						"views" => (int)$video["stats"]["plays"],
-						"thumb" => [
-							"ratio" => "16:9",
-							"url" => $video["pictures"]["base_link"]
-						],
-						"url" => $video["link"]
-					];
-				}
-				break;
-			
-			case "ondemand":
-				foreach($json["data"] as $video){
-					
-					$video = $video["ondemand"];
-					
-					$description = [];
-					if(isset($video["metadata"]["interactions"]["rent"]["display_price"])){
-						
-						$description[] = "Rent for " . $video["metadata"]["interactions"]["rent"]["display_price"];
-					}
-					
-					if(isset($video["metadata"]["interactions"]["buy"]["display_price"])){
-						
-						$description[] = "Buy for " . $video["metadata"]["interactions"]["buy"]["display_price"];
-					}
-					
-					$description = implode(", ", $description);
-					
-					$out["video"][] = [
-						"title" => $video["name"],
-						"description" => $description,
-						"author" => [
-							"name" => null,
-							"url" => null,
-							"avatar" => null
-						],
-						"date" => null,
-						"duration" => null,
-						"views" => null,
-						"thumb" => [
-							"ratio" => "9:16",
-							"url" => $video["pictures"]["sizes"][0]["link"]
-						],
-						"url" => $video["link"]
-					];
-				}
-				break;
-			
-			case "people":
-				foreach($json["data"] as $user){
-					
-					$user = $user["people"];
-					
-					if(
-						isset($user["pictures"]["sizes"]) &&
-						count($user["pictures"]["sizes"]) !== 0
-					){
-						
-						$thumb = [
-							"ratio" => "1:1",
-							"url" => $user["pictures"]["sizes"][count($user["pictures"]["sizes"]) - 1]["link"]
-						];
-					}else{
-						
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["author"][] = [
-						"title" => $user["name"],
-						"followers" => (int)$user["metadata"]["connections"]["followers"]["total"],
-						"description" => $user["metadata"]["public_videos"]["total"] . " videos.",
-						"thumb" => $thumb,
-						"url" => $user["link"]
-					];
-				}
-				break;
-			
-			case "channel":
-			case "group":
-				foreach($json["data"] as $channel){
-					
-					$channel = $channel[$npt["filter_type"]];
-					
-					if(
-						isset($channel["pictures"]["sizes"]) &&
-						count($channel["pictures"]["sizes"]) !== 0
-					){
-						
-						$thumb = [
-							"ratio" => "16:9",
-							"url" => $channel["pictures"]["sizes"][count($channel["pictures"]["sizes"]) - 1]["link"]
-						];
-					}else{
-						
-						$thumb = [
-							"ratio" => null,
-							"url" => null
-						];
-					}
-					
-					$out["author"][] = [
-						"title" => $channel["name"],
-						"followers" => (int)$channel["metadata"]["connections"]["users"]["total"],
-						"description" => $channel["metadata"]["connections"]["videos"]["total"] . " videos.",
-						"thumb" => $thumb,
-						"url" => $channel["link"]
-					];
-				}
-				break;
-		}
-		
-		//
-		// get next page
-		//
-		if(
-			isset($json["paging"]["next"]) &&
-			is_string($json["paging"]["next"])
-		){
-			
-			$out["npt"] =
-				$this->backend
-				->store(
-					json_encode([
-						"npt" => $json["paging"]["next"],
-						"pagetype" => $pagetype
-					]),
-					"videos",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function get_jwt(&$proxy){
-		
-		//
-		// get jwt token
-		// it's probably safe to cache this across proxies, cause the jwt doesnt contain an userID
-		// only an appID, whatever shit that is
-		// we can only cache it for 5 minutes though, otherwise vimeo cries about it
-		//
-		if($proxy === null){
-			
-			$proxy = $this->backend->get_ip();
-		}
-		
-		$jwt = apcu_fetch("vimeo_jwt");
-		
-		if($jwt === false){
-			/*
-			$html =
-				$this->get(
-					$proxy,
-					"https://vimeo.com/search",
-					[],
-					false
-				);
-			
-			$this->fuckhtml->load($html);
-			
-			$captcha =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"title"
-				);
-			
-			if(
-				count($captcha) !== 0 &&
-				$this->fuckhtml
-				->getTextContent(
-					$captcha[0]
-				) == "Vimeo / CAPTCHA Challenge"
-			){
-				
-				throw new Exception("Vimeo returned a Captcha");
-			}
-			
-			$html =
-				explode(
-					'<script id="viewer-bootstrap" type="application/json">',
-					$html,
-					2
-				);
-			
-			if(count($html) !== 2){
-				
-				throw new Exception("Failed to find JWT json");
-			}
-			
-			$jwt =
-				json_decode(
-					$this->fuckhtml
-					->extract_json(
-						$html[1]
-					),
-					true
-				);
-			
-			if($jwt === null){
-				
-				throw new Exception("Failed to decode JWT json");
-			}
-			
-			if(!isset($jwt["jwt"])){
-				
-				throw new Exception("Failed to grep JWT");
-			}
-			
-			$jwt = $jwt["jwt"];
-			*/
-			
-			try{
-				$json =
-					$this->get(
-						$proxy,
-						"https://vimeo.com/_next/jwt",
-						[],
-						false
-					);
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch JWT token");
-			}
-			
-			$this->fuckhtml->load($json);
-			
-			$captcha =
-				$this->fuckhtml
-				->getElementsByTagName(
-					"title"
-				);
-			
-			if(
-				count($captcha) !== 0 &&
-				$this->fuckhtml
-				->getTextContent(
-					$captcha[0]
-				) == "Vimeo / CAPTCHA Challenge"
-			){
-				
-				throw new Exception("Vimeo returned a Captcha");
-			}
-				
-			$json = json_decode($json, true);
-			
-			if($json === null){
-				
-				throw new Exception("The JWT object could not be decoded");
-			}
-			
-			if(!isset($json["token"])){
-				
-				throw new Exception("Vimeo did not return a JWT");
-			}
-			
-			$jwt = $json["token"];
-			
-			apcu_store("vimeo_jwt", $jwt, 300);
-		}
-		
-		return $jwt;
-	}
-	
-	private function titledots($title){
-		
-		$substr = substr($title, -3);
-		
-		if(
-			$substr == "..." ||
-			$substr == "…"
-		){
-			
-			return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0");
-		}
-		
-		return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0");
-	}
-	
-	private function limitstrlen($text){
-		
-		return
-			explode(
-				"\n",
-				wordwrap(
-					str_replace(
-						["\n\r", "\r\n", "\n", "\r"],
-						" ",
-						$text
-					),
-					300,
-					"\n"
-				),
-				2
-			)[0];
-	}
-}

scraper/vsco.php

@@ -1,256 +0,0 @@
-<?php
-
-class vsco{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("vsco");
-	}
-	
-	public function getfilters($page){
-		
-		return [];
-	}
-	
-	private function get($proxy, $url, $get = [], $bearer = null){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get_tmp = http_build_query($get);
-			$url .= "?" . $get_tmp;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		
-		if($bearer === null){
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip, deflate, br, zstd",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Upgrade-Insecure-Requests: 1",
-				"Sec-Fetch-Dest: document",
-				"Sec-Fetch-Mode: navigate",
-				"Sec-Fetch-Site: none",
-				"Sec-Fetch-User: ?1",
-				"Priority: u=0, i"]
-			);
-		}else{
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US",
-				"Accept-Encoding: gzip",
-				"Referer: https://vsco.co/search/images/" . urlencode($get["query"]),
-				"authorization: Bearer " . $bearer,
-				"content-type: application/json",
-				"x-client-build: 1",
-				"x-client-platform: web",
-				"DNT: 1",
-				"Sec-GPC: 1",
-				"Connection: keep-alive",
-				"Sec-Fetch-Dest: empty",
-				"Sec-Fetch-Mode: cors",
-				"Sec-Fetch-Site: same-origin",
-				"Priority: u=0",
-				"TE: trailers"]
-			);
-		}
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-		
-		// http1 bypass?!?!?!
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
-		
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$data, $proxy] =
-				$this->backend->get(
-					$get["npt"], "images"
-				);
-			
-			$data = json_decode($data, true);
-			
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			
-			// get bearer token
-			try{
-				
-				$html =
-					$this->get(
-						$proxy,
-						"https://vsco.co/feed"
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Failed to fetch feed page");
-			}
-			
-			preg_match(
-				'/"tkn":"([A-z0-9]+)"/',
-				$html,
-				$bearer
-			);
-			
-			if(!isset($bearer[1])){
-				
-				throw new Exception("Failed to grep bearer token");
-			}
-			
-			$data = [
-				"pagination" => [
-					"query" => $search,
-					"page" => 0,
-					"size" => 100
-				],
-				"bearer" => $bearer[1]
-			];
-		}
-		
-		try{
-			
-			$json =
-				$this->get(
-					$proxy,
-					"https://vsco.co/api/2.0/search/images",
-					$data["pagination"],
-					$data["bearer"]
-				);
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		if(!isset($json["results"])){
-			
-			throw new Exception("Failed to access results object");
-		}
-		
-		foreach($json["results"] as $image){
-			
-			$image_domain = parse_url("https://" . $image["responsive_url"], PHP_URL_HOST);
-			$thumbnail = explode($image_domain, $image["responsive_url"], 2)[1];
-			
-			if(substr($thumbnail, 0, 3) != "/1/"){
-				
-				$thumbnail =
-					preg_replace(
-						'/^\/[^\/]+/',
-						"",
-						$thumbnail
-					);
-			}
-			
-			$thumbnail = "https://img.vsco.co/cdn-cgi/image/width=480,height=360" . $thumbnail;
-			$size =
-				$this->image_ratio(
-					(int)$image["dimensions"]["width"],
-					(int)$image["dimensions"]["height"]
-				);
-			
-			$out["image"][] = [
-				"title" => $image["description"],
-				"source" => [
-					[
-						"url" => "https://" . $image["responsive_url"],
-						"width" => (int)$image["dimensions"]["width"],
-						"height" => (int)$image["dimensions"]["height"]
-					],
-					[
-						"url" => $thumbnail,
-						"width" => $size[0],
-						"height" => $size[1]
-					]
-				],
-				"url" => "https://" . $image["grid"]["domain"] . "/media/" . $image["imageId"]
-			];
-		}
-		
-		// get NPT
-		$max_page = ceil($json["total"] / 100);
-		$data["pagination"]["page"]++;
-		
-		if($max_page > $data["pagination"]["page"]){
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($data),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function image_ratio($width, $height){
-		
-		$ratio = [
-			480 / $width,
-			360 / $height
-		];
-		
-		if($ratio[0] < $ratio[1]){
-			
-			$ratio = $ratio[0];
-		}else{
-			
-			$ratio = $ratio[1];
-		}
-		
-		return [
-			floor($width * $ratio),
-			floor($height * $ratio)
-		];
-	}
-}

scraper/yandex.php

@@ -14,7 +14,7 @@ class yandex{
 		// backend included in the scraper functions
 	}
 	
-	private function get($proxy, $url, $get = [], $nsfw, $get_cookie = 1){
+	private function get($proxy, $url, $get = [], $nsfw){
 		
 		$curlproc = curl_init();
 		
@@ -25,55 +25,19 @@ class yandex{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// extract "i" cookie
-		if($get_cookie === 0){
-			
-			$cookies_tmp = [];
-			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies_tmp){
-				
-				$length = strlen($header);
-				
-				$header = explode(":", $header, 2);
-				
-				if(trim(strtolower($header[0])) == "set-cookie"){
-					
-					$cookie_tmp = explode("=", trim($header[1]), 2);
-					
-					$cookies_tmp[trim($cookie_tmp[0])] =
-						explode(";", $cookie_tmp[1], 2)[0];
-				}
-				
-				return $length;
-			});
-		}
-		
 		switch($nsfw){
 			case "yes": $nsfw = "0"; break;
 			case "maybe": $nsfw = "1"; break;
 			case "no": $nsfw = "2"; break;
 		}
 		
-		switch($get_cookie){
-			
-			case 0:
-				$cookie = "";
-				break;
-			
-			case 1:
-				$cookie = "Cookie: yp=" . (time() - 4000033) . ".szm.1:1920x1080:876x1000#" . time() . ".sp.family:" . $nsfw;
-				break;
-			
-			default:
-				$cookie = "Cookie: i=" . $get_cookie;
-		}
-		
 		$headers =
 			["User-Agent: " . config::USER_AGENT,
 			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
 			"Accept-Encoding: gzip",
 			"Accept-Language: en-US,en;q=0.5",
 			"DNT: 1",
-			$cookie,
+			"Cookie: yp=1716337604.sp.family%3A{$nsfw}#1685406411.szm.1:1920x1080:1920x999",
 			"Referer: https://yandex.com/images/search",
 			"Connection: keep-alive",
 			"Upgrade-Insecure-Requests: 1",
@@ -95,17 +59,6 @@ class yandex{
 		
 		$data = curl_exec($curlproc);
 		
-		if($get_cookie === 0){
-			
-			if(isset($cookies_tmp["i"])){
-				
-				return $cookies_tmp["i"];
-			}else{
-				
-				throw new Exception("Failed to get Yandex clearance cookie");
-			}
-		}
-		
 		if(curl_errno($curlproc)){
 			
 			throw new Exception(curl_error($curlproc));
@@ -264,23 +217,6 @@ class yandex{
 		// https://yandex.com/search/site/?text=minecraft&web=1&frame=1&v=2.0&searchid=3131712
 		// &within=777&from_day=26&from_month=8&from_year=2023&to_day=26&to_month=8&to_year=2023
 		
-		// get clearance cookie
-		if(($cookie = apcu_fetch("yandexweb_cookie")) === false){
-			
-			$proxy = $this->backend->get_ip();
-			
-			$cookie =
-				$this->get(
-					$proxy,
-					"https://yandex.ru/support2/smart-captcha/ru/",
-					[],
-					false,
-					0
-				);
-			
-			apcu_store("yandexweb_cookie", $cookie);
-		}
-		
 		if($get["npt"]){
 			
 			[$npt, $proxy] = $this->backend->get($get["npt"], "web");
@@ -290,8 +226,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com" . $npt,
 					[],
-					"yes",
+					"yes"
-					$cookie
 				);
 		}else{
 			
@@ -301,7 +236,7 @@ class yandex{
 				throw new Exception("Search term is empty!");
 			}
 			
-			$proxy = !isset($proxy) ? $this->backend->get_ip() : $proxy;
+			$proxy = $this->backend->get_ip();
 			$lang = $get["lang"];
 			$older = $get["older"];
 			$newer = $get["newer"];
@@ -348,8 +283,7 @@ class yandex{
 						$proxy,
 						"https://yandex.com/search/site/",
 						$params,
-						"yes",
+						"yes"
-						$cookie
 					);
 			}catch(Exception $error){
 				
@@ -380,19 +314,6 @@ class yandex{
 		
 		$this->fuckhtml->load($html);
 		
-		// Scrape page blocked error
-		$title =
-			$this->fuckhtml
-			->getElementsByTagName("title");
-		
-		if(
-			count($title) !== 0 &&
-			$title[0]["innerHTML"] == "403"
-		){
-			
-			throw new Exception("Yandex blocked this proxy or 4get instance.");
-		}
-		
 		// get nextpage
 		$npt =
 			$this->fuckhtml
@@ -747,6 +668,7 @@ class yandex{
 		foreach($json["blocks"] as $block){
 			
 			$html .= $block["html"];
+			
 			// get next page
 			if(
 				isset($block["params"]["nextPageUrl"]) &&
@@ -868,71 +790,123 @@ class yandex{
 		
 		if($get["npt"]){
 			
-			[$get, $proxy] =
+			[$params, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"video"
 				);
 			
-			$get = json_decode($get, true);
+			$params = json_decode($params, true);
+			
+			$nsfw = $params["nsfw"];
+			unset($params["nsfw"]);
 		}else{
 			
-			if(strlen($get["s"]) === 0){
+			$search = $get["s"];
+			if(strlen($search) === 0){
 				
 				throw new Exception("Search term is empty!");
 			}
 			
 			$proxy = $this->backend->get_ip();
-		}
+			$nsfw = $get["nsfw"];
+			$time = $get["time"];
+			$duration = $get["duration"];
 			
-		// https://yandex.com/video/search?text=skycamefalling&from=tabbar&format=json&ncrnd=7271&p=0&parent-reqid=&request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]}&serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL&yu=3091577281773194415&tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
 			// https://yandex.com/video/search
-		// ?text=skycamefalling
+			// ?tmpl_version=releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63
-		// &from=tabbar
 			// &format=json
-		// &ncrnd=7271
+			// &request=
-		// &p=0
+			// {
-		// &parent-reqid=
+			//		"blocks":[
-		// &request={%22blocks%22%3A[{%22block%22%3A%22video-app%22%2C%22params%22%3A{}}]} {"blocks":[{"block":"video-app","params":{}}]}
+			//			{"block":"extra-content","params":{},"version":2},
-		// &serpid=1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL
+			//			{"block":"i-global__params:ajax","params":{},"version":2},
-		// &yu=3091577281773194415
+			//			{"block":"search2:ajax","params":{},"version":2},
-		// &tmpl_version=releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a
+			//			{"block":"vital-incut","params":{},"version":2},
+			//			{"block":"content_type_search","params":{},"version":2},
+			//			{"block":"serp-controller","params":{},"version":2},
+			//			{"block":"cookies_ajax","params":{},"version":2}
+			//		],
+			//		"metadata":{
+			//			"bundles":{"lb":"^G]!q<X120"},
+			//			"assets":{"las":"react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"},
+			//			"extraContent":{"names":["i-react-ajax-adapter"]}
+			//		}
+			// }
+			// &yu=4861394161661655015
+			// &from=tabbar
+			// &reqid=1693106278500184-6825210746979814879-balancer-l7leveler-kubr-yp-sas-7-BAL-4237
+			// &suggest_reqid=486139416166165501562797413447032
+			// &text=minecraft
 			
 			$params = [
-			"text" => $get["s"],
+				"tmpl_version" => "releases/frontend/video/v1.1168.0#8d942de0f4ebc4eb6b8f3c24ffbd1f8dbc5bbe63",
-			"from" => "tabbar",
 				"format" => "json",
-			"ncrnd" => 7271,
+				"request" => json_encode([
-			"p" => 0,
-			"parent-reqid" => "",
-			"request" => json_encode((object)[
 					"blocks" => [
 						(object)[
-						"block" => "video-app",
+							"block" => "extra-content",
-						"params" => (object)[]
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "i-global__params:ajax",
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "search2:ajax",
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "vital-incut",
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "content_type_search",
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "serp-controller",
+							"params" => (object)[],
+							"version" => 2
+						],
+						(object)[
+							"block" => "cookies_ajax",
+							"params" => (object)[],
+							"version" => 2
+						]
+					],
+					"metadata" => (object)[
+						"bundles" => (object)[
+							"lb" => "^G]!q<X120"
+						],
+						"assets" => (object)[
+							"las" => "react-with-dom=1;185.0=1;73.0=1;145.0=1;5a502a.0=1;32c342.0=1;b84ac8.0=1"
+						],
+						"extraContent" => (object)[
+							"names" => [
+								"i-react-ajax-adapter"
+							]
 						]
 					]
 				]),
-			"serpid" => "1777751040971457-16832445014469941403-balancer-l7leveler-kubr-yp-klg-151-BAL",
+				"text" => $search
-			"yu" => 3091577281773194415,
-			"tmpl_version" => "releases-frontend-video-v1.1816.0__3bdc24e10a8a138a1194877428e220a3ca0dbc5a"
 			];
 			
-		if(isset($get["p"])){
+			if($duration != "any"){
 				
-			$params["p"] = $get["p"];
+				$params["duration"] = $duration;
 			}
 			
-		if($get["duration"] != "any"){
+			if($time != "any"){
 				
-			$params["duration"] = $get["duration"];
+				$params["within"] = $time;
 			}
-		
-		if($get["time"] != "any"){
-			
-			$params["within"] = $get["time"];
 		}
-		
 		/*
 		$handle = fopen("scraper/yandex-video.json", "r");
 		$json = fread($handle, filesize("scraper/yandex-video.json"));
@@ -944,7 +918,7 @@ class yandex{
 					$proxy,
 					"https://yandex.com/video/search",
 					$params,
-					$get["nsfw"],
+					$nsfw,
 					"yandex_v"
 				);
 		}catch(Exception $error){
@@ -959,7 +933,7 @@ class yandex{
 			throw new Exception("Could not parse JSON");
 		}
 		
-		if(!isset($json["results"]["clips"]["items"])){
+		if(!isset($json["blocks"])){
 			
 			throw new Exception("Yandex blocked this 4get instance. Please try again in 7~ minutes.");
 		}
@@ -974,120 +948,209 @@ class yandex{
 			"reel" => []
 		];
 		
-		foreach($json["results"]["clips"]["items"] as $k => $data){
+		$html = null;
+		foreach($json["blocks"] as $block){
 			
-			if(isset($data["preview"]["posterSrc"])){
+			if(isset($block["html"])){
 				
-				$poster = $data["preview"]["posterSrc"];
+				$html .= $block["html"];
-				
+			}
-				if(
-					preg_match(
-						'/^\/\//',
-						$data["preview"]["posterSrc"]
-					)
-				){
-					
-					$poster = "https:" . $poster;
 		}
 		
-				$thumb = [
+		$this->fuckhtml->load($html);
-					"ratio" => "16:9",
-					"url" => $poster
-				];
-			}else{
 		
-				$thumb = [
+		$div =
-					"ratio" => null,
+			$this->fuckhtml
-					"url" => null
+			->getElementsByTagName("div");
-				];
-			}
 		
-			$out["video"][] = [
+		/*
-				"title" => $data["relatedParams"]["text"],
+			Get nextpage
-				"description" => $this->titledots($data["description"]),
+		*/
-				"author" => [
+		$npt =
-					"name" =>
+			$this->fuckhtml
-						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"]) ?
+			->getElementsByClassName(
-						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["name"] : null,
+				"more more_direction_next i-bem",
-					"url" => 
+				$div
-						isset($json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"]) ?
+			);
-						$json["results"]["clips"]["dups"][$k]["host"]["secondPart"]["origUrl"] : null,
-					"avatar" => null
-				],
-				"date" =>
-					isset($json["results"]["clips"]["dups"][$k]["date"]) ?
-					strtotime($json["results"]["clips"]["dups"][$k]["date"]) : null,
-				"duration" =>
-					isset($json["results"]["clips"]["dups"][$k]["duration"]["value"]) ?
-					(int)$json["results"]["clips"]["dups"][$k]["duration"]["value"] : null,
-				"views" =>
-					isset($json["results"]["clips"]["dups"][$k]["views"]["text"]) ?
-					$this->parseviews($json["results"]["clips"]["dups"][$k]["views"]["text"]) : null,
-				"thumb" => $thumb,
-				"url" =>
-					preg_replace(
-						'/^http:\/\//',
-						"https://",
-						$data["relatedParams"]["related_url"]
-					)
-			];
-		}
 		
-		// get npt
+		if(count($npt) !== 0){
-		if($json["results"]["search"]["hasNextPage"]){
-			
-			$get["p"] = (int)$json["results"]["search"]["currentPage"] + 1;
 			
+			$params["p"] = "1";
+			$params["nsfw"] = $nsfw;
 			$out["npt"] =
 				$this->backend->store(
-					json_encode($get),
+					json_encode($params),
 					"video",
 					$proxy
 				);
 		}
 		
+		$items =
+			$this->fuckhtml
+			->getElementsByClassName(
+				"serp-item",
+				$div
+			);
+		
+		foreach($items as $item){
+			
+			$data =
+				json_decode(
+					$this->fuckhtml
+					->getTextContent(
+						$item["attributes"]["data-video"]
+					),
+					true
+				);
+			
+			$this->fuckhtml->load($item);
+			
+			$thumb =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"thumb-image__image",
+					"img"
+				);
+			
+			$c = 1;
+			if(count($thumb) === 0){
+				
+				$thumb = [
+					"url" => null,
+					"ratio" => null
+				];
+			}else{
+				
+				$thumb = [
+					"url" =>
+						str_replace(
+							"//",
+							"https://",
+							$this->fuckhtml
+							->getTextContent(
+								$thumb
+								[0]
+								["attributes"]
+								["src"]
+							),
+							$c
+						),
+					"ratio" => "16:9"
+				];
+			}
+			
+			$smallinfos =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"serp-item__sitelinks-item",
+					"div"
+				);
+			
+			$date = null;
+			$views = null;
+			$first = true;
+			
+			foreach($smallinfos as $info){
+				
+				if($first){
+					
+					$first = false;
+					continue;
+				}
+				
+				$info =
+					$this->fuckhtml
+					->getTextContent(
+						$info
+					);
+				
+				if($temp_date = strtotime($info)){
+					
+					$date = $temp_date;
+				}else{
+					
+					$views = $this->parseviews($info);
+				}
+			}
+			
+			$description =
+				$this->fuckhtml
+				->getElementsByClassName(
+					"serp-item__text serp-item__text_visibleText_always",
+					"div"
+				);
+			
+			if(count($description) === 0){
+				
+				$description = null;
+			}else{
+				
+				$description =
+					$this->titledots(
+						$this->fuckhtml
+						->getTextContent(
+							$description[0]
+						)
+					);
+			}
+			
+			$out["video"][] = [
+				"title" =>
+					$this->fuckhtml
+					->getTextContent(
+						$this->titledots(
+							$data["title"]
+						)
+					),
+				"description" => $description,
+				"author" => [
+					"name" => null,
+					"url" => null,
+					"avatar" => null
+				],
+				"date" => $date,
+				"duration" =>
+					(int)$data
+					["counters"]
+					["toHostingLoaded"]
+					["stredParams"]
+					["duration"],
+				"views" => $views,
+				"thumb" => $thumb,
+				"url" =>
+					str_replace(
+						"http://",
+						"https://",
+						$this->fuckhtml
+						->getTextContent(
+							$data["counters"]
+							["toHostingLoaded"]
+							["postfix"]
+							["href"]
+						),
+						$c
+					)
+			];
+		}
+		
 		return $out;
 	}
 	
-	private function parseviews($number){
+	private function parseviews($text){
 		
-		// decimal should always be 1 number long
+		$text = explode(" ", $text);
-		$number = explode(" ", $number, 2);
-		$number = $number[0];
 		
-		$unit = strtolower($number[strlen($number) - 1]);
+		$num = (float)$text[0];
+		$mod = $text[1];
 		
-		$tmp = explode(".", $number, 2);
+		switch($mod){
-		$number = (int)$number;
 			
-		if(count($tmp) === 2){
+			case "bln.": $num = $num * 1000000000; break;
-			
+			case "mln.": $num = $num * 1000000; break;
-			$decimal = (int)$tmp[1];
+			case "thsd.": $num = $num * 1000; break;
-		}else{
-			
-			$decimal = 0;
 		}
 		
-		switch($unit){
+		return $num;
-			
-			case "k":
-				$exponant = 1000;
-				break;
-			
-			case "m":
-				$exponant = 1000000;
-				break;
-			
-			case "b";
-				$exponant = 1000000000;
-				break;
-			
-			default:
-				$exponant = 1;
-				break;
-		}
-		
-		return ($number * $exponant) + ($decimal * ($exponant / 10));
 	}
 	
 	private function titledots($title){

scraper/yep.php

@@ -14,209 +14,234 @@ class yep{
 	public function getfilters($page){
 		
 		return [
-			"lang" => [
+			"country" => [
-				"display" => "Language",
+				"display" => "Country",
 				"option" => [
-					"any" => "Any language",
+					"all" => "All regions",
-					"aa" => "Afar",
+					"af" => "Afghanistan",
-					"ab" => "Abkhazian",
+					"al" => "Albania",
-					"ae" => "Avestan",
+					"dz" => "Algeria",
-					"af" => "Afrikaans",
+					"as" => "American Samoa",
-					"ak" => "Akan",
+					"ad" => "Andorra",
-					"am" => "Amharic",
+					"ao" => "Angola",
-					"an" => "Aragonese",
+					"ai" => "Anguilla",
-					"ar" => "Arabic",
+					"ag" => "Antigua and Barbuda",
-					"as" => "Assamese",
+					"ar" => "Argentina",
-					"av" => "Avaric",
+					"am" => "Armenia",
-					"ay" => "Aymara",
+					"aw" => "Aruba",
-					"az" => "Azerbaijani",
+					"au" => "Australia",
-					"ba" => "Bashkir",
+					"at" => "Austria",
-					"be" => "Belarusian",
+					"az" => "Azerbaijan",
-					"bg" => "Bulgarian",
+					"bs" => "Bahamas",
-					"bh" => "Bihari",
+					"bh" => "Bahrain",
-					"bi" => "Bislama",
+					"bd" => "Bangladesh",
-					"bm" => "Bambara",
+					"bb" => "Barbados",
-					"bn" => "Bengali",
+					"by" => "Belarus",
-					"bo" => "Tibetan",
+					"be" => "Belgium",
-					"br" => "Breton",
+					"bz" => "Belize",
-					"bs" => "Bosnian",
+					"bj" => "Benin",
-					"ca" => "Catalan",
+					"bt" => "Bhutan",
-					"ce" => "Chechen",
+					"bo" => "Bolivia",
-					"ch" => "Chamorro",
+					"ba" => "Bosnia and Herzegovina",
-					"co" => "Corsican",
+					"bw" => "Botswana",
-					"cr" => "Cree",
+					"br" => "Brazil",
-					"cs" => "Czech",
+					"bn" => "Brunei Darussalam",
-					"cu" => "Church Slavic",
+					"bg" => "Bulgaria",
-					"cv" => "Chuvash",
+					"bf" => "Burkina Faso",
-					"cy" => "Welsh",
+					"bi" => "Burundi",
-					"da" => "Danish",
+					"cv" => "Cabo Verde",
-					"de" => "German",
+					"kh" => "Cambodia",
-					"dv" => "Divehi",
+					"cm" => "Cameroon",
-					"dz" => "Dzongkha",
+					"ca" => "Canada",
-					"ee" => "Ewe",
+					"ky" => "Cayman Islands",
-					"el" => "Greek",
+					"cf" => "Central African Republic",
-					"en" => "English",
+					"td" => "Chad",
-					"eo" => "Esperanto",
+					"cl" => "Chile",
-					"es" => "Spanish",
+					"cn" => "China",
-					"et" => "Estonian",
+					"co" => "Colombia",
-					"eu" => "Basque",
+					"cg" => "Congo",
-					"fa" => "Persian",
+					"cd" => "Congo, Democratic Republic",
-					"ff" => "Fulah",
+					"ck" => "Cook Islands",
-					"fi" => "Finnish",
+					"cr" => "Costa Rica",
-					"fj" => "Fijian",
+					"hr" => "Croatia",
-					"fo" => "Faroese",
+					"cu" => "Cuba",
-					"fr" => "French",
+					"cy" => "Cyprus",
-					"fy" => "Western Frisian",
+					"cz" => "Czechia",
-					"ga" => "Irish",
+					"ci" => "Côte d'Ivoire",
-					"gd" => "Scottish Gaelic",
+					"dk" => "Denmark",
-					"gl" => "Galician",
+					"dj" => "Djibouti",
-					"gn" => "Guarani",
+					"dm" => "Dominica",
-					"gu" => "Gujarati",
+					"do" => "Dominican Republic",
-					"gv" => "Manx",
+					"ec" => "Ecuador",
-					"ha" => "Hausa",
+					"eg" => "Egypt",
-					"he" => "Hebrew",
+					"sv" => "El Salvador",
-					"hi" => "Hindi",
+					"gq" => "Equatorial Guinea",
-					"ho" => "Hiri Motu",
+					"ee" => "Estonia",
-					"hr" => "Croatian",
+					"et" => "Ethiopia",
-					"ht" => "Haitian",
+					"fo" => "Faroe Islands",
-					"hu" => "Hungarian",
+					"fj" => "Fiji",
-					"hy" => "Armenian",
+					"fi" => "Finland",
-					"hz" => "Herero",
+					"fr" => "France",
-					"ia" => "Interlingua",
+					"gf" => "French Guiana",
-					"id" => "Indonesian",
+					"pf" => "French Polynesia",
-					"ie" => "Interlingue",
+					"ga" => "Gabon",
-					"ig" => "Igbo",
+					"gm" => "Gambia",
-					"ii" => "Sichuan Yi",
+					"ge" => "Georgia",
-					"ik" => "Inupiaq",
+					"de" => "Germany",
-					"io" => "Ido",
+					"gh" => "Ghana",
-					"is" => "Icelandic",
+					"gi" => "Gibraltar",
-					"it" => "Italian",
+					"gr" => "Greece",
-					"iu" => "Inuktitut",
+					"gl" => "Greenland",
-					"ja" => "Japanese",
+					"gd" => "Grenada",
-					"jv" => "Javanese",
+					"gp" => "Guadeloupe",
-					"ka" => "Georgian",
+					"gu" => "Guam",
-					"kg" => "Kongo",
+					"gt" => "Guatemala",
-					"ki" => "Kikuyu",
+					"gg" => "Guernsey",
-					"kj" => "Kuanyama",
+					"gn" => "Guinea",
-					"kk" => "Kazakh",
+					"gy" => "Guyana",
-					"kl" => "Kalaallisut",
+					"ht" => "Haiti",
-					"km" => "Central Khmer",
+					"hn" => "Honduras",
-					"kn" => "Kannada",
+					"hk" => "Hong Kong",
-					"ko" => "Korean",
+					"hu" => "Hungary",
-					"kr" => "Kanuri",
+					"is" => "Iceland",
-					"ks" => "Kashmiri",
+					"in" => "India",
-					"ku" => "Kurdish",
+					"id" => "Indonesia",
-					"kv" => "Komi",
+					"iq" => "Iraq",
-					"kw" => "Cornish",
+					"ie" => "Ireland",
-					"ky" => "Kyrgyz",
+					"im" => "Isle of Man",
-					"la" => "Latin",
+					"il" => "Israel",
-					"lb" => "Luxembourgish",
+					"it" => "Italy",
-					"lg" => "Ganda",
+					"jm" => "Jamaica",
-					"li" => "Limburgish",
+					"jp" => "Japan",
-					"ln" => "Lingala",
+					"je" => "Jersey",
-					"lo" => "Lao",
+					"jo" => "Jordan",
-					"lt" => "Lithuanian",
+					"kz" => "Kazakhstan",
-					"lu" => "Luba-Katanga",
+					"ke" => "Kenya",
-					"lv" => "Latvian",
+					"ki" => "Kiribati",
-					"mg" => "Malagasy",
+					"kw" => "Kuwait",
-					"mh" => "Marshallese",
+					"kg" => "Kyrgyzstan",
-					"mi" => "Maori",
+					"la" => "Lao People's Democratic Republic",
-					"mk" => "Macedonian",
+					"lv" => "Latvia",
-					"ml" => "Malayalam",
+					"lb" => "Lebanon",
-					"mn" => "Mongolian",
+					"ls" => "Lesotho",
-					"mr" => "Marathi",
+					"ly" => "Libya",
-					"ms" => "Malay",
+					"li" => "Liechtenstein",
-					"mt" => "Maltese",
+					"lt" => "Lithuania",
-					"my" => "Burmese",
+					"lu" => "Luxembourg",
-					"na" => "Nauru",
+					"mk" => "Macedonia",
-					"nb" => "Norwegian Bokmål",
+					"mg" => "Madagascar",
-					"nd" => "North Ndebele",
+					"mw" => "Malawi",
-					"ne" => "Nepali",
+					"my" => "Malaysia",
-					"ng" => "Ndonga",
+					"mv" => "Maldives",
-					"nl" => "Dutch",
+					"ml" => "Mali",
-					"nn" => "Norwegian Nynorsk",
+					"mt" => "Malta",
-					"no" => "Norwegian",
+					"mq" => "Martinique",
-					"nr" => "South Ndebele",
+					"mr" => "Mauritania",
-					"nv" => "Navajo",
+					"mu" => "Mauritius",
-					"ny" => "Chichewa",
+					"yt" => "Mayotte",
-					"oc" => "Occitan",
+					"mx" => "Mexico",
-					"oj" => "Ojibwa",
+					"fm" => "Micronesia, Federated States of",
-					"om" => "Oromo",
+					"md" => "Moldova",
-					"or" => "Oriya",
+					"mc" => "Monaco",
-					"os" => "Ossetian",
+					"mn" => "Mongolia",
-					"pa" => "Punjabi",
+					"me" => "Montenegro",
-					"pi" => "Pali",
+					"ms" => "Montserrat",
-					"pl" => "Polish",
+					"ma" => "Morocco",
-					"ps" => "Pashto",
+					"mz" => "Mozambique",
-					"pt" => "Portuguese",
+					"mm" => "Myanmar",
-					"qu" => "Quechua",
+					"na" => "Namibia",
-					"rm" => "Romansh",
+					"nr" => "Nauru",
-					"rn" => "Rundi",
+					"np" => "Nepal",
-					"ro" => "Romanian",
+					"nl" => "Netherlands",
-					"ru" => "Russian",
+					"nc" => "New Caledonia",
-					"rw" => "Kinyarwanda",
+					"nz" => "New Zealand",
-					"sa" => "Sanskrit",
+					"ni" => "Nicaragua",
-					"sc" => "Sardinian",
+					"ne" => "Niger",
-					"sd" => "Sindhi",
+					"ng" => "Nigeria",
-					"se" => "Northern Sami",
+					"nu" => "Niue",
-					"sg" => "Sango",
+					"no" => "Norway",
-					"si" => "Sinhala",
+					"om" => "Oman",
-					"sk" => "Slovak",
+					"pk" => "Pakistan",
-					"sl" => "Slovenian",
+					"ps" => "Palestine, State of",
-					"sm" => "Samoan",
+					"pa" => "Panama",
-					"sn" => "Shona",
+					"pg" => "Papua New Guinea",
-					"so" => "Somali",
+					"py" => "Paraguay",
-					"sq" => "Albanian",
+					"pe" => "Peru",
-					"sr" => "Serbian",
+					"ph" => "Philippines",
-					"ss" => "Swati",
+					"pn" => "Pitcairn",
-					"st" => "Southern Sotho",
+					"pl" => "Poland",
-					"su" => "Sundanese",
+					"pt" => "Portugal",
-					"sv" => "Swedish",
+					"pr" => "Puerto Rico",
-					"sw" => "Swahili",
+					"qa" => "Qatar",
-					"ta" => "Tamil",
+					"ro" => "Romania",
-					"te" => "Telugu",
+					"ru" => "Russian Federation",
-					"tg" => "Tajik",
+					"rw" => "Rwanda",
-					"th" => "Thai",
+					"re" => "Réunion",
-					"ti" => "Tigrinya",
+					"sh" => "Saint Helena",
-					"tk" => "Turkmen",
+					"kn" => "Saint Kitts and Nevis",
-					"tl" => "Tagalog",
+					"lc" => "Saint Lucia",
-					"tn" => "Tswana",
+					"vc" => "Saint Vincent and the Grenadines",
+					"ws" => "Samoa",
+					"sm" => "San Marino",
+					"st" => "Sao Tome and Principe",
+					"sa" => "Saudi Arabia",
+					"sn" => "Senegal",
+					"rs" => "Serbia",
+					"sc" => "Seychelles",
+					"sl" => "Sierra Leone",
+					"sg" => "Singapore",
+					"sk" => "Slovakia",
+					"si" => "Slovenia",
+					"sb" => "Solomon Islands",
+					"so" => "Somalia",
+					"kr" => "Sourth Korea",
+					"za" => "South Africa",
+					"es" => "Spain",
+					"lk" => "Sri Lanka",
+					"sr" => "Suriname",
+					"se" => "Sweden",
+					"ch" => "Switzerland",
+					"tw" => "Taiwan",
+					"tj" => "Tajikistan",
+					"tz" => "Tanzania",
+					"th" => "Thailand",
+					"tl" => "Timor-Leste",
+					"tg" => "Togo",
+					"tk" => "Tokelau",
 					"to" => "Tonga",
-					"tr" => "Turkish",
+					"tt" => "Trinidad and Tobago",
-					"ts" => "Tsonga",
+					"tn" => "Tunisia",
-					"tt" => "Tatar",
+					"tr" => "Turkey",
-					"tw" => "Twi",
+					"tm" => "Turkmenistan",
-					"ty" => "Tahitian",
+					"ug" => "Uganda",
-					"ug" => "Uyghur",
+					"ua" => "Ukraine",
-					"uk" => "Ukrainian",
+					"ae" => "United Arab Emirates",
-					"ur" => "Urdu",
+					"gb" => "United Kingdom",
-					"uz" => "Uzbek",
+					"us" => "United States",
-					"ve" => "Venda",
+					"uy" => "Uruguay",
-					"vi" => "Vietnamese",
+					"uz" => "Uzbekistan",
-					"vo" => "Volapük",
+					"vu" => "Vanuatu",
-					"wa" => "Walloon",
+					"ve" => "Venezuela",
-					"wo" => "Wolof",
+					"vn" => "Vietnam",
-					"xh" => "Xhosa",
+					"vg" => "Virgin Islands, British",
-					"yi" => "Yiddish",
+					"vi" => "Virgin Islands, U.S.",
-					"yo" => "Yoruba",
+					"ye" => "Yemen",
-					"za" => "Zhuang",
+					"zm" => "Zambia",
-					"zh" => "Chinese",
+					"zw" => "Zimbabwe"
-					"zh-cn" => "Chinese (Simplified)",
-					"zh-tw" => "Chinese (Traditional)",
-					"zu" => "Zulu"
 				]
 			],
 			"nsfw" => [
 				"display" => "NSFW",
 				"option" => [
 					"yes" => "Yes",
+					"maybe" => "Maybe",
 					"no" => "No"
 				]
 			]
 		];
 	}
 	
-	private function get($proxy, $url, $get = [], $use_api = false, $post_data = null, $bearer = null){
+	private function get($proxy, $url, $get = []){
 		
 		$curlproc = curl_init();
 		
@@ -227,29 +252,21 @@ class yep{
 		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
-		// use http1 lol
+		// use http2
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_1_1);
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		// set ciphers
-		
+		curl_setopt(
-		if($use_api){
+			$curlproc,
-			
+			CURLOPT_SSL_CIPHER_LIST,
-			$post_data = json_encode($post_data);
+			"aes_128_gcm_sha_256,chacha20_poly1305_sha_256,aes_256_gcm_sha_384,ecdhe_ecdsa_aes_128_gcm_sha_256,ecdhe_rsa_aes_128_gcm_sha_256,ecdhe_ecdsa_chacha20_poly1305_sha_256,ecdhe_rsa_chacha20_poly1305_sha_256,ecdhe_ecdsa_aes_256_gcm_sha_384,ecdhe_rsa_aes_256_gcm_sha_384,ecdhe_ecdsa_aes_256_sha,ecdhe_ecdsa_aes_128_sha,ecdhe_rsa_aes_128_sha,ecdhe_rsa_aes_256_sha,rsa_aes_128_gcm_sha_256,rsa_aes_256_gcm_sha_384,rsa_aes_128_sha,rsa_aes_256_sha"
-			
-			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-				["Content-Type: application/json",
-				"Authorization: Bearer $bearer",
-				"Content-Length: " . strlen($post_data)]
 		);
 		
-			curl_setopt($curlproc, CURLOPT_POST, true);
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-			curl_setopt($curlproc, CURLOPT_POSTFIELDS, $post_data);
-		}else{
-			
 		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 			["User-Agent: " . config::USER_AGENT,
 			"Accept: */*",
-				"Accept-Language: en-US,en;q=0.9",
+			"Accept-Language: en-US,en;q=0.5",
 			"Accept-Encoding: gzip, deflate, br, zstd",
 			"Referer: https://yep.com/",
 			"Origin: https://yep.com",
@@ -259,11 +276,8 @@ class yep{
 			"Sec-Fetch-Mode: cors",
 			"Sec-Fetch-Site: same-site",
 			"Priority: u=4",
-				"Pragma: no-cache",
-				"Cache-Control: no-cache",
 			"TE: trailers"]
 		);
-		}
 		
 		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
 		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
@@ -288,17 +302,22 @@ class yep{
 	
 	public function web($get){
 		
-		if(config::YEP_USE_API){
-			
-			return $this->web_api($get);
-		}
-		
 		$search = $get["s"];
 		if(strlen($search) === 0){
 			
 			throw new Exception("Search term is empty!");
 		}
 		
+		$country = $get["country"];
+		$nsfw = $get["nsfw"];
+		
+		switch($nsfw){
+			
+			case "yes": $nsfw = "off"; break;
+			case "maybe": $nsfw = "moderate"; break;
+			case "no": $nsfw = "strict"; break;
+		}
+		
 		$out = [
 			"status" => "ok",
 			"spelling" => [
@@ -315,25 +334,23 @@ class yep{
 			"related" => []
 		];
 		
-		// parse filters
-		$filters = [
-			"limit" => 100, // wwwwwwwwwwwwwww
-			"query" => $search,
-		];
-		
-		if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
-		if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
-		
 		try{
 			
-			// https://api.yep.com/search?limit=20&query=asmr
+			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
 			$json =
 				$this->get(
 					$this->backend->get_ip(),
-					"https://api.yep.com/search",
+					"https://api.yep.com/fs/2/search",
-					$filters
+					[
+						"client" => "web",
+						"gl" => $country == "all" ? $country : strtoupper($country),
+						"limit" => "99999",
+						"no_correct" => "false",
+						"q" => $search,
+						"safeSearch" => $nsfw,
+						"type" => "web"
+					]
 				);
-			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
@@ -397,7 +414,7 @@ class yep{
 									)
 								),
 							"url" => $item["url"],
-							"date" => null,
+							"date" => strtotime($item["first_seen"]),
 							"type" => "web",
 							"thumb" => [
 								"url" => null,
@@ -411,11 +428,83 @@ class yep{
 			}
 		}
 		
+		if(isset($json[1]["featured_news"])){
+			
+			foreach($json[1]["featured_news"] as $news){
+				
+				$out["news"][] = [
+					"title" => $news["title"],
+					"description" =>
+						$this->titledots(
+							strip_tags(
+								html_entity_decode(
+									$news["snippet"]
+								)
+							)
+						),
+					"date" => strtotime($news["first_seen"]),
+					"thumb" =>
+						isset($news["img"]) ?
+						[
+							"url" => $this->unshiturl($news["img"]),
+							"ratio" => "16:9"
+						] :
+						[
+							"url" => null,
+							"ratio" => null
+						],
+					"url" => $news["url"]
+				];
+			}
+		}
+		
+		if(isset($json[1]["featured_images"])){
+			
+			foreach($json[1]["featured_images"] as $image){
+				
+				if(
+					$image["width"] !== 0 &&
+					$image["height"] !== 0
+				){
+					
+					$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
+					$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
+					
+					$width = $image["width"];
+					$height = $image["height"];
+				}else{
+					
+					$thumb_width = null;
+					$thumb_height = null;
+					$width = null;
+					$height = null;
+				}
+				
+				$out["image"][] = [
+					"title" => $image["title"],
+					"source" => [
+						[
+							"url" => $image["image_id"],
+							"width" => $width,
+							"height" => $height
+						],
+						[
+							"url" => $image["src"],
+							"width" => $thumb_width,
+							"height" => $thumb_height
+						]
+					],
+					"url" => $image["host_page"]
+				];
+			}
+		}
+		
 		return $out;
 	}
 	
 	
-	private function web_api($get){
+	
+	public function image($get){
 		
 		$search = $get["s"];
 		if(strlen($search) === 0){
@@ -423,53 +512,142 @@ class yep{
 			throw new Exception("Search term is empty!");
 		}
 		
+		$country = $get["country"];
+		$nsfw = $get["nsfw"];
+		
+		switch($nsfw){
+			
+			case "yes": $nsfw = "off"; break;
+			case "maybe": $nsfw = "moderate"; break;
+			case "no": $nsfw = "strict"; break;
+		}
+		
 		$out = [
 			"status" => "ok",
-			"spelling" => [
-				"type" => "no_correction",
-				"using" => null,
-				"correction" => null
-			],
 			"npt" => null,
-			"answer" => [],
+			"image" => []
-			"web" => [],
-			"image" => [],
-			"video" => [],
-			"news" => [],
-			"related" => []
 		];
 		
-		// parse filters
-		$filters = [
-			"query" => $search,
-			"limit" => 99 // for some reason they dont show anything sometimes at 100 results
-		];
-		
-		if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
-		if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
-		
-		// add api key
-		$key_data = $this->backend->get_key();
-		
 		try{
 			
 			$json =
 				$this->get(
-					$this->backend->get_ip($key_data["increment"]),
+					$this->backend->get_ip(), // no nextpage!
-					"https://platform.yep.com/api/search",
+					"https://api.yep.com/fs/2/search",
-					[],
+					[
-					true,
+						"client" => "web",
-					$filters,
+						"gl" => $country == "all" ? $country : strtoupper($country),
-					$key_data["key"]
+						"no_correct" => "false",
+						"q" => $search,
+						"safeSearch" => $nsfw,
+						"type" => "images"
+					]
 				);
-			
 		}catch(Exception $error){
 			
 			throw new Exception("Failed to fetch JSON");
 		}
 		
-		// should never happen
+		$this->detect_cf($json);
-		//$this->detect_cf($json);
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		if(isset($json[1]["results"])){
+			foreach($json[1]["results"] as $item){
+				
+				if(
+					$item["width"] !== 0 &&
+					$item["height"] !== 0
+				){
+					
+					$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
+					$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
+					
+					$width = $item["width"];
+					$height = $item["height"];
+				}else{
+					
+					$thumb_width = null;
+					$thumb_height = null;
+					$width = null;
+					$height = null;
+				}
+				
+				$out["image"][] = [
+					"title" => $item["title"],
+					"source" => [
+						[
+							"url" => $item["image_id"],
+							"width" => $width,
+							"height" => $height
+						],
+						[
+							"url" => $item["src"],
+							"width" => $thumb_width,
+							"height" => $thumb_height
+						]
+					],
+					"url" => $item["host_page"]
+				];
+			}
+		}
+		
+		return $out;
+	}
+	
+	
+	public function news($get){
+		
+		$search = $get["s"];
+		if(strlen($search) === 0){
+			
+			throw new Exception("Search term is empty!");
+		}
+		
+		$country = $get["country"];
+		$nsfw = $get["nsfw"];
+		
+		switch($nsfw){
+			
+			case "yes": $nsfw = "off"; break;
+			case "maybe": $nsfw = "moderate"; break;
+			case "no": $nsfw = "strict"; break;
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"news" => []
+		];
+		
+		try{
+			
+			// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
+			$json =
+				$this->get(
+					$this->backend->get_ip(),
+					"https://api.yep.com/fs/2/search",
+					[
+						"client" => "web",
+						"gl" => $country == "all" ? $country : strtoupper($country),
+						"limit" => "99999",
+						"no_correct" => "false",
+						"q" => $search,
+						"safeSearch" => $nsfw,
+						"type" => "news"
+					]
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to fetch JSON");
+		}
+		
+		$this->detect_cf($json);
 		
 		$json = json_decode($json, true);
 		//$json = json_decode(file_get_contents("scraper/yep.json"), true);
@@ -479,54 +657,35 @@ class yep{
 			throw new Exception("Failed to decode JSON");
 		}
 		
-		if(isset($json["error"])){
+		if(isset($json[1]["results"])){
+			foreach($json[1]["results"] as $item){
 				
-			throw new Exception("Yep API returned an error: " . $json["error"]);
+				$out["news"][] = [
-		}
-		
-		if(isset($json["errors"])){
-			
-			throw new Exception("Yep API returned the following errors: {$json["message"]}");
-		}
-		
-		if(
-			isset($json["success"]) &&
-			$json["success"] !== true
-		){
-			
-			throw new Exception("Yep API returned a false-y success value");
-		}
-		
-		if(!isset($json["results"])){
-			
-			throw new Exception("Yep API did not return a results object");
-		}
-		
-		foreach($json["results"] as $item){
-			
-			if(
-				$item["url"] === null ||
-				$item["url"] == ""
-			){
-				
-				// sometimes API fucks up
-				continue;
-			}
-			
-			$out["web"][] = [
 					"title" => $item["title"],
-				"description" => $item["description"],
+					"author" => null,
-				"url" => $item["url"],
+					"description" =>
-				"date" => null,
+						$this->titledots(
-				"type" => "web",
+							strip_tags(
-				"thumb" => [
+								html_entity_decode(
+									$item["snippet"]
+								)
+							)
+						),
+					"date" => strtotime($item["first_seen"]),
+					"thumb" =>
+						isset($item["img"]) ?
+						[
+							"url" => $this->unshiturl($item["img"]),
+							"ratio" => "16:9"
+						] :
+						[
 							"url" => null,
 							"ratio" => null
 						],
-				"sublink" => [],
+					"url" => $item["url"]
-				"table" => []
 				];
 			}
+		}
 		
 		return $out;
 	}

scraper/yt.php

@@ -1209,16 +1209,15 @@ class yt{
 				
 				$reel =
 					$reel
-					->shortsLockupViewModel;
+					->reelItemRenderer;
 				
 				array_push(
 					$this->out["reel"],
 					[
 						"title" =>
 							$reel
-							->overlayMetadata
+							->headline
-							->primaryText
+							->simpleText,
-							->content,
 						"description" => null,
 						"author" => [
 							"name" => null,
@@ -1226,22 +1225,30 @@ class yt{
 							"avatar" => null
 						],
 						"date" => null,
-						"duration" => null,
+						"duration" =>
-						"views" => null,
+							$this->textualtime2int(
+								$reel
+								->accessibility
+								->accessibilityData
+								->label
+							),
+						"views" =>
+							$this->truncatedcount2int(
+								$reel
+								->viewCountText
+								->simpleText
+							),
 						"thumb" => [
 							"url" =>
 								$reel
 								->thumbnail
-								->sources[0]
+								->thumbnails[0]
 								->url,
 							"ratio" => "9:16"
 						],
 						"url" =>
 							"https://www.youtube.com/watch?v=" .
 							$reel
-							->onTap
-							->innertubeCommand
-							->reelWatchEndpoint
 							->videoId
 					]
 				);

settings.php

@@ -91,6 +91,10 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
+					[
+						"value" => "ghostery",
+						"text" => "Ghostery"
+					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -129,18 +133,10 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
-					[
-						"value" => "google_api",
-						"text" => "Google API"
-					],
 					[
 						"value" => "google_cse",
 						"text" => "Google CSE"
 					],
-					[
-						"value" => "yahoo_japan",
-						"text" => "Yahoo! JAPAN",
-					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
@@ -157,6 +153,14 @@ $settings = [
 						"value" => "yep",
 						"text" => "Yep"
 					],
+					[
+						"value" => "greppr",
+						"text" => "Greppr"
+					],
+					[
+						"value" => "crowdview",
+						"text" => "Crowdview"
+					],
 					[
 						"value" => "mwmbl",
 						"text" => "Mwmbl"
@@ -165,18 +169,6 @@ $settings = [
 						"value" => "mojeek",
 						"text" => "Mojeek"
 					],
-					[
-						"value" => "naver",
-						"text" => "Naver"
-					],
-					[
-						"value" => "baidu",
-						"text" => "Baidu"
-					],
-					[
-						"value" => "coccoc",
-						"text" => "Cốc Cốc"
-					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
@@ -185,13 +177,13 @@ $settings = [
 						"value" => "marginalia",
 						"text" => "Marginalia"
 					],
-					[
-						"value" => "purili",
-						"text" => "Purili"
-					],
 					[
 						"value" => "wiby",
 						"text" => "wiby"
+					],
+					[
+						"value" => "curlie",
+						"text" => "Curlie"
 					]
 				]
 			],
@@ -215,18 +207,10 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
-					[
-						"value" => "google_api",
-						"text" => "Google API"
-					],
 					[
 						"value" => "google_cse",
 						"text" => "Google CSE"
 					],
-					[
-						"value" => "yahoo_japan",
-						"text" => "Yahoo! JAPAN",
-					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
@@ -236,49 +220,17 @@ $settings = [
 						"text" => "Qwant"
 					],
 					[
-						"value" => "naver",
+						"value" => "yep",
-						"text" => "Naver"
+						"text" => "Yep"
-					],
-					[
-						"value" => "baidu",
-						"text" => "Baidu"
 					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
 					],
-					[
+					/*[
 						"value" => "pinterest",
 						"text" => "Pinterest"
-					],
+					],*/
-					[
-						"value" => "cara",
-						"text" => "Cara"
-					],
-					[
-						"value" => "flickr",
-						"text" => "Flickr"
-					],
-					[
-						"value" => "pexels",
-						"text" => "Pexels"
-					],
-					[
-						"value" => "pixabay",
-						"text" => "Pixabay"
-					],
-					[
-						"value" => "unsplash",
-						"text" => "Unsplash"
-					],
-					[
-						"value" => "fivehpx",
-						"text" => "500px"
-					],
-					[
-						"value" => "vsco",
-						"text" => "VSCO"
-					],
 					[
 						"value" => "imgur",
 						"text" => "Imgur"
@@ -297,14 +249,6 @@ $settings = [
 						"value" => "yt",
 						"text" => "YouTube"
 					],
-					[
-						"value" => "vimeo",
-						"text" => "Vimeo"
-					],
-					[
-						"value" => "sepiasearch",
-						"text" => "Sepia Search"
-					],
 					[
 						"value" => "ddg",
 						"text" => "DuckDuckGo"
@@ -321,10 +265,6 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
-					[
-						"value" => "yahoo_japan",
-						"text" => "Yahoo! JAPAN",
-					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
@@ -333,22 +273,6 @@ $settings = [
 						"value" => "qwant",
 						"text" => "Qwant"
 					],
-					[
-						"value" => "naver",
-						"text" => "Naver"
-					],
-					[
-						"value" => "baidu",
-						"text" => "Baidu"
-					],
-					[
-						"value" => "coccoc",
-						"text" => "Cốc Cốc"
-					],
-					[
-						"value" => "purili",
-						"text" => "Purili"
-					],
 					[
 						"value" => "solofield",
 						"text" => "Solofield"
@@ -371,10 +295,6 @@ $settings = [
 						"value" => "google",
 						"text" => "Google"
 					],
-					[
-						"value" => "yahoo_japan",
-						"text" => "Yahoo! JAPAN",
-					],
 					[
 						"value" => "startpage",
 						"text" => "Startpage"
@@ -384,12 +304,12 @@ $settings = [
 						"text" => "Qwant"
 					],
 					[
-						"value" => "mojeek",
+						"value" => "yep",
-						"text" => "Mojeek"
+						"text" => "Yep"
 					],
 					[
-						"value" => "baidu",
+						"value" => "mojeek",
-						"text" => "Baidu"
+						"text" => "Mojeek"
 					]
 				]
 			],
@@ -400,15 +320,11 @@ $settings = [
 					[
 						"value" => "sc",
 						"text" => "SoundCloud"
-					],
+					]//,
-					[
+					//[
-						"value" => "swisscows",
+					//	"value" => "spotify",
-						"text" => "Swisscows"
+					//	"text" => "Spotify"
-					]
+					//]
-					/*[
-						"value" => "spotify",
-						"text" => "Spotify"
-					]*/
 				]
 			]
 		]

static/serverping.js

@@ -80,15 +80,6 @@ document.body.insertBefore(table, quote.nextSibling);
 
 // handle sorting clicks
 var tbody = table.getElementsByTagName("tbody")[0];
-
-// server popup without inline javascript
-tbody.addEventListener('click', function(event){
-	var row = event.target.closest('.show_server');
-	if(!row) return;
-
-	show_server(row.id);
-})
-
 var th = table.getElementsByTagName("th");
 
 for(var i=0; i<th.length; i++){
@@ -352,7 +343,7 @@ function render_list(){
 	
 	for(var k=0; k<sorted_list.length; k++){
 		
-		html += '<tr class="show_server" id="' + sorted_list[k].index + '">';
+		html += '<tr onclick="show_server(' + sorted_list[k].index + ');">';
 		
 		for(var i=0; i<7; i++){
 			
@@ -415,7 +406,6 @@ function render_list(){
 	console.log(html);
 	
 	tbody.innerHTML = html;
-
 }
 
 var popup_bg = document.getElementById("popup-bg");

static/style.css

@@ -1013,23 +1013,6 @@ table tr a:last-child{
 	font-weight:bold;
 }
 
-/* booru */
-#images .image .i-title-wrapper{
-	margin-bottom:7px;
-	white-space:nowrap;
-	overflow:hidden;
-	text-overflow:ellipsis;
-}
-#images .image .i-title-wrapper a{
-	display:inline;
-}
-
-#images .image .i-title-wrapper .title{
-	display:inline;
-	margin-bottom:0;
-}
-/* end booru */
-
 #images .image .description{
 	overflow:hidden;
 	height:45px;
@@ -1130,17 +1113,6 @@ table tr a:last-child{
 /*
 	Settings page
 */
-.web .settings select{
-	all:unset;
-	user-select:none;
-	display:block;
-	border:1px solid var(--504945);
-	border-radius:2px;
-	font-size:14px;
-	padding:0 20px 0 4px;
-	height:24px;
-}
-
 .web .settings{
 	margin-top:17px;
 	border:1px dashed var(--504945);

static/themes/Gore's shitty theme.css

@@ -12,9 +12,10 @@
 	--ebdbb2: #ebdbb2;
 }
 
+
+
 body{
 	padding:15px 4% 40px;
-	margin:unset;
 }
 
 h1,h2,h3,h4,h5,h6{
@@ -40,6 +41,7 @@ h3, h4, h5, h6{
   background: #723c0b;
 }
 
+
 .searchbox input{
 	all:unset;
 	line-height:36px;
@@ -94,6 +96,7 @@ h3, h4, h5, h6{
 	display:inline-block;
 }
 
+
 .tabs .tab.selected{
 	border-bottom:2px solid #fc92a5;
 }
@@ -103,7 +106,7 @@ h3, h4, h5, h6{
 	padding-bottom:12px;
 	padding-top:7px;
 	margin-bottom:7px;
-	background-color:#232525;
+	background-color:#232525
 }
 
 .filters .filter{
@@ -166,6 +169,7 @@ h3, h4, h5, h6{
 	font-size:12px;
 }
 
+
 .web .hover{
 	display:block;
 	text-decoration:none;
@@ -189,13 +193,16 @@ h3, h4, h5, h6{
 	color:#9760b1 !important;
 }
 
+
 .web .text-result .greentext{
 	font-size:14px;
 	color:var(--bdae93);
 }
 
+
 /* favicon */
 
+
 .favicon-dropdown a{
 	text-decoration:none;
 	color:#d3d0c1;
@@ -204,31 +211,37 @@ h3, h4, h5, h6{
 	font-size:13px;
 }
 
-.web .favicon img, .favicon-dropdown img{
+
+.web .favicon img,
+.favicon-dropdown img{
 	margin:3px 7px 0 0;
 	height:16px;
 	font-size:12px;
-	line-height:16px;
+	line-height:16px;;
 	display:block;
 	text-align:left;
 }
 
+
 .web .sublinks{
 	padding:17px 10px;
 	font-size:15px;
 	color:var(--#928374);
 }
 
+
 .web .text-result .sublinks:last-child{
 	padding-bottom:0;
 }
 
+
 /* Wikipedia head */
 .wiki-head{
 	padding:5px;
-	background-color:#322f2b;
+	background-color: #322f2b
 }
 
+
 /*
 	Images tab
 */
@@ -244,6 +257,8 @@ h3, h4, h5, h6{
 	float:left;
 }
 
+
+
 #images .image .title{
 	white-space:nowrap;
 	overflow:hidden;
@@ -252,6 +267,7 @@ h3, h4, h5, h6{
     color:var(--bdae93);
 }
 
+
 #popup-status{
 	display:none;
 	position:fixed;
@@ -267,56 +283,40 @@ h3, h4, h5, h6{
 	Settings page
 */
 
+
 .web .settings-submit a{
 	margin-right:17px;
 	color:#bdae93;
 }
 
+
 /*
 	Responsive image
 */
-@media only screen and (max-width:1454px){
+@media only screen and (max-width: 1454px){ #images .image-wrapper{ width:25%; } }
-	#images .image-wrapper{
+@media only screen and (max-width: 1161px){ #images .image-wrapper{ width:25%; } }
-		width:25%;
+@media only screen and (max-width: 750px){ #images .image-wrapper{ width:50%; } }
-	}
+@media only screen and (max-width: 450px){ #images .image-wrapper{ width:100%; } }
-}
 
-@media only screen and (max-width:1161px){
-	#images .image-wrapper{
-		width:25%;
-	}
-}
-
-@media only screen and (max-width:750px){
-	#images .image-wrapper{
-		width:50%;
-	}
-}
-
-@media only screen and (max-width:450px){
-	#images .image-wrapper{
-		width:100%;
-	}
-}
 
 /*
 	Responsive design
 */
-
 @media only screen and (max-width: 1550px){
+	
+	
 	.web .left,
 	.searchbox{
 		width:60%;
 	}
+
 }
 
-@media only screen and (max-width:1100px){
+@media only screen and (max-width: 1000px){
-.web .left,
+
-	.searchbox{
-		width:100%;
-	}
 }
 
     .type{
     color:var(--bdae93);
     }
+}

static/themes/Kuuro.css

@@ -1,17 +0,0 @@
-:root{
-	--1d2021: #101010;
-	--282828: #1a1a1a;
-	--3c3836: #1e1e1e;
-	--504945: #232323;
-
-	--928374: #949494;
-	--a89984: #d2d2d2;
-	--bdae93: #d2d2d2;
-	--8ec07c: #99c794;
-	--ebdbb2: #d2d2d2;
-
-	--comment: #5f6364;
-	--default: #cccece;
-	--keyword: #c594c5;
-	--string:  #99c794;
-}

static/themes/Rosé Pine.css

@@ -1,20 +0,0 @@
-:root{
-	/* background */
-	--1d2021: #191724;
-	--282828: #1f1d2e;
-	--3c3836: #26233a;
-	--504945: #524f67;
-
-	/* font */
-	--928374: #e0def4;
-	--a89984: #908caa;
-	--bdae93: #6e6a86;
-	--8ec07c: #9ccfd8;
-	--ebdbb2: #f6c177;
-
-	/* code highlighter */
-	--comment: #6e6a86;
-	--default: #c4a7e7;
-	--keyword: #31748f;
-	--string: #f6c177;
-}

static/themes/evergarden.css

@@ -1,25 +0,0 @@
-:root {
-	/* background */
-	--1d2021: #232a2e;
-	--282828: #2b3337;
-	--3c3836: #374145;
-	--504945: #4a585c;
-
-	/* font */
-	--928374: #839e9a;
-	--a89984: #adc9bc;
-	--bdae93: #f8f9e8;
-	--8ec07c: #cbe3b3;
-	--ebdbb2: #adc9bc;
-
-	/* code highlighter */
-	--comment: #839e9a;
-	--default: #f8f9e8;
-	--keyword: #6f8788;
-	--string: #cbe3b3;
-
-	/* color codes for instance list */
-	--green: #cbe3b3;
-	--yellow: #f5d098;
-	--red: #f57f82;
-}

web.php

@@ -89,7 +89,7 @@ if($results["spelling"]["type"] != "no_correction"){
 			'&' .
 			$frontend->buildquery($get, true) .
 			'&spellcheck=no">' .
-			htmlspecialchars($results["spelling"]["correction"]) .
+			$results["spelling"]["correction"] .
 			'</a>?' .
 		'</div>';
 }