update php

.gitignore

@@ -26,4 +26,3 @@ data/captcha/minecraft/
 !banner/*default*
 scraper/curlie.html
 icons/*
-node_modules/

Dockerfile

@@ -1,10 +1,10 @@
 FROM lwthiker/curl-impersonate:0.6.1-ff-alpine AS curl-impersonate
 
-FROM alpine:3.21
+FROM alpine:3.24
 WORKDIR /var/www/html/4get
 
 RUN apk update && apk upgrade
-RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates 
+RUN apk add php apache2-ssl php85-fileinfo php85-openssl php85-iconv php85-common php85-dom php85-sodium php85-curl curl php85-pecl-apcu php85-apache2 imagemagick php85-pecl-imagick php85-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates ghostscript-fonts
 
 COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
 COPY --from=curl-impersonate /usr/local/lib /usr/local/lib

README.md

@@ -37,21 +37,20 @@ tl;dr 4get is the best way to browse for shit.
 | Brave        | Yandex       | Vimeo        | Brave        | Swisscows  | DuckDuckGo   |
 | Yandex       | Brave        | Sepia Search | Google       |            | Yandex       |
 | Google       | Google       | DuckDuckGo   | Yahoo! JAPAN |            | Google       |
-| Google API   | Google API   | Brave        | Startpage    |            | Startpage    |
-| Google CSE   | Google CSE   | Yandex       | Qwant        |            | Kagi         |
-| Yahoo! JAPAN | Yahoo! JAPAN | Google       | Mojeek       |            | Qwant        |
-| Startpage    | Startpage    | Yahoo! JAPAN | Baidu        |            | Ghostery     |
-| Qwant        | Qwant        | Startpage    |              |            | Yep          |
-| Ghostery     | Baidu        | Qwant        |              |            | Marginalia   |
-| Yep          | Solofield    | Baidu        |              |            | YouTube      |
-| Mwmbl        | Pinterest    | Coc Coc      |              |            | SoundCloud   |
-| Mojeek       | Cara         | Solofield    |              |            |              |
-| Baidu        | Flickr       |              |              |            |              |
-| Coc Coc      | Pexels       |              |              |            |              |
+| Google API   | Google API   | Brave        | Startpage    |            | Qwant        |
+| Google CSE   | Google CSE   | Yandex       | Baidu        |            | Marginalia   |
+| Yahoo! JAPAN | Yahoo! JAPAN | Google       |              |            | YouTube      |
+| Startpage    | Startpage    | Yahoo! JAPAN |              |            | SoundCloud   |
+| Yep          | Naver        | Startpage    |              |            | Startpage    |
+| Mwmbl        | Baidu        | Naver        |              |            | Kagi         |
+| Mojeek       | Solofield    | Baidu        |              |            |              |
+| Naver        | Pinterest    | Cốc Cốc      |              |            |              |
+| Baidu        | Flickr       | Purili       |              |            |              |
+| Cốc Cốc      | Pexels       | Solofield    |              |            |              |
 | Solofield    | Pixabay      |              |              |            |              |
 | Marginalia   | Unsplash     |              |              |            |              |
-| wiby         | 500px        |              |              |            |              |
-|              | VSCO         |              |              |            |              |
+| Purili       | 500px        |              |              |            |              |
+| wiby         | VSCO         |              |              |            |              |
 |              | Imgur        |              |              |            |              |
 |              | FindThatMeme |              |              |            |              |
 

ami4get.php

@@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *");
 
 include "data/config.php";
 
-$real_requests = apcu_fetch("real_requests");
-$bot_requests = apcu_fetch("captcha_gen");
+// get request count in last 24 hours
+$bot_requests = 0;
+$real_requests = 0;
+$time = time();
+
+for($i=0; $i<24; $i++){
+	
+	$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
+}
+
+for($i=0; $i<24; $i++){
+	
+	$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
+}
 
 echo json_encode(
 	[
@@ -16,8 +28,8 @@ echo json_encode(
 			"name" => config::SERVER_NAME,
 			"description" => config::SERVER_LONG_DESCRIPTION,
 			"bot_protection" => config::BOT_PROTECTION,
-			"real_requests" => $real_requests === false ? 0 : $real_requests,
-			"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
+			"real_requests" => $real_requests,
+			"bot_requests" => $bot_requests,
 			"api_enabled" => config::API_ENABLED,
 			"alt_addresses" => config::ALT_ADDRESSES,
 			"version" => config::VERSION

api/v1/ac.php

@@ -15,13 +15,11 @@ class autocomplete{
 			"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
 			"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
 			"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
-			"yep" => "https://api.yep.com/ac/?query={searchTerms}",
 			"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
 			"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
 			"sc" => "",
 			"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
-			"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
-			"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
+			"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
 		];
 		
 		/*

api/v1/images.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->image($get),

api/v1/music.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->music($get),

api/v1/news.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->news($get),

api/v1/videos.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->video($get),

api/v1/web.php

@@ -38,6 +38,9 @@ if(
 	$get["extendedsearch"] = "no";
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	
 	echo

api/v2/graph.php

@@ -0,0 +1,100 @@
+<?php
+
+header("Content-Type: image/png");
+date_default_timezone_set('America/Toronto');
+include "../../data/config.php";
+
+function get_requests($slug){
+	
+	$time = time();
+	$requests = [];
+	
+	for($i=0; $i<73; $i++){
+		
+		$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
+		
+		$requests[] = [$i, ($data === false ? 0 : $data)];
+	}
+	
+	// rtrim() out all entries that are 0
+	$out = [];
+	$found = false;
+	for($i=count($requests) - 1; $i>=0; $i--){
+		
+		if($requests[$i][1] !== 0){
+			
+			$found = $i;
+			break;
+		}
+	}
+	
+	if($found === false){
+		
+		return [];
+	}
+	
+	$requests = array_slice($requests, 0, $found + 1);
+	
+	return $requests;
+}
+
+$bot_requests = get_requests("bot_requests");
+$real_requests = get_requests("real_requests");
+
+$real_reqs_8h =
+	array_sum(
+		array_column(
+			array_slice(
+				$real_requests,
+				0,
+				8
+			),
+			1
+		)
+	);
+
+$real_reqs_24h =
+	array_sum(
+		array_column(
+			array_slice(
+				$real_requests,
+				0,
+				24
+			),
+			1
+		)
+	);
+
+$real_reqs_72h =
+	array_sum(
+		array_column(
+			$real_requests,
+			1
+		)
+	);
+
+include "../../lib/graph_gen.php";
+$graph = new graph();
+
+$c = count($real_requests);
+
+if($c > 25){
+	
+	$c = 25;
+}elseif($c <= 2){
+	
+	$c = 2;
+}
+
+echo $graph->render([
+	"title" => "Number of requests within the last 72 hours (non-cummulative)",
+	"subtitle" => date("jS M y @ g:ia", time()) . " || 8h=" . number_format($real_reqs_8h) . " - 24h=" . number_format($real_reqs_24h) . " - 72h=" . number_format($real_reqs_72h) . " || " . config::SERVER_NAME,
+	"x_label" => "<= Hours passed =>",
+	"y_label" => "<= Request count/hour =>",
+	"grad_x" => $c,
+	"grad_y" => 20,
+	"data" => [
+		"Bot requests" => $bot_requests,
+		"Real requests" => $real_requests,
+	]
+]);

data/config.php

@@ -5,7 +5,7 @@ class config{
 	// any parameters.
 	
 	// 4get version. Please keep this updated
-	const VERSION = 8;
+	const VERSION = 9;
 	
 	// Will be shown pretty much everywhere.
 	const SERVER_NAME = "4get";
@@ -23,6 +23,9 @@ class config{
 	// Enable the API?
 	const API_ENABLED = true;
 	
+	// Display errors. This should ONLY be enabled for debugging and constitutes a security risk.
+	const DISPLAY_ERRORS = false;
+	
 	//
 	// 4play settings
 	//
@@ -39,6 +42,29 @@ class config{
 	// Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw)
 	const FPLAY_EXTERNAL_ENDPOINT = null;
 	
+	//
+	// RATELIMITING
+	// Limits also applies to the captcha page
+	//
+	
+	// Amount of requests allowed before we block
+	// Set to 0 to disable ratelimiting
+	const RATELIMIT_BURST = 0;
+	
+	// IP source for ratelimiting. Choices: false for getting the IP directly,
+	// or "X-Header-Name" to get the IP from a forwarded header
+	const RATELIMIT_IP_SOURCE = false;
+	
+	// How long till the counter resets? Note: counter resets everytime
+	// a new request occurs while being blocked.
+	const RATELIMIT_TIME = 2;
+	
+	// How many seconds to add every time a new request occurs while being blocked?
+	const RATELIMIT_PUNISH = 1;
+	
+	// Add seconds of punishment time to users of the API?
+	const RATELIMIT_PUNISH_API = false;
+	
 	//
 	// BOT PROTECTION
 	//
@@ -93,11 +119,9 @@ class config{
 	const MAX_SEARCHES = 100;
 	
 	// List of domains that point to your servers. Include your tor/i2p
-	// addresses here! Must be a valid URL. Won't affect links placed on
-	// the homepage.
-	const ALT_ADDRESSES = [
-		//"https://4get.alt-tld",
-		//"http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion"
+	// addresses here! Must be a valid URL.
+	const ADDRESSES = [
+		//"https://4get.ca",
 	];
 	
 	// Known 4get instances. MUST use the https protocol if your instance uses
@@ -106,35 +130,31 @@ class config{
 	// eachother your serber should appear everywhere.
 	const INSTANCES = [
 		"https://4get.ca",
-		"https://4get.zzls.xyz",
-		"https://4getus.zzls.xyz",
+		"https://search.fischbytes.de",
+		"https://4get.nadeko.net",
 		"https://4get.silly.computer",
-		"https://4get.konakona.moe",
-		"https://4get.lvkaszus.pl",
-		"https://4g.ggtyler.dev",
-		"https://4get.perennialte.ch",
 		"https://4get.sijh.net",
 		"https://4get.hbubli.cc",
-		"https://4get.plunked.party",
-		"https://4get.etenie.pl",
 		"https://4get.lunar.icu",
 		"https://4get.dcs0.hu",
-		"https://4get.kizuki.lol",
-		"https://4get.psily.garden",
-		"https://search.milivojevic.in.rs",
-		"https://4get.snine.nl",
-		"https://4get.datura.network",
 		"https://4get.neco.lol",
-		"https://4get.lol",
-		"https://4get.ch",
 		"https://4get.edmateo.site",
 		"https://4get.sudovanilla.org",
-		"https://search.mint.lgbt"
+		"https://search.mint.lgbt",
+		"https://4get.aishiteiru.moe",
+		"https://4.nboeck.de",
+		"https://search.yonderly.org",
+		"https://4get.lurx.net",
+		"https://4get.canine.tools",
+		"https://4get.sny.sh",
+		"https://4g.opnxng.com",
+		"https://s.307200.xyz",
+		"https://4get.privadency.com/"
 	];
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0";
 	
 	// User agent to use with 4get-friendly APIs
 	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
@@ -168,6 +188,7 @@ class config{
 	const PROXY_ARCHIVEORG = false;
 	const PROXY_SEPIASEARCH = false;
 	const PROXY_ODYSEE = false;
+	const PROXY_DAILYMOTION = false;
 	const PROXY_VIMEO = false;
 	const PROXY_YEP = false;
 	const PROXY_PINTEREST = false;

docker/gen_config.php

@@ -13,7 +13,8 @@ $fourget_env = array_filter($env, function($v, $k) {
 
 foreach($fourget_env as $key => $val)  {
         $target_key = preg_replace('/^FOURGET_/', '', $key);
-        $from_env[$target_key] = trim($val, '\'"');
+        $val = trim($val, '\'"');
+        $from_env[$target_key] = match(strtolower($val)) { "true" => true, "false" => false, default => $val };
 };
 
 $merged_config = array_merge($from_config, $from_env);

docs/apache2.md

@@ -104,7 +104,7 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
 	</Directory>
 	
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
 		Require all denied
 	</DirectoryMatch>
 

docs/configure.md

@@ -152,6 +152,8 @@ This should expose a webservice located at `http://localhost:3000`. I highly rec
 
 If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
 
+**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
+
 ## Setting up 4get
 Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
 
@@ -163,3 +165,7 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
 **IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
 
 If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
+
+In case it isn't clear, here is a visual representation of what you want to achieve:
+
+![Network map](https://i.imgur.com/jFjbBtG.png)

docs/develop.md

@@ -5,3 +5,32 @@ Start the PHP webserver like this, assuming you have curl-impersonate installed
 ```sh
 LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
 ```
+
+Make sure to add `.php` to paths, the embedded PHP webserver doesnt support `mod_rewrite`-like directives.
+
+If, for some reason, you're in an environment where accessing the debug logs is impossible (like those shitty shared webhosts), you can set `DISPLAY_ERRORS` to `true` in `data/config.php`.
+
+# Debug guide for docker
+
+Install:
+```sh
+git clone https://git.lolcat.ca/lolcat/4get
+# make changes to docker/apache/http/httpd.conf (comment out the lines that disable logging)
+docker build -t luuul/4get .
+```
+
+Run:
+```sh
+docker run --name 4get -p 80:80 -p 443:443 luuul/4get 
+```
+
+Get debug logs:
+```sh
+docker exec -it `docker ps -qf ancestor=luuul/4get:latest` sh # browse the container
+docker exec -it `docker ps -qf ancestor=luuul/4get:latest` tail -f /var/log/apache2/error_log
+```
+
+Restart the container:
+```sh
+docker restart 4get
+```

docs/docker.md

@@ -150,7 +150,3 @@ services:
 ##### Tor
 
 You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
-
-##### 4play
-
-You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)

docs/docker_4play.md

@@ -1,48 +0,0 @@
-### Running 4get and 4play via docker guide
-
-This setup will run both 4get and the 4play sidecar 
-
-# Docker compose 
-
-```
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"
-
-```
-
-There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
-
-# Steps
-
-1. start the services with
-
-```
-docker compose up -d
-
-```
-
-2. configure firefox
-
-folllow the steps in [configure](./configure.md) for firefox setup

extra/4play/Dockerfile

@@ -1,19 +0,0 @@
-FROM alpine:3.21
-WORKDIR /app
-
-RUN apk update && apk upgrade
-RUN apk add nodejs npm
-
-COPY package*.json ./
-
-RUN npm ci --only=production && npm cache clean --force
-
-COPY . .
-
-EXPOSE 3030
-
-ENTRYPOINT  ["node"]
-CMD ["page-render.js"]
-
-
-

extra/4play/docker-compose.yaml

@@ -1,24 +0,0 @@
-# docker-compose.yaml
-
-services:
-  fourget:
-    image: luuul/4get:latest
-    restart: unless-stopped
-    environment:
-      - FOURGET_PROTO=http
-      - FOURGET_SERVER_NAME=4get.ca
-      - FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
-      - FOURGET_FPLAY_PASSWORD=cnc
-      - FOURGET_FPLAY_TIMEOUT=30000
-      - FOURGET_FPLAY_ENABLE_API=true
-    ports:
-      - "80:80"
-  fourplay:
-    image: luuul/4get-4play:latest
-    environment:
-      - FOURPLAY_PORT=3030
-      - FOURPLAY_PASSWORD=cnc
-      - FOURPLAY_COMMAND_TIMEOUT=5000
-    ports:
-      - "3000:3000"
-      - "3030:3030"

extra/4play/package-lock.json

@@ -1,47 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "lockfileVersion": 3,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "4play",
-      "version": "1.0.0",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "@lawlers/4play": "^1.2.5",
-        "ws": "^8.21.2"
-      }
-    },
-    "node_modules/@lawlers/4play": {
-      "version": "1.2.5",
-      "resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
-      "integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
-      "license": "AGPL-3.0-only",
-      "dependencies": {
-        "ws": "^8.21.1"
-      }
-    },
-    "node_modules/ws": {
-      "version": "8.21.2",
-      "resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
-      "integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
-      "license": "MIT",
-      "engines": {
-        "node": ">=10.0.0"
-      },
-      "peerDependencies": {
-        "bufferutil": "^4.0.1",
-        "utf-8-validate": ">=5.0.2"
-      },
-      "peerDependenciesMeta": {
-        "bufferutil": {
-          "optional": true
-        },
-        "utf-8-validate": {
-          "optional": true
-        }
-      }
-    }
-  }
-}

extra/4play/package.json

@@ -1,20 +0,0 @@
-{
-  "name": "4play",
-  "version": "1.0.0",
-  "description": "",
-  "repository": {
-    "type": "git",
-    "url": "https://git.lolcat.ca/lolcat/4get/"
-  },
-  "license": "AGPL-3.0-only",
-  "author": "lolcat",
-  "type": "commonjs",
-  "main": "page-render.js",
-  "scripts": {
-    "test": "echo \"Error: no test specified\" && exit 1"
-  },
-  "dependencies": {
-    "@lawlers/4play": "^1.2.5",
-    "ws": "^8.21.2"
-  }
-}

extra/4play/page-render.js

@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
 const http = require("http");
 
 // 4play settings
-var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
-var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
-var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
+var port = 3030;
+var timeout = 30000;
+var password = "cnc";
 
 // ==================================================
 

extra/gen_table.php

@@ -0,0 +1,151 @@
+<?php
+
+$scrapers = [
+	"web" => [
+		"DuckDuckGo",
+		"Brave",
+		"Yandex",
+		"Google",
+		"Google API",
+		"Google CSE",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Yep",
+		"Mwmbl",
+		"Mojeek",
+		"Naver",
+		"Baidu",
+		"Cốc Cốc",
+		"Solofield",
+		"Marginalia",
+		"Purili",
+		"wiby"
+	],
+	"images" => [
+		"DuckDuckGo",
+		"Yandex",
+		"Brave",
+		"Google",
+		"Google API",
+		"Google CSE",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Naver",
+		"Baidu",
+		"Solofield",
+		"Pinterest",
+		"Flickr",
+		"Pexels",
+		"Pixabay",
+		"Unsplash",
+		"500px",
+		"VSCO",
+		"Imgur",
+		"FindThatMeme"
+	],
+	"videos" => [
+		"YouTube",
+		"Vimeo",
+		"Sepia Search",
+		"DuckDuckGo",
+		"Brave",
+		"Yandex",
+		"Google",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Naver",
+		"Baidu",
+		"Cốc Cốc",
+		"Purili",
+		"Solofield"
+	],
+	"news" => [
+		"DuckDuckGo",
+		"Brave",
+		"Google",
+		"Yahoo! JAPAN",
+		"Startpage",
+		"Baidu"
+	],
+	"music" => [
+		"SoundCloud",
+		"Swisscows"
+	],
+	"autocomplete" => [
+		"Brave",
+		"DuckDuckGo",
+		"Yandex",
+		"Google",
+		"Qwant",
+		"Marginalia",
+		"YouTube",
+		"SoundCloud",
+		"Startpage",
+		"Kagi"
+	]
+];
+
+$lines = [
+	0 => "",
+	1 => ""
+];
+
+// get longest scraper string
+// also get list of scrapers with most entries
+$biggest_category_len = 0;
+foreach($scrapers as $category_name => $category){
+	
+	$count = count($category);
+	
+	if($count > $biggest_category_len){
+		
+		$biggest_category_len = $count;
+	}
+}
+
+foreach($scrapers as $category_name => $category){
+	
+	// get longest string of the category
+	$longest_len = mb_strlen($category_name);
+	foreach($category as $scraper){
+		
+		$strlen = mb_strlen($scraper);
+		
+		if($strlen > $longest_len){
+			
+			$longest_len = $strlen;
+		}
+	}
+	
+	// add header
+	$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
+	$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
+	
+	for($i=2; $i<count($category) + 2; $i++){
+		
+		if(!isset($lines[$i])){
+			
+			$lines[$i] = "";
+		}
+		
+		$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
+	}
+	
+	if($i - 2 < $biggest_category_len){
+		
+		for($k=$i; $k<$biggest_category_len + 2; $k++){
+			
+			if(!isset($lines[$k])){
+				
+				$lines[$k] = "";
+			}
+			
+			$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
+		}
+	}
+}
+
+foreach($lines as $line){
+	
+	echo $line . "|\n";
+}

images.php

@@ -24,6 +24,9 @@ $payload = [
 	"nextpage" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->image($get);
 	

lib/anubis.php

@@ -45,15 +45,15 @@ class anubis{
 		
 		if(
 			!isset($script["challenge"]) ||
+			!is_string($script["challenge"]["randomData"]) ||
 			!isset($script["rules"]["difficulty"]) ||
-			!is_int($script["rules"]["difficulty"]) ||
-			!is_string($script["challenge"])
+			!is_int($script["rules"]["difficulty"])
 		){
 			
 			throw new Exception("Found invalid challenge data");
 		}
 		
-		return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
+		return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
 	}
 	
 	private function is_valid_hash($hash, $difficulty){
@@ -74,9 +74,10 @@ class anubis{
 		return true;
 	}
 	
-	public function rape($data, $difficulty = 5){
+	public function rape($id, $data, $difficulty = 5){
 		
 		$nonce = 0;
+		$microtime = microtime(true);
 		
 		while(true){
 			
@@ -87,10 +88,12 @@ class anubis{
 				$hash_hex = bin2hex($hash_binary);
 				
 				return [
+					"id" => $id,
 					"response" => $hash_hex,
 					//"data" => $data,
 					//"difficulty" => $difficulty,
-					"nonce" => $nonce
+					"nonce" => $nonce,
+					"elapsedTime" => round((microtime(true) - $microtime) * 1000)
 				];
 			}
 			

lib/bot_protection.php

@@ -4,10 +4,96 @@ class bot_protection{
 	
 	public function __construct($frontend, $get, $filters, $page, $output){
 		
+		if(config::RATELIMIT_BURST !== 0){
+			
+			// ratelimit requests per IP
+			if(config::RATELIMIT_IP_SOURCE === false){
+				
+				$source = $_SERVER["REMOTE_ADDR"];
+			}else{
+				
+				$source = $_SERVER["HTTP_" . explode(";", strtoupper(str_replace("-", "_", config::RATELIMIT_IP_SOURCE)))[0]];
+			}
+			
+			// check if its not a local IP. Probably local tor bridge, dont ratelimit
+			if(
+				filter_var(
+					$source,
+					FILTER_VALIDATE_IP,
+					FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE
+				) !== false
+			){
+				
+				$source = "r." . $source;
+				
+				// counter
+				// resets on every req
+				$counter = apcu_fetch($source);
+				$counter = (int)$counter + 1;
+				
+				if(
+					(
+						CONFIG::RATELIMIT_PUNISH !== 0 &&
+						$counter > config::RATELIMIT_BURST + 1
+					) &&
+					(
+						(
+							$output === false &&
+							CONFIG::RATELIMIT_PUNISH_API === true
+						) ||
+						$output === true
+					)
+				){
+					
+					$block_time = config::RATELIMIT_TIME + (($counter - 1) * config::RATELIMIT_PUNISH);
+				}else{
+					
+					$block_time = config::RATELIMIT_TIME;
+				}
+				
+				apcu_store($source, $counter, $block_time);
+				
+				if($counter > config::RATELIMIT_BURST){
+					
+					apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
+					
+					if($output === false){
+						
+						http_response_code(429);
+						echo json_encode([
+							"status" => "You love 4get too much. Try again in " . $this->format_time($block_time) . "."
+						]);
+						die();
+					}
+					
+					$payload = [
+						"timetaken" => microtime(true),
+						"class" => "",
+						"right-left" => "",
+						"right-right" => "",
+						"left" =>
+							'<div class="infobox">' .
+								'<h1>You\'re damaging my calm</h1>' .
+								'You love 4get too much. Try again in <b>' . $this->format_time($block_time) . '</b>.' .
+							'</div>'
+					];
+					
+					$frontend->loadheader(
+						$get,
+						$filters,
+						$page,
+						false
+					);
+					
+					echo $frontend->load("search.html", $payload);
+					die();
+				}
+			}
+		}
+				
 		// check if we want captcha
 		if(config::BOT_PROTECTION !== 1){
 			
-			apcu_inc("real_requests");
 			if($output === true){
 				$frontend->loadheader(
 					$get,
@@ -46,8 +132,6 @@ class bot_protection{
 				}else{
 					
 					// the cookie is OK! dont die() and give results
-					apcu_inc("real_requests");
-					
 					if($output === true){
 						$frontend->loadheader(
 							$get,
@@ -164,8 +248,7 @@ class bot_protection{
 				$key = "k" . $inc . "." . $this->randomchars();
 				
 				apcu_inc($key, 1, $stupid, 86400);
-				
-				apcu_inc("real_requests");
+				apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
 				
 				setcookie(
 					"pass",
@@ -190,6 +273,7 @@ class bot_protection{
 			}
 		}
 		
+		apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
 		$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
 		
 		$payload = [
@@ -252,13 +336,28 @@ class bot_protection{
 		$frontend->loadheader(
 			$get,
 			$filters,
-			$page
+			$page,
+			false
 		);
 		
 		echo $frontend->load("search.html", $payload);
 		die();
 	}
 	
+	private function format_time($seconds){
+		
+		$hours = floor($seconds / 3600);
+		$minutes = floor(($seconds % 3600) / 60);
+		$secs = $seconds % 60;
+		
+		if($hours > 0){
+			
+			return sprintf('%dh%02dm%02ds', $hours, $minutes, $secs);
+		}
+		
+		return sprintf('%dm%02ds', $minutes, $secs);
+	}
+	
 	private function randomchars(){
 		
 		$chars =

lib/frontend.php

@@ -118,7 +118,7 @@ class frontend{
 		return trim($html);
 	}
 	
-	public function loadheader(array $get, array $filters, string $page){
+	public function loadheader(array $get, array $filters, string $page, bool $increment_bot_counter = true){
 		
 		echo
 			$this->load("header.html", [
@@ -177,13 +177,15 @@ class frontend{
 		){
 			
 			// bot detected !!
-			apcu_inc("captcha_gen");
+			if($increment_bot_counter){
+				
+				apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
+			}
 			
 			$this->drawerror(
 				"Tshh, blocked!",
 				'Your browser, IP or IP range has been blocked from this 4get instance. If this is an error, please <a href="/about">contact the administrator</a>.'
 			);
-			die();
 		}
 	}
 	
@@ -533,14 +535,490 @@ class frontend{
 					'</button>' .
 					'<div class="favicon-dropdown">';
 			
-			$payload .=
-					'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
-					'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
-					'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
-					'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
-					'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
-					'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
-				'</div>';
+			$archives = [];
+			
+			$fansites = [
+				[
+					"url" => "https://cum.st/creators?cq=",
+					"favicon" => "cum.st",
+					"favicon_alt" => "cu",
+					"title" => "OnlyHaven"
+				],
+				[
+					"url" => "https://pawchive.pw/artists?q=",
+					"favicon" => "pawchive.pw",
+					"favicon_alt" => "pa",
+					"title" => "Pawchive"
+				],
+				[
+					"url" => "https://bakemono.app/creators?q=",
+					"favicon" => "bakemono.app",
+					"favicon_alt" => "ba",
+					"title" => "Bakemono"
+				],
+				[
+					"url" => "https://kemono.cr/artists?q=",
+					"favicon" => "kemono.cr",
+					"favicon_alt" => "ke",
+					"title" => "Kemono"
+				],
+				[
+					"url" => "https://coomer.st/artists?q=",
+					"favicon" => "coomer.st",
+					"favicon_alt" => "co",
+					"title" => "Coomer"
+				]
+			];
+			
+			// add website-specific archives
+			switch($host["host"]){
+				
+				case "www.youtube.com":
+				case "music.youtube.com":
+				case "youtube.com":
+				case "m.youtube.com":
+				case "youtu.be":
+					
+					if(
+						(
+							$host["host"] == "youtu.be" &&
+							isset($host["path"]) &&
+							preg_match(
+								'/^\/([A-Za-z0-9_-]+)/',
+								$host["path"],
+								$slug
+							)
+						) ||
+						(
+							isset($host["query"]) &&
+							preg_match(
+								'/v=([A-Za-z0-9_-]+)/',
+								$host["query"],
+								$slug
+							)
+						)
+					){
+						
+						// for watch?v=slug
+						$archives[] = [
+							"url" => "https://findyoutubevideo.thetechrobo.ca/?q=" . $slug[1],
+							"favicon" => "findyoutubevideo.thetechrobo.ca",
+							"favicon_alt" => "fi",
+							"title" => "FindYouTubeVideo"
+						];
+						
+						$archives[] = [
+							"url" => "https://filmot.com/video/" . $slug[1] . "/",
+							"favicon" => "filmot.com",
+							"favicon_alt" => "fi",
+							"title" => "Filmot (metadata only)"
+						];
+					}elseif(
+						preg_match(
+							'/^\/(?:channel)\/@?([A-Za-z0-9_-]+)/',
+							$host["path"],
+							$username
+						)
+					){
+						
+						$archives[] = [
+							"url" => "https://filmot.com/channel/" . $username[1] . "/",
+							"favicon" => "filmot.com",
+							"favicon_alt" => "fi",
+							"title" => "Filmot (metadata only)"
+						];
+					}
+					break;
+				
+				case "m.twitch.tv":
+				case "twitch.tv":
+				case "player.twitch.tv":
+				case "www.twitch.tv":
+					if(
+						isset($host["path"]) &&
+						preg_match(
+							'/^\/([A-Za-z0-9_.]+)/',
+							$host["path"],
+							$username
+						) &&
+						$username[1] != "videos"
+					){
+						
+						$archives[] = [
+							"url" => "https://vodvod.top/channels/@" . $username[1],
+							"favicon" => "vodvod.top",
+							"favicon_alt" => "vo",
+							"title" => "vodvod"
+						];
+						
+						$archives[] = [
+							"url" => "https://twitchtracker.com/" . $username[1],
+							"favicon" => "twitchtracker.com",
+							"favicon_alt" => "tw",
+							"title" => "TwitchTracker"
+						];
+					}
+					
+					break;
+				
+				case "kick.com":
+				case "player.kick.com":
+					if(
+						isset($host["path"]) &&
+						preg_match(
+							'/^\/([A-Za-z0-9_.]+)/',
+							$host["path"],
+							$username
+						)
+					){
+						
+						$archives[] = [
+							"url" => "https://lick.lolcat.ca/channel?name=" . $username[1],
+							"favicon" => "lick.lolcat.ca",
+							"favicon_alt" => "li",
+							"title" => "lick"
+						];
+						
+						$archives[] = [
+							"url" => "https://kicktracker.net/" . $username[1],
+							"favicon" => "kicktracker.net",
+							"favicon_alt" => "ki",
+							"title" => "Kick tracker"
+						];
+					}
+					break;
+				
+				case "x.com":
+				case "twitter.com":
+					if(
+						isset($host["path"]) &&
+						preg_match(
+							'/^\/([A-Za-z0-9_.]+)/',
+							$host["path"],
+							$username
+						)
+					){
+						
+						$archives[] = [
+							"url" => "https://web.archive.org/web/*/https://x.com/" . $username[1] . "/status*",
+							"favicon" => "archive.org",
+							"favicon_alt" => "ar",
+							"title" => "Archive.org: Tweets &gt;2023"
+						];
+						
+						$archives[] = [
+							"url" => "https://web.archive.org/web/*/https://twitter.com/" . $username[1] . "/status*",
+							"favicon" => "archive.org",
+							"favicon_alt" => "ar",
+							"title" => "Archive.org: Tweets &lt;2023"
+						];
+					}
+					break;
+				
+				case "www.instagram.com":
+				case "instagram.com":
+					if(
+						isset($host["path"]) &&
+						preg_match(
+							'/^\/([A-Za-z0-9_.]+)/',
+							$host["path"],
+							$username
+						) &&
+						$username[1] != "p"
+					){
+						
+						$archives[] = [
+							"url" => "https://instarchiver.net/users?q=" . $username[1],
+							"favicon" => "instarchiver.net",
+							"favicon_alt" => "in",
+							"title" => "Instarchiver"
+						];
+						
+						$archives[] = [
+							"url" => "https://www.storiesdb.ch/" . $username[1],
+							"favicon" => "www.storiesdb.ch",
+							"favicon_alt" => "st",
+							"title" => "StoriesDB"
+						];
+					}
+					break;
+				
+				case "reddit.com":
+				case "old.reddit.com":
+				case "www.reddit.com":
+					if(isset($host["path"])){
+						// direct thread lookup
+						// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=pullpush
+						// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=artic_shift
+						if(
+							preg_match(
+								'/^\/r\/[^\/]+\/comments\/[^?&]+/',
+								$host["path"],
+								$slug
+							)
+						){
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io{$slug[0]}?backend=artic_shift",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "Artic Shift"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io{$slug[0]}?backend=pullpush",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "PullPush"
+							];
+						}
+						
+						// subreddit thread search
+						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=submissions&sort=desc
+						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=submissions&sort=desc
+						// subreddit comment search
+						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=comments&sort=desc
+						// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=comments&sort=desc
+						elseif(
+							preg_match(
+								'/^\/r\/([^\/]+)\/(?:(?:search|wiki|about)\/?)?(?:$|\?|&)/',
+								$host["path"],
+								$slug
+							)
+						){
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=submissions&sort=desc",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "Artic Shift (Search threads)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=submissions&sort=desc",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "PullPush (Search threads)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=comments&sort=desc",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "Artic Shift (Search comments)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=comments&sort=desc",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "PullPush (Search comments)"
+							];
+						}
+						
+						// user thread lookup
+						// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=artic_shift
+						// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=pullpush
+						// user comment lookup
+						// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=artic_shift
+						// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=pullpush
+						elseif(
+							preg_match(
+								'/^\/(?:u|user)\/([^\/]+)\/(?:(?:comments|submitted)\/?)?(?:$|\?|&)/',
+								$host["path"],
+								$slug
+							)
+						){
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=artic_shift",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "Artic Shift (Search threads)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=pullpush",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "PullPush (Search threads)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=artic_shift",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "Artic Shift (Search comments)"
+							];
+							
+							$archives[] = [
+								"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=pullpush",
+								"favicon" => "reddit.com",
+								"favicon_alt" => "re",
+								"title" => "PullPush (Search comments)"
+							];
+						}
+					}
+					break;
+				
+				// https://cum.st/creators?cq=			onlyfans fansly patreon
+				// https://pawchive.pw/artists?q=		patreon pixiv/fanbox discord
+				// https://bakemono.app/creators?q= 	pixiv/fanbox fansly onlyfans patreon
+				// https://kemono.cr/artists?q=			patreon pixiv/fanbox fantia afdian boosty gumroad subscribestar dlsite
+				// https://coomer.st/artists?q=			onlyfans fansly candfans
+				
+				// cant look up
+				/*
+				case "fantia.jp":				// https://fantia.jp/fanclubs/ <integer>
+				*/
+				
+				// normal
+				case "onlyfans.com":			// https://onlyfans.com/ <username>
+				case "fansly.com":				// https://fansly.com/ <username>
+				case "fanbox.cc":
+				case "patreon.com":				// https://www.patreon.com/ <username>
+				case "www.patreon.com":
+				case "boosty.to":				// https://boosty.to/ <username>
+				case "www.subscribestar.com":	// https://www.subscribestar.com/ <username>
+				case "subscribestar.com":
+				case "www.dlsite.com":			// https://www.dlsite.com/ <username> /
+				case "dlsite.com":
+				case "candfans.com":			// https://candfans.com/ <username>
+				case "afdian.com":				// https://afdian.com/a/ <username>
+					if(
+						isset($host["path"]) &&
+						(
+							(
+								$host["host"] == "afdian.com" &&
+								preg_match(
+									'/^\/a\/([A-Za-z0-9-_.]+)/',
+									$host["path"],
+									$username
+								)
+							) ||
+							preg_match(
+								'/^\/([A-Za-z0-9-_.]+)/',
+								$host["path"],
+								$username
+							)
+						)
+					){
+						
+						foreach($fansites as $fansite){
+							
+							$archives[] = [
+								"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
+								"favicon" =>  $fansite["favicon"],
+								"favicon_alt" =>  $fansite["favicon_alt"],
+								"title" =>  $fansite["title"]
+							];
+						}
+					}
+					break;
+			}
+			
+			// special fansite cases
+			if(
+				preg_match(
+					'/^([A-Za-z0-9_]+)\.(?:fanbox\.cc|gumroad\.com)/',
+					$host["host"],
+					$username
+				)
+			){
+				
+				foreach($fansites as $fansite){
+					
+					$archives[] = [
+						"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
+						"favicon" =>  $fansite["favicon"],
+						"favicon_alt" =>  $fansite["favicon_alt"],
+						"title" =>  $fansite["title"]
+					];
+				}
+			}
+			
+			// detect git
+			if(
+				(
+					stripos(
+						$host["host"],
+						"git."
+					) !== false ||
+					$host["host"] == "codeberg.org" ||
+					$host["host"] == "sourceforge.net" ||
+					$host["host"] == "github.com"
+				) &&
+				isset($host["path"]) &&
+				preg_match(
+					'/^(\/[^\/]+\/[^\/]+\/?)/',
+					$host["path"],
+					$edge
+				)
+			){
+				
+				$archives[] = [
+					"url" => "https://archive.softwareheritage.org/browse/origin/directory/?origin_url=" . urlencode($host["scheme"] . "://" . $host["host"] . $edge[1]),
+					"favicon" => "www.softwareheritage.org",
+					"favicon_alt" => "so",
+					"title" => "SoftwareHeritage"
+				];
+			}
+			
+			$archives =
+				array_merge(
+					$archives,
+					[
+						[
+							"url" => "https://web.archive.org/web/" . $urlencode,
+							"favicon" => "archive.org",
+							"favicon_alt" => "ar",
+							"title" => "Archive.org"
+						],
+						[
+							"url" => "https://archive.ph/newest/" . htmlspecialchars($link),
+							"favicon" => "archive.ph",
+							"favicon_alt" => "ar",
+							"title" => "Archive.ph"
+						],
+						[
+							"url" => "https://yandex.com/search/?text=" . urlencode("url:" . $link),
+							"favicon" => "yandex.com",
+							"favicon_alt" => "ya",
+							"title" => "Yandex cache"
+						],
+						[
+							"url" => "https://ghostarchive.org/search?term=" . $urlencode,
+							"favicon" => "ghostarchive.org",
+							"favicon_alt" => "gh",
+							"title" => "Ghostarchive"
+						],
+						[
+							"url" => "https://arquivo.pt/wayback/" . htmlspecialchars($link),
+							"favicon" => "arquivo.pt",
+							"favicon_alt" => "ar",
+							"title" => "Arquivo.pt"
+						],
+						[
+							"url" => "https://megalodon.jp/?url=" . $urlencode,
+							"favicon" => "megalodon.jp",
+							"favicon_alt" => "me",
+							"title" => "Megalodon"
+						],
+						[
+							"url" => "https://www.webcitation.org/query?url=" . $urlencode,
+							"favicon" => "webcitation.org",
+							"favicon_alt" => "we",
+							"title" => "Webcitation"
+						]
+					]
+				);
+			
+			foreach($archives as $archive){
+				
+				$payload .= '<a href="' . $archive["url"] . '" class="list" target="_BLANK"><img src="/favicon?s=https://' . $archive["favicon"] . '" alt="' . $archive["favicon_alt"] . '">' . $archive["title"] . '</a>';
+			}
+			
+			$payload .=	'</div>';
 		}
 		
 		/*
@@ -593,6 +1071,13 @@ class frontend{
 	
 	public function getscraperfilters($page){
 		
+		// hack: enable error reporting if configured
+		if(config::DISPLAY_ERRORS === true){
+			
+			ini_set('display_errors', 1);
+			ini_set('display_startup_errors', 1);
+		}
+		
 		$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
 		
 		if(
@@ -644,7 +1129,6 @@ class frontend{
 						"google_cse" => "Google CSE",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
-						"qwant" => "Qwant",
 						"yep" => "Yep",
 						"mwmbl" => "Mwmbl",
 						"mojeek" => "Mojeek",
@@ -671,12 +1155,10 @@ class frontend{
 						"google_cse" => "Google CSE",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
-						"qwant" => "Qwant",
 						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"solofield" => "Solofield",
 						"pinterest" => "Pinterest",
-						"cara" => "Cara",
 						"flickr" => "Flickr",
 						"pexels" => "Pexels",
 						"pixabay" => "Pixabay",
@@ -695,6 +1177,7 @@ class frontend{
 					"option" => [
 						"yt" => "YouTube",
 						//"archiveorg" => "Archive.org",
+						//"dailymotion" => "Dailymotion",
 						"vimeo" => "Vimeo",
 						//"odysee" => "Odysee",
 						"sepiasearch" => "Sepia Search",
@@ -705,7 +1188,6 @@ class frontend{
 						"google" => "Google",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
-						"qwant" => "Qwant",
 						"naver" => "Naver",
 						"baidu" => "Baidu",
 						"coccoc" => "Cốc Cốc",
@@ -724,8 +1206,7 @@ class frontend{
 						"google" => "Google",
 						"yahoo_japan" => "Yahoo! JAPAN",
 						"startpage" => "Startpage",
-						"qwant" => "Qwant",
-						"mojeek" => "Mojeek",
+						//"mojeek" => "Mojeek",
 						"baidu" => "Baidu"
 					]
 				];
@@ -1057,6 +1538,17 @@ class frontend{
 		return http_build_query($out);
 	}
 	
+	private function sanitize_slug($username){
+		
+		return urlencode(preg_replace('/[-_]/', " ", $username));
+	}
+	
+	public function increment_real_reqs($scraper){
+		
+		apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
+		apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
+	}
+	
 	public function htmlimage($image, $format){
 		
 		if(

lib/graph_gen.php

@@ -0,0 +1,388 @@
+<?php
+
+class graph{
+	
+	public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
+		
+		$graph = array_merge([
+			"title" => "Title",
+			"subtitle" => "description",
+			"grad_x" => 25,
+			"grad_y" => 20,
+			"x_label" => "<= x =>",
+			"y_label" => "<= y =>",
+			"antialias" => false,
+			"data" => []
+		], $graph);
+		
+		$size = array_merge([
+			"image_width" => 750,
+			"image_height" => 500,
+			"x" => 90,
+			"y" => 50,
+			"width" => 650,
+			"height" => 300,
+			"square" => 11,
+			"column" => 6
+		], $size);
+		
+		$colors = array_merge([
+			"bg" =>				new ImagickPixel("#1d2021"),
+			"lines_bg" =>		new ImagickPixel("#3c3836"),
+			"outline" =>		new ImagickPixel("#a89984"),
+			"text" =>			new ImagickPixel("#ebdbb2"),
+			"transparent" =>	new ImagickPixel("transparent")
+		], $colors);
+		
+		$colors_line = array_merge([
+			new ImagickPixel("#83a598"),
+			new ImagickPixel("#fb4934"),
+			new ImagickPixel("#b8bb26"),
+			new ImagickPixel("#fabd2f"),
+			new ImagickPixel("#d3869b"),
+			new ImagickPixel("#8ec07c"),
+			new ImagickPixel("#fe8019"),
+			new ImagickPixel("#458588"),
+			new ImagickPixel("#cc241d"),
+			new ImagickPixel("#98971a"),
+			new ImagickPixel("#d79921"),
+			new ImagickPixel("#b16286"),
+			new ImagickPixel("#689d6a"),
+			new ImagickPixel("#d65d0e"),
+			new ImagickPixel("#a89984") // other
+		], $colors_line);
+		
+		$not_enough_data = false;
+		
+		// compute datapoint max ranges
+		$max_range_x = 0;
+		$max_range_y = 0;
+		if(count($graph["data"]) === 0){
+			
+			$not_enough_data = true;
+		}else{
+			
+			foreach($graph["data"] as $datapoint => $value){
+				
+				if(count($value) < 2){
+					
+					$not_enough_data = true;
+				}else{
+					
+					$nv1 = [];
+					$nv2 = [];
+					foreach($value as $k => $v){
+						
+						$nv1[] = $v[0];
+						$nv2[] = $v[1];
+					}
+					
+					$max1 = max($nv1);
+					$max2 = max($nv2);
+					if($max_range_x < $max1){ $max_range_x = $max1; }
+					if($max_range_y < $max2){ $max_range_y = $max2; }
+				}
+			}
+		}
+
+		$im = new Imagick();
+		$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
+		$im->setImageFormat("png");
+
+		$draw = new ImagickDraw();
+
+		//
+		// Draw background grid
+		//
+		$draw->setStrokeColor($colors["lines_bg"]);
+		$draw->setStrokeAntialias($graph["antialias"]);
+		$draw->setStrokeWidth(1);
+		$draw->setFillColor($colors["transparent"]);
+
+		$text = new ImagickDraw();
+		$text->setFontSize(11);
+		$text->setTextAntialias($graph["antialias"]);
+		$text->setFillColor($colors["text"]);
+
+
+		// vertical lines
+		$offset = $size["width"] / ($graph["grad_x"] - 1);
+
+		for($i=0; $i<$graph["grad_x"]; $i++){
+			
+			$x = round($size["x"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_x"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$x,
+				$size["y"],
+				$x,
+				$size["y"] + $size["height"],
+			);
+			
+			if($not_enough_data === false){
+				// add numbas
+				$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
+				$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
+				
+				$im->annotateImage(
+					$text,
+					round($x - ($number_width / 2)),
+					$size["y"] + $size["height"] + 14,
+					0,
+					$number
+				);
+			}
+		}
+
+
+		// horizontal lines
+		$text->setTextAlignment(Imagick::ALIGN_RIGHT);
+		$offset = $size["height"] / ($graph["grad_y"] - 1);
+
+		$vertical_text_size = 0;
+
+		for($i=0; $i<$graph["grad_y"]; $i++){
+			
+			$y = round($size["y"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_y"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$size["x"] + 1,
+				$y,
+				$size["x"] + $size["width"] - 1,
+				$y,
+			);
+			
+			if($not_enough_data === false){
+				$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
+				$query = $im->queryFontMetrics($text, $val);
+				
+				if($query["textWidth"] > $vertical_text_size){
+					
+					$vertical_text_size = $query["textWidth"];
+				}
+				
+				// add numbas
+				$im->annotateImage(
+					$text,
+					$size["x"] - 4,
+					$y + 4,
+					0,
+					$val
+				);
+			}
+		}
+
+
+		//
+		// Draw legend
+		//
+		$text->setTextAlignment(Imagick::ALIGN_LEFT);
+		$top_padding = 44;
+		$c = count($colors_line);
+		$padding = $size["width"] / $size["column"];
+
+		$line = new ImagickDraw();
+		//$line->setStrokeWidth(1);
+		$line->setStrokeAntialias($graph["antialias"]);
+
+		if($not_enough_data === false){
+			$i = 0;
+			foreach($graph["data"] as $datafield => $data){
+				
+				$used_color = $colors_line[$i % $c];
+				$draw->setStrokeColor($used_color);
+				$row = floor($i / $size["column"]);
+				
+				$x = round($size["x"] + (($i % $size["column"]) * $padding));
+				$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
+				
+				// draw legend
+				$draw->rectangle(
+					$x,
+					$y,
+					$x + $size["square"],
+					$y + $size["square"]
+				);
+				
+				$im->annotateImage(
+					$text,
+					$x + $size["square"] + 4,
+					$y + 10,
+					0,
+					$datafield
+				);
+				
+				//
+				// Draw lines
+				//
+				$line->setStrokeColor($used_color);
+				$old_x = null;
+				$old_y = null;
+				$continue = true;
+				
+				foreach($data as $field){
+					
+					// field 0 = hour (sorted)
+					// field 1 = datapoint
+					
+					$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
+					$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
+					
+					if($continue){
+						
+						$continue = false;
+						$old_x = $x;
+						$old_y = $y;
+						continue;
+					}
+					
+					$line->line($old_x, $old_y, $x, $y);
+							
+					$old_x = $x;
+					$old_y = $y;
+				}
+				
+				$i++;
+			}
+		}
+
+		//
+		// add background labels
+		//
+		$text->setFontSize(13);
+		$text->setFillColor($colors["bg"]);
+		$draw->setFillColor($colors["text"]);
+		$draw->setStrokeColor($colors["transparent"]);
+
+		// horizontal
+		$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
+		$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
+		$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
+
+		$draw->rectangle(
+			$x_x - 2,
+			$x_y - $label_width_x["textHeight"] + 4,
+			$x_x + $label_width_x["textWidth"] + 2,
+			$x_y + 3,
+		);
+
+		// vertical
+		$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
+		$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
+		$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
+
+		$draw->rectangle(
+			$y_x + 4,
+			$y_y - 2,
+			$y_x + $label_width_y["textHeight"] + 3,
+			$y_y + $label_width_y["textWidth"] + 2,
+		);
+
+		$im->drawImage($draw);
+		$im->drawImage($line);
+
+		// horizontal text
+		$im->annotateImage(
+			$text,
+			$x_x,
+			$x_y,
+			0,
+			$graph["x_label"]
+		);
+
+		// vertical text
+		$im->annotateImage(
+			$text,
+			$y_x + $label_width_y["textHeight"],
+			$y_y + $label_width_y["textWidth"],
+			270,
+			$graph["y_label"]
+		);
+
+		//
+		// draw title + subtext
+		//
+		$text->setFontSize(20);
+		$text->setFontWeight(900);
+		$text->setFillColor($colors["text"]);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 25,
+			0,
+			$graph["title"]
+		);
+
+		$text->setFontSize(13);
+		$text->setFontWeight(100);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 10,
+			0,
+			$graph["subtitle"]
+		);
+
+		if($not_enough_data){
+			
+			// add not enough data notice
+			$rect = new ImagickDraw();
+			$rect->setFillColor($colors["bg"]);
+			$rect->setStrokeColor($colors_line[1]);
+			
+			$rect_w = 200;
+			$rect_h = 24;
+				
+			$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
+			$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
+			
+			$rect->rectangle(
+				$rect_x,
+				$rect_y,
+				$rect_x + $rect_w,
+				$rect_y + $rect_h
+			);
+			
+			$im->drawImage($rect);
+			
+			$label = "Not enough data";
+			
+			$text->setFontWeight(900);
+			$query = $im->queryFontMetrics($text, $label);
+			
+			$im->annotateImage(
+				$text,
+				round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
+				round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
+				0,
+				$label
+			);
+		}
+
+		return $im->getImageBlob();
+	}
+}

music.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->music($get);
 	

news.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->news($get);
 	

oracles/base.php

@@ -1,36 +0,0 @@
-<?php
-abstract class oracle {
-	// some info to spit out alongside the result, so the user knows
-	//  what exactly is giving out the answer. prevents confusion
-	//   about what oracle is answering them for ambiguous queries.
-	public $info = [
-		"name" => "some oracle"
-	];
-	// this function should take in a query string search from $_GET,
-	//  and return a bool determining whether or not it is a question
-	//   intended for the oracle.
-	public function check_query($q) {
-		return false;
-	}
-	// produce the correct answer for the query using the oracle.
-	//  note: if it becomes apparent /during generation/ that the
-	//   query is not in fact for the oracle, returning an empty
-	//    string will kill the oracle pane.
-	// answer format: ["ans1 title" => "ans1", ...]
-	public function generate_response($q) {
-		return "";
-	}
-}
-// backwards compatibility
-if (!function_exists('str_starts_with')) {
-	function str_starts_with($haystack, $needle) {
-		return strncmp($haystack, $needle, strlen($needle)) === 0;;
-	}
-}
-if (!function_exists('str_contains')) {
-	function str_contains($haystack, $needle) {
-		return strpos((string)$haystack, (string)$needle) !== false;
-	}
-}
-
-?>

oracles/calc.php

@@ -1,165 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class calculator extends oracle {
-	public $info = [
-		"name" => "calculator"
-	];
-	public function check_query($q) {
-		// straight numerics should go to that oracle
-		if (is_numeric($q)) {
-			return false;
-		}
-		// all chars should be number-y or operator-y
-		$char_whitelist = str_split("1234567890.+-/*^%() ");
-		foreach (str_split($q) as $char) {
-			if (!in_array($char, $char_whitelist)) {
-				return false;
-			}
-		}
-		return true;
-	}
-	// a custom parser and calculator because FUCK YUO, libraries are
-	//  gay.
-	public function generate_response($q)
-	{
-		$nums = str_split("1234567890.");
-		$ops = str_split("+-/*^%;");
-		$grouping = str_split("()");
-
-		$q = str_replace(" ", "", $q);
-
-		// backstop for the parser so it catches the last
-		//  numeric token
-		$q .= ";"; 
-
-		// the following comments refer to this example input:
-		//  21+9*(3+2^9)+1
-
-		// 2-length lists of the following patterns:
-		//  ["n" (umeric), <some number>]
-		//  ["o" (perator), "<some operator>"]
-		//  ["g" (roup explicit), <"(" or ")">]
-		// e.g. [["n", 21], ["o", "+"], ["n", 9], ["o", *],
-		//       ["g", "("], ["n", 3], ["o", "+"], ["n", 2],
-		//       ["o", "^"], ["n", 9], ["g", ")"], ["o", "+"],
-		//       ["n", "1"]]
-		$tokens = array();
-		$dragline = 0;
-		foreach(str_split($q) as $i=>$char) {
-			if (in_array($char, $nums)) {
-				continue;
-			}
-			elseif (in_array($char, $ops) || in_array($char, $grouping)) {
-				// hitting a non-numeric implies everything since the
-				//  last hit has been part of a number
-				$capture = substr($q, $dragline, $i - $dragline);
-				// prevent the int cast from creating imaginary
-				//  ["n", 0] tokens
-				if ($capture != "") {
-					if (substr_count($capture, ".") > 1) {
-						return "";
-					}
-					array_push($tokens, ["n", (float)$capture]);
-				}
-				// reset to one past the current (non-numeric) char
-				$dragline = $i + 1; 
-				// the `;' backstop is not a real token and this should
-				//  never be present in the token list
-				if ($char != ";") {
-					array_push($tokens, [
-						($char == "(" || $char == ")") ? "g" : "o",
-						$char
-					]);
-				}
-			}
-			else {
-				return "";
-			}
-		}
-
-		// two operators back to back should fail
-		for ($i = 1; $i < count($tokens); $i++) {
-			if ($tokens[$i][0] == "o" && $tokens[$i-1][0] == "o") {
-				return "";
-			}
-		}
-
-		// no implicit multiplication
-		for ($i = 0; $i < count($tokens) - 1; $i++) {
-			if ($tokens[$i][0] == "n" && $tokens[$i+1] == ["g", "("]) {
-				return "";
-			}
-		}
-
-		//strategy:
-		// traverse to group open (if there is one)
-		//  - return to start with the internals
-		// traverse to ^, attack token previous and after
-		// same but for *, then / then + then -
-		// poppers all teh way down
-		try {
-			return [
-				substr($q, 0, strlen($q)-1)." = " => $this->executeBlock($tokens)[0][1]
-			];
-		}   
-		catch (\Throwable $e) {
-			if (get_class($e) == "DivisionByZeroError") {
-				return [
-					$q." = " => "Division by Zero Error!!"
-				];
-			}
-			return "";
-		}
-	}
-	public function executeBlock($tokens) {
-		if (count($tokens) >= 2 && $tokens[0][0] == "o" && $tokens[0][1] == "-" && $tokens[1][0] == "n") {
-			array_splice($tokens, 0, 2, [["n", -1 * (float)$tokens[1][1]]]);
-		}
-		if (count($tokens) > 0 && $tokens[0][0] == "o" || $tokens[count($tokens)-1][0] == "o") {
-			throw new Exception("Error Processing Request", 1);
-		}
-		while (in_array(["g", "("], $tokens)) {
-			$first_open = array_search(["g", "("], $tokens);
-			$enclosedality = 1;
-			for ($i = $first_open+1; $i < count($tokens); $i++) {
-				if ($tokens[$i][0] == "g") {
-					$enclosedality += ($tokens[$i][1] == "(") ? 1 : -1;
-				}
-				if ($enclosedality == 0) {
-					array_splice($tokens, 
-						$first_open, 
-						$i+1 - $first_open, 
-						$this->executeBlock(
-							array_slice($tokens, $first_open+1, $i-1 - $first_open)
-						)
-					);
-					break;
-				}
-			}
-		}
-		$operators_in_pemdas_order = [
-			"^" => (fn($x, $y) => $x ** $y),
-			"*" => (fn($x, $y) => $x * $y),
-			"/" => (fn($x, $y) => $x / $y), 
-			"%" => (fn($x, $y) => $x % $y),
-			"+" => (fn($x, $y) => $x + $y), 
-			"-" => (fn($x, $y) => $x - $y)
-		];
-		foreach ($operators_in_pemdas_order as $op=>$func) {
-			while (in_array(["o", $op], $tokens)) {
-				for ($i = 0; $i < count($tokens); $i++) {
-					if ($tokens[$i] == ["o", $op]) {
-						array_splice(
-							$tokens,
-							$i-1,
-							3,
-							[["n", (string)($func((float)$tokens[$i-1][1], (float)$tokens[$i+1][1]))]]
-						);
-					}
-				}
-			}
-		}
-		return $tokens;
-	}
-}
-?>

oracles/encoder.php

@@ -1,40 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class encoder extends oracle {
-	public $info = [
-		"name" => "text encoder/hasher"
-	];
-	private $special_types = [
-		"rot13",
-		"base64"
-	];
-	public function check_query($q) {
-		$types = array_merge($this->special_types, hash_algos());
-		foreach ($types as $type) {
-			$type .= " ";
-			if (str_starts_with($q, $type)) {
-				return true;
-			}
-		}
-		return false;
-	}
-	public function generate_response($q)
-	{
-		$type = explode(" ", $q)[0];
-		$victim = substr($q, strlen($type)+1);
-		if (in_array($type, hash_algos())) {
-			return [$type." hash" => hash($type, $victim)];
-		}
-		switch ($type) {
-			case "rot13":
-				return ["rot13 encoded" => str_rot13($victim)];
-			case "base64":
-				return [
-					"base64 encoded" => base64_encode($victim),
-					"base64 decoded" => base64_decode($victim)
-				];
-		}
-		return "";
-	}
-}
-?>

oracles/numerics.php

@@ -1,54 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class numerics extends oracle {
-	public $info = [
-		"name" => "numeric base conversion"
-	];
-	public function check_query($q) {
-		if (str_contains($q, " ")) {
-			return false;
-		}
-
-		$q = strtolower($q);
-
-		$profiles = [
-			["0x", str_split("0123456789abcdef")],
-			["", str_split("1234567890")],
-			["b", str_split("10")]
-		];
-
-		foreach ($profiles as $profile) {
-			$good = true;
-			$good &= str_starts_with($q, $profile[0]);
-			$nq = substr($q, strlen($profile[0]));
-			foreach (str_split($nq) as $c) {
-				$good &= in_array($c, $profile[1]);
-			}
-			if ($good) {
-				return true;
-			}
-		}
-		return false;
-	}
-	public function generate_response($q) {
-		$n = 0;
-		if (str_starts_with($q, "0x")) {
-			$nq = substr($q, strlen("0x"));
-			$n = hexdec($nq);
-		}
-		elseif (str_starts_with($q, "b")) {
-			$nq = substr($q, strlen("b"));
-			$n = bindec($nq);
-		}
-		else {
-			$n = (int)$q;
-		}
-		return [
-			"decimal (base 10)" => (string)$n,
-			"hexadecimal (base 16)" => "0x".(string)dechex($n),
-			"binary (base 2)" => "b".(string)decbin($n),
-			"" => "binary inputs should be prefixed with 'b', hex with '0x'."
-		];
-	}
-}
-?>

oracles/time.php

@@ -1,45 +0,0 @@
-<?php
-include_once("oracles/base.php");
-class time extends oracle {
-	public $info = [
-		"name" => "what time is it?"
-	];
-	public function check_query($q) {
-		$prompts = [
-			"what", "time", "is", "it",
-			"right", "now", "the", "current",
-			"get", "date"
-		];
-		$q = str_replace(",", "", $q);
-		$q = str_replace("?", "", $q);
-		$q = str_replace("what's", "what is", $q);
-		$oq = $q;
-		$q = explode(" ", $q);
-		$count = 0;
-		foreach ($q as $word) {
-			if (in_array($word, $prompts)) {
-				$count++;
-			}
-		}
-		// remove one from total count if a timezone is specified
-		return ($count/(count($q) + (str_contains($oq, "tz:") ? -1 : 0))) > 3/4;
-	}
-	public function generate_response($q) {
-		$timezone = timezone_name_from_abbr("UTC");
-		foreach (explode(" ", $q) as $word) {
-			if (str_starts_with($word, "tz:")) {
-				$decltz = timezone_name_from_abbr(substr($word, 3, 3));
-				if ($decltz) {
-					$timezone = $decltz;
-				}
-			}
-		}
-		date_default_timezone_set($timezone);
-		return [
-			"The time in ".$timezone => date("H:i:s"),
-			" " => date("l, F jS"),
-			"" => "include the string \"tz:XXX\" to use timezone XXX"
-		];
-	}
-}
-?>

resolve/google.php

@@ -0,0 +1,107 @@
+<?php
+
+class google{
+	
+	public function __construct(){
+		
+		include "lib/backend.php";
+		$this->backend = new backend("google");
+		
+		include "lib/fuckhtml.php";
+		$this->fuckhtml = new fuckhtml();
+	}
+	
+	private function get(/*$proxy, */$url, $get = []){
+		
+		$curlproc = curl_init();
+		
+		if($get !== []){
+			$get = http_build_query($get);
+			$url .= "?" . $get;
+		}
+		
+		curl_setopt($curlproc, CURLOPT_URL, $url);
+		
+		// http2 bypass
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		
+		$headers =
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"];
+		
+		// i dont think proxies are necessary here...
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		$data = curl_exec($curlproc);
+		
+		if(curl_errno($curlproc)){
+			throw new Exception(curl_error($curlproc));
+		}
+		
+		curl_close($curlproc);
+		return $data;
+	}
+	
+	function resolve($id){
+		
+		$html =
+			$this->get(
+				//$this->backend->get_ip(),
+				"https://www.google.com/goto",
+				[
+					"url" => $id
+				]
+			);
+		
+		$this->fuckhtml->load($html);
+		
+		$a =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"a"
+			);
+		
+		if(count($a) === 0){
+			
+			throw new Exception("Failed to find link");
+		}
+		
+		$link =
+			$this->fuckhtml
+			->getTextContent(
+				$a[0]["attributes"]["href"]
+			);
+		
+		if(
+			preg_match(
+				'/^(https?:|)\/\//',
+				$link
+			)
+		){
+			
+			return $link;
+		}
+		
+		throw new Exception("Got malformed link: " . $link);
+	}
+}

resolver.php

@@ -5,7 +5,8 @@ $resolver = new resolver();
 class resolver{
 	
 	public const resolvers = [
-		"sc"
+		"sc",
+		"google"
 	];
 	
 	public function __construct(){
@@ -40,9 +41,13 @@ class resolver{
 			$resolver = new $scraper();
 			$link = $resolver->resolve($target);
 			
-			if(is_string($link)){
+			if(
+				is_string($link) &&
+				!empty($link)
+			){
 				
 				header("Location: {$link}");
+				echo 'If your ass is not getting redirected, <a href="' . htmlspecialchars($link) . '">click here</a>.';
 			}
 		}catch(Exception $error){
 			
@@ -54,13 +59,13 @@ class resolver{
 		
 		header("Content-Type: text/plain");
 		http_response_code(400);
-		echo $message;
+		echo htmlspecialchars($message);
 	}
 	
 	public function do404($message){
 		
 		header("Content-Type: text/plain");
 		http_response_code(404);
-		echo $message;
+		echo htmlspecialchars($message);
 	}
 }

robots.txt

@@ -1,28 +1,23 @@
-# When the robots.txt is sus
-
-# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌
-# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌
-# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌
-# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪
-# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡
-# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐
-# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔
-# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘
-# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎
-# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗
-# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷
-# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟
-# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰
-# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊
-# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌
-# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁
-# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀
-# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀
-# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟
-# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾
-# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽
+#   44
+# 4444444      44
+# 44444444   44444       444
+# 44444444  444444    444444444
+#  44444   44444444  444444444
+#         444444444   4444444
+#        4444444444    444444
+#      4444444444444
+#    444444444444444444
+#       444444444444444
+#          44444444
+#          4444
+#           44
 
 User-agent: *
-Disallow:
-Host: 4get.ca
-Sitemap: https://4get.ca/sitemap
+Disallow: /web
+Disallow: /images
+Disallow: /videos
+Disallow: /news
+Disallow: /music
+Disallow: /proxy
+Disallow: /api/v1/ac
+Allow: /

scraper/baidu.php

@@ -919,8 +919,8 @@ class baidu{
 				$description =
 					$this->fuckhtml
 					->getElementsByAttributeValue(
-						"data-sanssr-cmpt",
-						"card/www-summary-1",
+						"data-module",
+						"abstract",
 						$div
 					);
 				

scraper/brave.php

@@ -212,59 +212,43 @@ class brave{
 	
 	private function get_js(){
 		
-		$script_disc =
+		$script_bodies =
 			$this->fuckhtml
 			->getElementsByTagName(
 				"script"
 			);
 		
-		$data = null;
-		foreach($script_disc as &$discs){
+		$datas = [];
+		foreach($script_bodies as &$script){
 			
-			if(
-				preg_match(
-					'/kit\.start\(/',
-					$discs["innerHTML"]
-				)
-			){
+			$jsons = explode("{type:\"data\"", $script["innerHTML"]);
+			array_shift($jsons);
 			
-				$data =
-					explode(
-						"data:",
-						$discs["innerHTML"],
-						2
-					);
-				
-				if(count($data) !== 2){
-					
-					throw new Exception("Failed to split up data field");
-				}
-				
-				$data = $data[1];
-				break;
-			}
-		}
-		
-		if($data === null){
-			
-			throw new Exception("Could not grep JavaScript object");
-		}
+			foreach($jsons as $json){
 				
 				$data =
 					$this->fuckhtml
 					->parseJsObject(
 						$this->fuckhtml
 						->extract_json(
-					$data
+							"{type:\"data\"" .
+							$json
 						)
 					);
 				
-		if($data === null){
+				if($data !== []){
 					
-			throw new Exception("Failed to decode JavaScript object");
+					$datas[] = $data;
+				}
+			}
 		}
 		
-		return $data;
+		if(count($datas) === 0){
+			
+			throw new Exception("Could not grep any JavaScript object");
+		}
+		
+		return $datas;
 	}
 	
 	public function web($get){
@@ -383,6 +367,8 @@ class brave{
 		// load html
 		$this->fuckhtml->load($html);
 		
+		$this->detect_captcha($html);
+		
 		/*
 			Get next page "token"
 		*/
@@ -437,8 +423,11 @@ class brave{
 		
 		// do some magic
 		$this->fuckhtml->load($html);
-		$data = $this->get_js();
+		$datas = $this->get_js();
 		
+		$has_found_body = false;
+		
+		foreach($datas as $data){
 			if(
 				isset($data[2]["data"]["title"]) &&
 				stripos($data[2]["data"]["title"], "PoW Captcha") !== false
@@ -447,12 +436,13 @@ class brave{
 				throw new Exception("Brave returned a PoW captcha");
 			}
 			
-		if(!isset($data[1]["data"]["body"]["response"])){
+			if(!isset($data["data"]["body"]["response"])){
 				
-			throw new Exception("Brave did not return a result object");
+				continue;
 			}
 			
-		$data = $data[1]["data"]["body"]["response"];
+			$has_found_body = true;
+			$data = $data["data"]["body"]["response"];
 			
 			/*
 				Get web results
@@ -972,11 +962,23 @@ class brave{
 						}
 					}
 					
+					$thumb = isset($info["images"][0]["original"]) ? $info["images"][0]["original"] : null;
+					
+					if(
+						preg_match(
+							'/^\/\//',
+							$thumb
+						) !== 0
+					){
+						
+						$thumb = "https:" . $thumb;
+					}
+					
 					$out["answer"][] = [
 						"title" => $this->fuckhtml->getTextContent($info["title"]),
 						"description" => $description,
 						"url" => $info["url"],
-					"thumb" => isset($info["images"][0]["original"]) ? $info["images"][0]["original"] : null,
+						"thumb" => $thumb,
 						"table" => $table,
 						"sublink" => $sublink
 					];
@@ -992,6 +994,16 @@ class brave{
 				
 				foreach($data["videos"]["results"] as $video){
 					
+					// get thumbnails
+					$thumbs = [];
+					
+					if(isset($video["video"]["thumbnail"]["src"])){ $thumbs[] = $video["video"]["thumbnail"]["src"]; }
+					if(isset($video["video"]["thumbnail"]["original"])){ $thumbs[] = $video["video"]["thumbnail"]["original"]; }
+					if(isset($video["thumbnail"]["src"])){ $thumbs[] = $video["thumbnail"]["src"]; }
+					if(isset($video["thumbnail"]["original"])){ $thumbs[] = $video["thumbnail"]["original"]; }
+					
+					$thumb = $this->check_thumbs($thumbs);
+					
 					$out["video"][] = [
 						"title" => $this->titledots($video["title"]),
 						"description" => $this->titledots($video["description"]),
@@ -999,10 +1011,10 @@ class brave{
 						"duration" => isset($video["video"]["duration"]) && $video["video"]["duration"] != "void 0" ? $this->hms2int($video["video"]["duration"]) : null,
 						"views" => isset($video["video"]["views"]) && $video["video"]["views"] != "void 0" ? (int)$video["video"]["views"] : null,
 						"thumb" =>
-						isset($video["thumbnail"]["src"]) ?
+							$thumb !== false ?
 							[
 								"ratio" => "16:9",
-							"url" => $this->unshiturl($video["thumbnail"]["src"])
+								"url" => $this->unshiturl($thumb)
 							] :
 							[
 								"ratio" => null,
@@ -1089,6 +1101,12 @@ class brave{
 			
 			// append discussions at position 2
 			array_splice($out["web"], 1, 0, $disc_out);
+		}
+		
+		if($has_found_body === false){
+			
+			throw new Exception("Brave did not return a search result object");
+		}
 		
 		return $out;
 	}
@@ -1175,6 +1193,8 @@ class brave{
 		// load html
 		$this->fuckhtml->load($html);
 		
+		$this->detect_captcha($html);
+		
 		// get npt
 		$out["npt"] =
 			$this->generatenextpagetoken(
@@ -1187,10 +1207,11 @@ class brave{
 			);
 		
 		$this->fuckhtml->load($html);
-		$json = $this->get_js();
+		$jsons = $this->get_js();
 		
+		foreach($jsons as $json){
 			foreach(
-			$json[1]["data"]["body"]["response"]["news"]["results"]
+				$json["data"]["response"]["news"]["results"]
 				as $news
 			){
 				
@@ -1220,6 +1241,7 @@ class brave{
 					"url" => $news["url"]
 				];
 			}
+		}
 		
 		return $out;
 	}
@@ -1270,10 +1292,14 @@ class brave{
 		fclose($handle);*/
 		
 		$this->fuckhtml->load($html);
-		$json = $this->get_js();
 		
+		$this->detect_captcha($html);
+		
+		$jsons = $this->get_js();
+		
+		foreach($jsons as $json){
 			foreach(
-			$json[1]
+				$json
 				["data"]
 				["body"]
 				["response"]
@@ -1298,6 +1324,7 @@ class brave{
 					"url" => $result["url"]
 				];
 			}
+		}
 			
 		return $out;
 	}
@@ -1374,6 +1401,8 @@ class brave{
 		
 		$this->fuckhtml->load($html);
 		
+		$this->detect_captcha($html);
+		
 		$out = [
 			"status" => "ok",
 			"npt" => null,
@@ -1401,11 +1430,11 @@ class brave{
 		fclose($handle);*/
 		
 		$this->fuckhtml->load($html);
-		$json = $this->get_js();
+		$jsons = $this->get_js();
 		
+		foreach($jsons as $json){
 			foreach(
 				$json
-			[1]
 				["data"]
 				["body"]
 				["response"]
@@ -1454,6 +1483,7 @@ class brave{
 					"url" => $result["url"]
 				];
 			}
+		}
 
 		return $out;
 	}
@@ -1853,4 +1883,52 @@ class brave{
 				)
 			);
 	}
+	
+	private function check_thumbs($thumbs){
+		
+		foreach($thumbs as $thumb){
+			
+			if(
+				preg_match(
+					'/^http/',
+					$thumb
+				) === 1
+			){
+				
+				return $this->unshiturl($thumb);
+			}
+		}
+		
+		return false;
+	}
+	
+	private function detect_captcha(){
+		
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"title"
+			);
+		
+		if(count($title) === 0){
+			
+			if(strtolower($this->fuckhtml->getloadedhtml()) == "this service is not available in your region"){
+				
+				throw new Exception("Brave rangebanned the IP range");
+			}
+			
+			throw new Exception("Brave returned a malformed page");
+		}
+		
+		$title =
+			strtolower(
+				$this->fuckhtml
+				->getTextContent($title[0])
+			);
+		
+		if($title == "human verification"){
+			
+			throw new Exception("Brave returned a Captcha");
+		}
+	}
 }

scraper/cara.php

@@ -1,847 +0,0 @@
-<?php
-
-class cara{
-	
-	public function __construct(){
-		
-		include "lib/backend.php";
-		$this->backend = new backend("cara");
-	}
-	
-	public function getfilters($page){
-		
-		return [
-			"sort" => [
-				"display" => "Sort by",
-				"option" => [
-					"Top" => "Top",
-					"MostRecent" => "Most Recent"
-				]
-			],
-			"type" => [
-				"display" => "Post type",
-				"option" => [
-					"any" => "Any type",
-					"portfolio" => "Portfolio", // {"posts":["portfolio"]}
-					"timeline" => "Timeline" // {"posts":["timeline"]}
-				]
-			],
-			"fields" => [
-				"display" => "Field/Medium",
-				"option" => [
-					"any" => "Any field",
-					"2D" => "2D Work",
-					"3D" => "3D Work",
-					"3DPrinting" => "3D Printing",
-					"Acrylic" => "Acrylic",
-					"AlcoholMarkers" => "Alcohol Markers",
-					"Animation" => "Animation",
-					"Chalk" => "Chalk",
-					"Charcoal" => "Charcoal",
-					"Colored pencil" => "Colored pencil",
-					"Conte" => "Conte",
-					"Crayon" => "Crayon",
-					"Digital" => "Digital",
-					"Gouache" => "Gouache",
-					"Ink" => "Ink",
-					"MixedMedia" => "Mixed-Media",
-					"Oil" => "Oil",
-					"Oil-based Markers" => "Oil-based Markers",
-					"Other" => "Other",
-					"Pastels" => "Pastels",
-					"Photography" => "Photography",
-					"Sculpture" => "Sculpture",
-					"Sketches" => "Sketches",
-					"Tattoos" => "Tattoos",
-					"Traditional" => "Traditional",
-					"VFX" => "VFX",
-					"Watercolor" => "Watercolor"
-				]
-			],
-			"category" => [
-				"display" => "Category",
-				"option" => [
-					"any" => "Any category",
-					"3DScanning" => "3D Scanning",
-					"Abstract" => "Abstract",
-					"Adoptable" => "Adoptable",
-					"Anatomy" => "Anatomy",
-					"Animals" => "Animals",
-					"Anime" => "Anime",
-					"App" => "App",
-					"ArchitecturalConcepts" => "Architectural Concepts",
-					"ArchitecturalVisualization" => "Architectural Visualization",
-					"AugmentedReality" => "Augmented Reality",
-					"Automotive" => "Automotive",
-					"BoardGameArt" => "Board Game Art",
-					"BookIllustration" => "Book Illustration",
-					"CardGameArt" => "Card Game Art",
-					"CeramicsPottery" => "Ceramics/Pottery",
-					"CharacterAnimation" => "Character Animation",
-					"CharacterDesign" => "Character Design",
-					"CharacterModeling" => "Character Modeling",
-					"ChildrensArt" => "Children's Illustration",
-					"Collectibles" => "Collectibles",
-					"ColoringPage" => "Coloring Page",
-					"ComicArt" => "Comic Art",
-					"ConceptArt" => "Concept Art",
-					"Cosplay" => "Cosplay",
-					"CostumeDesign" => "Costume Design",
-					"CoverArt" => "Cover Art",
-					"Creatures" => "Creatures",
-					"Diorama" => "Diorama",
-					"EditorialIllustration" => "Editorial Illustration",
-					"EmbroiderySewing" => "Embroidery/Sewing",
-					"EnvironmentalConceptArt" => "Environmental Concept Art",
-					"EnvironmentalConceptDesign" => "Environmental Concept Design",
-					"FanArt" => "Fan Art",
-					"Fantasy" => "Fantasy",
-					"Fashion" => "Fashion",
-					"FashionStyling" => "Fashion Styling",
-					"FiberArts" => "Fiber Arts",
-					"Furry" => "Furry",
-					"GameArt" => "Game Art",
-					"GameplayDesign" => "Gameplay Design",
-					"GamesEnvironmentArt" => "Games Environment Art",
-					"Gem" => "Gem",
-					"GraphicDesign" => "Graphic Design",
-					"Handicraft" => "Handicraft",
-					"HairStyling" => "Hair Styling",
-					"HardSurface" => "Hard Surface",
-					"Horror" => "Horror",
-					"Illustration" => "Illustration",
-					"IllustrationVisualization" => "Illustration Visualization",
-					"IndustrialDesign" => "Industrial Design",
-					"Jewelry" => "Jewelry",
-					"KnittingCrochet" => "Knitting/Crochet",
-					"Landscape" => "Landscape",
-					"LevelDesign" => "Level Design",
-					"Lighting" => "Lighting",
-					"Makeup" => "Makeup",
-					"Manga" => "Manga",
-					"MapsCartography" => "Maps/Cartography",
-					"MattePainting" => "Matte Painting",
-					"Materials" => "Materials",
-					"MechanicalDesign" => "Mechanical Design",
-					"Medical" => "Medical",
-					"Mecha" => "Mecha",
-					"MiniatureArt" => "Miniature Art",
-					"MotionGraphics" => "Motion Graphics",
-					"FrescoMurals" => "Fresco/Murals",
-					"Natural" => "Natural",
-					"Original Character" => "Original Character",
-					"Overlay" => "Overlay",
-					"PleinAir" => "Plein Air",
-					"Photogrammetry" => "Photogrammetry",
-					"PixelArt" => "Pixel Art",
-					"Portraits" => "Portraits",
-					"Props" => "Props",
-					"ProductDesign" => "Product Design",
-					"PublicDomain" => "Public Domain or Royalty Free",
-					"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
-					"Realism" => "Realism",
-					"ScienceFiction" => "Science Fiction",
-					"ScientificVisualization" => "Scientific Visualization",
-					"Scripts" => "Scripts",
-					"StillLife" => "Still Life",
-					"Storyboards" => "Storyboards",
-					"Stylized" => "Stylized",
-					"Surreal" => "Surreal",
-					"TechnicalArt" => "Technical Art",
-					"Textures" => "Textures",
-					"Tools" => "Tools",
-					"Toys" => "Toys",
-					"ToyPackaging" => "Toy Packaging",
-					"Tutorials" => "Tutorials",
-					"UIArt" => "User Interface (UI) Art",
-					"UrbanSketch" => "Urban Sketch",
-					"VFXforAnimation" => "VFX for Animation",
-					"VFXforFilm" => "VFX for Film",
-					"VFXforGames" => "VFX for Games",
-					"VFXforRealTime" => "VFX for Real-Time",
-					"VFXforTV" => "VFX for TV",
-					"Vehicles" => "Vehicles",
-					"VirtualReality" => "Virtual Reality",
-					"VisualDevelopment" => "Visual Development",
-					"VoxelArt" => "Voxel Art",
-					"Vtubers" => "Vtubers",
-					"WIP" => "WIP (Work in Progress)",
-					"Web" => "Web",
-					"Weapons" => "Weapons",
-					"Wildlife" => "Wildlife",
-					"Woodcutting" => "Woodcutting"
-				]
-			],
-			"software" => [
-				"display" => "Software",
-				"option" => [
-					"any" => "Any software",
-					"123D" => "123D",
-					"123DCatch" => "123D Catch",
-					"3DBee" => "3DBee",
-					"3DCoat" => "3DCoat",
-					"3DCoatPrint" => "3DCoatPrint",
-					"3DCoatTextura" => "3DCoatTextura",
-					"3DEqualizer" => "3DEqualizer",
-					"3DFZephyr" => "3DF Zephyr",
-					"3Delight" => "3Delight",
-					"3dpeople" => "3dpeople",
-					"3dsMax" => "3ds Max",
-					"3DSPaint" => "3DS Paint",
-					"ACDSeeCanvas" => "ACDSee Canvas",
-					"AbletonLive" => "Ableton Live",
-					"Acrobat" => "Acrobat",
-					"AdobeDraw" => "Adobe Draw",
-					"AdobeFlash" => "Adobe Flash",
-					"AdobeFresco" => "Adobe Fresco",
-					"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
-					"AdobeXD" => "Adobe XD",
-					"AffinityDesigner" => "Affinity Designer",
-					"AffinityPhoto" => "Affinity Photo",
-					"AfterEffects" => "After Effects",
-					"Akeytsu" => "Akeytsu",
-					"Alchemy" => "Alchemy",
-					"AliasDesign" => "Alias Design",
-					"AlightMotion" => "Alight Motion",
-					"Amadine" => "Amadine",
-					"Amberlight" => "Amberlight",
-					"Animate" => "Animate",
-					"AnimationMaster" => "Animation:Master",
-					"AnimeStudio" => "Anime Studio",
-					"Apophysis" => "Apophysis",
-					"ArchiCAD" => "ArchiCAD",
-					"Arion" => "Arion",
-					"ArionFX" => "ArionFX",
-					"Arnold" => "Arnold",
-					"ArtEngine" => "ArtEngine",
-					"ArtFlow" => "ArtFlow",
-					"ArtRage" => "ArtRage",
-					"ArtstudioPro" => "Artstudio Pro",
-					"Artweaver" => "Artweaver",
-					"Aseprite" => "Aseprite",
-					"Audition" => "Audition",
-					"AutoCAD" => "AutoCAD",
-					"AutodeskSketchBook" => "Autodesk SketchBook",
-					"AvidMediaComposer" => "Avid Media Composer",
-					"AzPainter" => "AzPainter",
-					"babylonjs" => "babylon.js",
-					"BalsamiqMockup" => "Balsamiq Mockup",
-					"Bforartists" => "Bforartists",
-					"BlackInk" => "Black Ink",
-					"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
-					"Blender" => "Blender",
-					"Blender DeepPaint" => "Blender DeepPaint",
-					"BlenderGreasePencil" => "Blender Grease Pencil",
-					"Blockbench" => "Blockbench",
-					"BodyPaint" => "BodyPaint",
-					"Boxcutter" => "Boxcutter",
-					"BraidMaker" => "Braid Maker",
-					"BrickLinkStudio" => "BrickLink Studio",
-					"Bridge" => "Bridge",
-					"Brushifyio" => "Brushify.io",
-					"C" => "C",
-					"C#" => "C#",
-					"C++" => "C++",
-					"CACANi" => "CACANi",
-					"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
-					"CLO" => "CLO",
-					"CRYENGINE" => "CRYENGINE",
-					"Callipeg" => "Callipeg",
-					"Canva" => "Canva",
-					"CaptureOne" => "Capture One",
-					"CartoonAnimator" => "Cartoon Animator",
-					"Carveco" => "Carveco",
-					"Cavalry" => "Cavalry",
-					"Chaotica" => "Chaotica",
-					"CharacterAnimator" => "Character Animator",
-					"CharacterCreator" => "Character Creator",
-					"Cinema4D" => "Cinema 4D",
-					"ClarisseiFX" => "Clarisse iFX",
-					"Coiffure" => "Coiffure",
-					"ColorsLive" => "Colors Live",
-					"Combustion" => "Combustion",
-					"Construct2" => "Construct 2",
-					"Core" => "Core",
-					"CorelPainter" => "Corel Painter",
-					"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
-					"CoronaRenderer" => "Corona Renderer",
-					"ProMotionNG" => "Cosmigo Pro Motion NG",
-					"CrazyBump" => "CrazyBump",
-					"Crocotile3D" => "Crocotile 3D",
-					"Curvy3D" => "Curvy 3D",
-					"Cycles4D" => "Cycles 4D",
-					"Darkroom" => "Darkroom",
-					"DAZStudio" => "DAZ Studio",
-					"DDO" => "DDO",
-					"DECIMA" => "DECIMA",
-					"Darktable" => "Darktable",
-					"DaVinciResolve" => "DaVinci Resolve",
-					"Dimension" => "Dimension",
-					"DragonBones" => "DragonBones",
-					"Dragonframe" => "Dragonframe",
-					"Drawpile" => "Drawpile",
-					"Dreams" => "Dreams",
-					"Dreamweaver" => "Dreamweaver",
-					"DxOPhotoLab" => "DxO PhotoLab",
-					"ECycles" => "E-Cycles",
-					"EmberGen" => "EmberGen",
-					"Encore" => "Encore",
-					"Expresii" => "Expresii",
-					"FStorm" => "FStorm",
-					"FadeIn" => "FadeIn",
-					"Feather3D" => "Feather 3D",
-					"FiberShop" => "FiberShop",
-					"Figma" => "Figma",
-					"FilmoraWondershare" => "Filmora Wondershare",
-					"FilterForge" => "Filter Forge",
-					"FinalCutPro" => "Final Cut Pro",
-					"FinalDraft" => "Final Draft",
-					"finalRender" => "finalRender",
-					"FireAlpaca" => "FireAlpaca",
-					"Fireworks" => "Fireworks",
-					"FlamePainter" => "Flame Painter",
-					"Flash" => "Flash",
-					"FlipaClip" => "FlipaClip",
-					"FlipnoteStudio" => "Flipnote Studio",
-					"Fluent" => "Fluent",
-					"ForestPack" => "Forest Pack",
-					"FormZ" => "Form-Z",
-					"Fractorium" => "Fractorium",
-					"FreeCAD" => "FreeCAD",
-					"FreeHand" => "FreeHand",
-					"Forger" => "Forger",
-					"FrostbiteEngine" => "Frostbite Engine",
-					"fSpy" => "fSpy",
-					"FumeFX" => "FumeFX",
-					"Fusion360" => "Fusion 360",
-					"GIMP" => "GIMP",
-					"GSCurveTools" => "GS CurveTools",
-					"GSToolbox" => "GS Toolbox",
-					"Gaea" => "Gaea",
-					"GameTextures" => "Game Textures",
-					"GameMakerStudio" => "GameMaker: Studio",
-					"GarageFarmNET" => "GarageFarm.NET",
-					"GeoGlyph" => "GeoGlyph",
-					"GigapixelAl" => "Gigapixel Al",
-					"Glaxnimate" => "Glaxnimate",
-					"GnomePaint" => "Gnome Paint",
-					"Godot" => "Godot",
-					"Goxel" => "Goxel",
-					"Graphite" => "Graphite",
-					"Graswald" => "Graswald",
-					"GravitySketch" => "Gravity Sketch",
-					"GuerillaRender" => "GuerillaRender",
-					"HDRLightStudio" => "HDR Light Studio",
-					"HairStrandDesigner" => "Hair Strand Designer",
-					"HairTGHairFur" => "HairTG - Hair &amp; Fur",
-					"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
-					"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
-					"Handplane" => "Handplane",
-					"Hansoft" => "Hansoft",
-					"HardOps" => "Hard Ops",
-					"HardMesh" => "HardMesh",
-					"Harmony" => "Harmony",
-					"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
-					"HelloPaint" => "HelloPaint",
-					"HeliconFocus" => "Helicon Focus",
-					"Hexels" => "Hexels",
-					"HiPaint" => "HiPaint",
-					"Houdini" => "Houdini",
-					"HydraRenderer" => "Hydra Renderer",
-					"iArtbook" => "iArtbook",
-					"IbisPaint" => "ibisPaint",
-					"Ideas" => "Ideas",
-					"IllustStudio" => "Illust Studio",
-					"Illustrator" => "Illustrator",
-					"IllustratorDraw" => "Illustrator Draw",
-					"InDesign" => "InDesign",
-					"Inochi2D" => "Inochi2D",
-					"InVision" => "InVision",
-					"InVisionCraft" => "InVision Craft",
-					"InfinitePainter" => "Infinite Painter",
-					"Inkscape" => "Inkscape",
-					"Inspirit" => "Inspirit",
-					"InstaLOD" => "InstaLOD",
-					"InstaMAT" => "InstaMAT",
-					"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
-					"InstantMeshes" => "Instant Meshes",
-					"InstantTerra" => "Instant Terra",
-					"Inventor" => "Inventor",
-					"Iray" => "Iray",
-					"JWildfire" => "JWildfire",
-					"Java" => "Java",
-					"Jira" => "Jira",
-					"JumpPaint" => "Jump Paint by MediBang",
-					"JSPaint" => "JS Paint",
-					"Katana" => "Katana",
-					"Keyshot" => "Keyshot",
-					"KidPix" => "Kid Pix",
-					"KitBash3D" => "KitBash3D",
-					"Knald" => "Knald",
-					"Kodon" => "Kodon",
-					"KolourPaint" => "KolourPaint",
-					"Krakatoa" => "Krakatoa",
-					"KRESKA" => "KRESKA",
-					"Krita" => "Krita",
-					"LensStudio" => "Lens Studio",
-					"LibreSprite" => "LibreSprite",
-					"LightWave3D" => "LightWave 3D",
-					"Lightroom" => "Lightroom",
-					"Linearity" => "Linearity",
-					"LiquiGen" => "LiquiGen",
-					"Live2DCubism" => "Live2D Cubism",
-					"LookatmyHair" => "Look at my Hair",
-					"Lotpixel" => "Lotpixel",
-					"Lumion" => "Lumion",
-					"LuxRender" => "LuxRender",
-					"MacPaint" => "MacPaint",
-					"MagicaCSG" => "MagicaCSG",
-					"MagicaVoxel" => "MagicaVoxel",
-					"Magma" => "Magma",
-					"MakeHuman" => "MakeHuman",
-					"Malmal" => "Malmal",
-					"Mandelbulb3D" => "Mandelbulb 3D",
-					"Mandelbulber" => "Mandelbulber",
-					"MangaStudio" => "Manga Studio",
-					"Mari" => "Mari",
-					"MarmosetToolbag" => "Marmoset Toolbag",
-					"MarvelousDesigner" => "Marvelous Designer",
-					"MasterpieceStudioPro" => "Masterpiece Studio Pro",
-					"MasterpieceVR" => "MasterpieceVR",
-					"Maverick" => "Maverick",
-					"MaxwellRender" => "Maxwell Render",
-					"Maya" => "Maya",
-					"MediBangPaint" => "MediBang Paint",
-					"MediumbyAdobe" => "Medium by Adobe",
-					"Megascans" => "Megascans",
-					"mentalray" => "mental ray",
-					"MeshLab" => "MeshLab",
-					"Meshroom" => "Meshroom",
-					"MetaHumanCreator" => "MetaHuman Creator",
-					"Metashape" => "Metashape",
-					"MightyBake" => "MightyBake",
-					"MikuMikuDance" => "MikuMikuDance",
-					"Minecraft" => "Minecraft",
-					"Mischief" => "Mischief",
-					"Mixamo" => "Mixamo",
-					"Mixer" => "Mixer",
-					"MoI3D" => "MoI3D",
-					"Mocha" => "Mocha",
-					"Modo" => "Modo",
-					"Moho" => "Moho",
-					"MotionBuilder" => "MotionBuilder",
-					"Mudbox" => "Mudbox",
-					"Muse" => "Muse",
-					"MSPaint" => "MS Paint",
-					"MyPaint" => "MyPaint",
-					"NDO" => "NDO",
-					"NX" => "NX",
-					"NdotCAD" => "NdotCAD",
-					"NintendoNotes" => "Nintendo Notes",
-					"NomadSculpt" => "Nomad Sculpt",
-					"Notability" => "Notability",
-					"Nuke" => "Nuke",
-					"Nvil" => "Nvil",
-					"OctaneRender" => "Octane Render",
-					"Omniverse" => "Omniverse",
-					"OmniverseCreate" => "Omniverse Create",
-					"ON1PhotoRAW" => "ON1 Photo RAW",
-					"Open3DEngine" => "Open 3D Engine",
-					"OpenCanvas" => "OpenCanvas",
-					"OpenGL" => "OpenGL",
-					"OpenToonz" => "OpenToonz",
-					"Ornatrix" => "Ornatrix",
-					"OsciRender" => "Osci-Render",
-					"OurPaint" => "Our Paint",
-					"PBRMAX" => "PBRMAX",
-					"PFTrack" => "PFTrack",
-					"PTGui" => "PTGui",
-					"Paintbrush" => "Paintbrush",
-					"PaintNET" => "Paint.NET",
-					"PaintShopPro" => "PaintShop Pro",
-					"PaintToolSAI" => "Paint Tool SAI",
-					"PaintstormStudio" => "Paintstorm Studio",
-					"Paper" => "Paper",
-					"Pencil2D" => "Pencil2D",
-					"Penpot" => "Penpot",
-					"PhoenixFD" => "Phoenix FD",
-					"Phonto" => "Phonto",
-					"PhotoLab2" => "PhotoLab 2",
-					"Photopea" => "Photopea",
-					"Photoscan" => "Photoscan",
-					"Photoshop" => "Photoshop",
-					"PhotoshopElements" => "Photoshop Elements",
-					"PicoCAD" => "picoCAD",
-					"PicoCAD2" => "picoCAD 2",
-					"Pinta" => "Pinta",
-					"Piskel" => "Piskel",
-					"Pixilart" => "Pixilart",
-					"Pixelitor" => "Pixelitor",
-					"Pixelmator" => "Pixelmator",
-					"Pixelorama" => "Pixelorama",
-					"PixivSketch" => "pixiv Sketch",
-					"Pixquare" => "Pixquare",
-					"PlantCatalog" => "PlantCatalog",
-					"PlantFactory" => "PlantFactory",
-					"Plasticity" => "Plasticity",
-					"PNGtuberPlus" => "PNGtuber Plus",
-					"Poliigon" => "Poliigon",
-					"Polybrush" => "Polybrush",
-					"PopcornFx" => "PopcornFx",
-					"Poser" => "Poser",
-					"Premiere" => "Premiere",
-					"PremiereElements" => "Premiere Elements",
-					"PresagisCreator" => "Presagis Creator",
-					"ProTools" => "Pro Tools",
-					"Procreate" => "Procreate",
-					"ProcreateDreams" => "Procreate Dreams",
-					"Producer" => "Producer",
-					"PrometheanAI" => "Promethean AI",
-					"PureRef" => "PureRef",
-					"Python" => "Python",
-					"PyxelEdit" => "PyxelEdit",
-					"QuadRemesher" => "Quad Remesher",
-					"QuarkXPress" => "QuarkXPress",
-					"Qubicle" => "Qubicle",
-					"Quill" => "Quill",
-					"QuixelBridge" => "Quixel Bridge",
-					"QuixelMegascans" => "Quixel Megascans",
-					"QuixelMixer" => "Quixel Mixer",
-					"QuixelSuite" => "Quixel Suite",
-					"R3DSWrap" => "R3DS Wrap",
-					"R3DSZWRAP" => "R3DS ZWRAP",
-					"RDTextures" => "RD-Textures",
-					"RailClone" => "RailClone",
-					"RealFlow" => "RealFlow",
-					"RealisticPaintStudio" => "Realistic Paint Studio",
-					"RealityCapture" => "RealityCapture",
-					"RealityScan" => "RealityScan",
-					"RealtimeBoard" => "Realtime Board",
-					"Rebelle" => "Rebelle",
-					"Redshift" => "Redshift",
-					"RenderMan" => "RenderMan",
-					"RenderNetwork" => "Render Network",
-					"Revit" => "Revit",
-					"Rhino" => "Rhino",
-					"Rhinoceros" => "Rhinoceros",
-					"RizomUV" => "RizomUV",
-					"RoughAnimator" => "Rough Animator",
-					"SamsungNotes" => "Samsung Notes",
-					"SamsungPENUP" => "Samsung PENUP",
-					"ScansLibrary" => "ScansLibrary",
-					"Scrivener" => "Scrivener",
-					"Sculpt+" => "Sculpt+",
-					"Sculptris" => "Sculptris",
-					"ShaveandaHaircut" => "Shave and a Haircut",
-					"ShiVa3D" => "ShiVa3D",
-					"Shotgun" => "Shotgun",
-					"Silo" => "Silo",
-					"Silugen" => "Silugen",
-					"Sketch" => "Sketch",
-					"SketchApp" => "Sketch App",
-					"SketchBookPro" => "SketchBook Pro",
-					"SketchClub" => "SketchClub",
-					"SketchUp" => "SketchUp",
-					"Sketchable" => "Sketchable",
-					"Sketchfab" => "Sketchfab",
-					"Skyshop" => "Skyshop",
-					"Snapseed" => "Snapseed",
-					"Snowdrop" => "Snowdrop",
-					"Softimage" => "Softimage",
-					"SolidWorks" => "SolidWorks",
-					"SonySketch" => "Sony Sketch",
-					"Soundbooth" => "Soundbooth",
-					"Source2" => "Source 2",
-					"SourceControl" => "Source Control",
-					"SourceFilmmaker" => "Source Filmmaker",
-					"SpeedTree" => "SpeedTree",
-					"Speedgrade" => "Speedgrade",
-					"SpeedyPainter" => "SpeedyPainter",
-					"Spine2D" => "Spine 2D",
-					"Spriter" => "Spriter",
-					"Stingray" => "Stingray",
-					"Storyboarder" => "Storyboarder",
-					"StoryboardPro" => "Storyboard Pro",
-					"SublimeText" => "Sublime Text",
-					"Substance3DDesigner" => "Substance 3D Designer",
-					"Substance3DModeler" => "Substance 3D Modeler",
-					"Substance3DPainter" => "Substance 3D Painter",
-					"Substance3DSampler" => "Substance 3D Sampler",
-					"Substance3DStager" => "Substance 3D Stager",
-					"SubstanceB2M" => "Substance B2M",
-					"SweetHome3D" => "Sweet Home 3D",
-					"SynthEyes" => "SynthEyes",
-					"TTools" => "TTools",
-					"TVPaint" => "TVPaint",
-					"TVPaintAnimation" => "TVPaint Animation",
-					"TayasuiSketches" => "Tayasui Sketches",
-					"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
-					"TayasuiSketchesPro" => "Tayasui Sketches Pro",
-					"Terragen" => "Terragen",
-					"Texturescom" => "Textures.com",
-					"Texturingxyz" => "Texturingxyz",
-					"TeyaConceptor" => "Teya Conceptor",
-					"TheGrove3D" => "The Grove 3D",
-					"TheaRender" => "Thea Render",
-					"Threejs" => "Three.js",
-					"Tiled" => "Tiled",
-					"TiltBrush" => "Tilt Brush",
-					"Tooll3" => "Tooll3",
-					"ToonBoomHarmony" => "Toon Boom Harmony",
-					"ToonBoomStudio" => "Toon Boom Studio",
-					"ToonSquid" => "ToonSquid",
-					"TopoGun" => "TopoGun",
-					"TuxPaint" => "Tux Paint",
-					"Tvori" => "Tvori",
-					"Twinmotion" => "Twinmotion",
-					"UNIGINEEngine" => "UNIGINE Engine",
-					"UVLayout" => "UVLayout",
-					"UltraFractal" => "Ultra Fractal",
-					"uMake" => "uMake",
-					"Unfold3D" => "Unfold 3D",
-					"Unity" => "Unity",
-					"UnrealEngine" => "Unreal Engine",
-					"Vengi" => "vengi",
-					"VRay" => "V-Ray",
-					"VRED" => "VRED",
-					"VTubeStudio" => "VTube Studio",
-					"Vectary" => "Vectary",
-					"VectorayGen" => "VectorayGen",
-					"Vectorworks" => "Vectorworks",
-					"VegasPro" => "Vegas Pro",
-					"VisualDesigner3D" => "Visual Designer 3D",
-					"VisualStudio" => "Visual Studio",
-					"VRoidStudio" => "VRoid Studio",
-					"Vue" => "Vue",
-					"Vuforia" => "Vuforia",
-					"WebGL" => "WebGL",
-					"WhiteboardFox" => "Whiteboard Fox",
-					"WickEditor" => "Wick Editor",
-					"Wings3D" => "Wings 3D",
-					"Word" => "Word",
-					"WorldCreator" => "World Creator",
-					"WorldMachine" => "World Machine",
-					"XParticles" => "X-Particles",
-					"Xfrog" => "Xfrog",
-					"Xgen" => "Xgen",
-					"xNormal" => "xNormal",
-					"xTex" => "xTex",
-					"XoliulShader" => "Xoliul Shader",
-					"Yafaray" => "Yafaray",
-					"Yeti" => "Yeti",
-					"ZBrush" => "ZBrush",
-					"ZBrushCore" => "ZBrushCore",
-					"ZenBrush" => "Zen Brush"
-				]
-			]
-		];
-	}
-	
-	private function get($proxy, $url, $get = [], $search){
-		
-		$curlproc = curl_init();
-		
-		if($get !== []){
-			$get = http_build_query($get);
-			$url .= "?" . $get;
-		}
-		
-		curl_setopt($curlproc, CURLOPT_URL, $url);
-		
-		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
-		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
-			["User-Agent: " . config::USER_AGENT,
-			"Accept: application/json, text/plain, */*",
-			"Accept-Language: en-US,en;q=0.5",
-			"Accept-Encoding: gzip, deflate, br, zstd",
-			//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
-			//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
-			"DNT: 1",
-			"Sec-GPC: 1",
-			"Connection: keep-alive",
-			//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
-			"Referer: https://cara.app/search?q=" . urlencode($search),
-			//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
-			"Sec-Fetch-Dest: empty",
-			"Sec-Fetch-Mode: cors",
-			"Sec-Fetch-Site: same-origin",
-			"TE: trailers"]
-		);
-		
-		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
-		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
-		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
-		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
-
-		$this->backend->assign_proxy($curlproc, $proxy);
-		
-		$data = curl_exec($curlproc);
-		
-		if(curl_errno($curlproc)){
-			
-			throw new Exception(curl_error($curlproc));
-		}
-		
-		curl_close($curlproc);
-		return $data;
-	}
-	
-	public function image($get){
-		
-		if($get["npt"]){
-			
-			[$npt, $proxy] =
-				$this->backend->get(
-					$get["npt"],
-					"images"
-				);
-			
-			$npt = json_decode($npt, true);
-		}else{
-			
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-						
-			$npt = [
-				"q" => $get["s"],
-				"sortBy" => $get["sort"],
-				"take" => 24,
-				"skip" => 0,
-				"filters" => []
-			];
-			
-			// parse filters
-			if($get["type"] != "any"){
-				
-				$npt["filters"]["posts"] = [$get["type"]];
-			}
-			
-			if($get["fields"] != "any"){
-				
-				$npt["filters"]["fields"] = [$get["fields"]];
-			}
-			
-			if($get["category"] != "any"){
-				
-				$npt["filters"]["categories"] = [$get["category"]];
-			}
-			
-			if($get["software"] != "any"){
-				
-				$npt["filters"]["softwares"] = [$get["software"]];
-			}
-			
-			if($npt["filters"] == []){
-				
-				$npt["filters"] = "{}";
-			}else{
-				
-				$npt["filters"] = json_encode($npt["filters"]);
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"image" => []
-		];
-		
-		// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
-		try{
-			$json =
-				$this->get(
-					$proxy,
-					"https://cara.app/api/search/posts",
-					$npt,
-					$npt["q"]
-				);
-			
-		}catch(Exception $error){
-			
-			throw new Exception("Failed to fetch JSON");
-		}
-		
-		$json = json_decode($json, true);
-		
-		if($json === null){
-			
-			throw new Exception("Failed to decode JSON");
-		}
-		
-		$imagecount = 0;
-		foreach($json as $image){
-			
-			if(count($image["images"]) === 0){
-				
-				// sometimes the api returns no images for an object
-				$imagecount++;
-				continue;
-			}
-			
-			$cover = null;
-			$sources = [];
-			
-			foreach($image["images"] as $source){
-				
-				if($source["isCoverImg"]){
-					
-					$cover = [
-						"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
-						"width" => 500,
-						"height" => 500
-					];
-				}else{
-					
-					$sources[] = [
-						"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
-						"width" => null,
-						"height" => null
-					];
-				}
-			}
-			
-			if($cover !== null){
-				
-				$sources[] = $cover;
-			}
-			
-			$out["image"][] = [
-				"title" => str_replace("\n", " ", $image["content"]),
-				"source" => $sources,
-				"url" => "https://cara.app/post/" . $image["id"]
-			];
-			
-			$imagecount++;
-		}
-		
-		if($imagecount === 24){
-			
-			$npt["skip"] += 24;
-			
-			$out["npt"] =
-				$this->backend->store(
-					json_encode($npt),
-					"images",
-					$proxy
-				);
-		}
-		
-		return $out;
-	}
-	
-	private function fix_url($url){
-		
-		return
-			str_replace(
-				[" "],
-				["%20"],
-				$url
-			);
-	}
-}

scraper/coccoc.php

@@ -9,10 +9,13 @@ class coccoc{
 		
 		include "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
+		
+		$this->proc = curl_multi_init();
+		$this->handles = [];
 	}
 	
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $cookie = false, $get = []){
 		
 		$curlproc = curl_init();
 		
@@ -26,20 +29,43 @@ class coccoc{
 		// http2 bypass
 		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
 		
+		if($cookie === false){
+			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+				"User-Agent: " . config::USER_AGENT,
 				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-			"Accept-Language: en-US,en;q=0.5",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
 				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
-			//"Cookie: _contentAB_15040_vi=V-06_01; split_test_search=new_search; uid=L_bauXyZBY1B; vid=uCVQJQSTgb9QGT3o; ls=1753742684; serp_version=29223843/7621a70; savedS=direct",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
-			"Sec-Fetch-Site: cross-site",
+				"Sec-Fetch-Site: none",
+				"Sec-Fetch-User: ?1",
 				"Priority: u=0, i"
 			]);
+		}else{
+			
+			curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
+				"User-Agent: " . config::USER_AGENT,
+				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
+				"Accept-Encoding: gzip, deflate, br, zstd",
+				"DNT: 1",
+				"Sec-GPC: 1",
+				"Connection: keep-alive",
+				"Referer: https://coccoc.com/search",
+				"Cookie: {$cookie}",
+				"Upgrade-Insecure-Requests: 1",
+				"Sec-Fetch-Dest: document",
+				"Sec-Fetch-Mode: navigate",
+				"Sec-Fetch-Site: same-origin",
+				"Priority: u=0, i",
+				"TE: trailers"
+			]);
+		}
 		
 		$this->backend->assign_proxy($curlproc, $proxy);
 		
@@ -93,6 +119,212 @@ class coccoc{
 		];
 	}
 	
+	private function handle_redir($proxy, $html){
+		
+		$this->fuckhtml->load($html);
+		
+		$title =
+			$this->fuckhtml
+			->getElementsByTagName("title");
+		
+		if(
+			count($title) !== 0 &&
+			$this->fuckhtml
+			->getTextContent(
+				$title[0]
+			) == "Redirecting..."
+		){
+			
+			// get cookie
+			if(
+				preg_match(
+					'/document\.cookie ?= ?"([^;"]+).*"/',
+					$html,
+					$cookie
+				) === 0
+			){
+				
+				throw new Exception("Failed to grep cookie");
+			}
+			
+			// get redirect URL
+			if(
+				preg_match(
+					'/window\.location\.replace\("([^"]+)"\)/',
+					$html,
+					$redir
+				) === 0
+			){
+				
+				throw new Exception("Failed to grep redirection URL");
+			}
+			
+			try{
+				
+				$html =
+					$this->get(
+						$proxy,
+						$redir[1],
+						$cookie[1],
+						[]
+					);
+			}catch(Exception $error){
+				
+				throw new Exception("Failed to get redirection page");
+			}
+		}
+		
+		return $html;
+	}
+	
+	private function redirect_add_url($url){
+		
+		// https://coccoc.com/composer/goto?url=0AWtXcuK-mJVvFl8Qhny9LZdMOyEd08nFmTtpg82bmuqtp8p8G_OnTebNhPtNyIXMGQ1TZZ-ADMz0QEiFJhsTLIFkkPDXnJjHo3kqsobmyntiOhFag-2aqTs
+		
+		if(
+			preg_match(
+				'/\/goto\?url=([^&]+)/',
+				$url,
+				$slug
+			) === 0
+		){
+			
+			// not a coccoc redirect
+			return $url;
+		}
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, "https://coccoc.com/composer/goto?url=" . urlencode($slug[1]));
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		// funny
+		curl_setopt($curlproc, CURLOPT_HEADER, true);
+		
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_multi_add_handle($this->proc, $curlproc);
+		$this->handles[] = [
+			"replacement" => $url,
+			"handle" => $curlproc
+		];
+		
+		return $url;
+	}
+	
+	private function resolve_urls(&$scraped_data){
+		
+		do{
+			do{
+				$status = curl_multi_exec($this->proc, $active);
+			}while($status === CURLM_CALL_MULTI_PERFORM);
+
+			if($active){
+				curl_multi_select($this->proc);
+			}
+
+		}while($active && $status === CURLM_OK);
+		
+		//
+		// if we reach this, we're done downloading garbage
+		//
+		
+		// parse redirect values & create data shit
+		$keys = [];
+		$replacements = [];
+		
+		for($i=0; $i<count($this->handles); $i++){
+			
+			$headers =
+				curl_multi_getcontent(
+					$this->handles[$i]["handle"]
+				);
+			
+			if(
+				!preg_match(
+					'/^location: ?([^ ]+)/im',
+					$headers,
+					$link
+				)
+			){
+				
+				// failed to extract redirect location
+				continue;
+			}
+			
+			// theyre too retarded to encode shit properly apparently.
+			// this is the best i can do, it will still yield errors when searching for "higurashi"
+			// example: https://coccoc.com/composer/goto?url=0AWtXcuIKWPOkxvarGvy3GJ36XipRirzzqAvltMM5QUfB1RPu49LrNB9qmuAQyvvI0eCTwf-FpjRjxUN8XbruhA0rvCHNNlhVNEozMUqgRcjzsdhyUMCEgwLCXVTQkXnLXqrUw8celcIZdfmZ1geOvioUcg
+			// redirects to https://vi.wikipedia.org/wiki/Higurashi_no_Naku_Koro_ni#C%E1ch_ch?i
+			// notice the encoding errors after #
+			// retards
+			
+			$link = mb_convert_encoding($link[1], "UTF-8", "ISO-8859-1");
+			
+			if(
+				preg_match(
+					'/^(https?:|)\/\//',
+					$link
+				)
+			){
+				
+				$keys[] = $this->handles[$i]["replacement"];
+				$replacements[] = $link;
+			}
+		}
+		
+		$this->recursive_replace($keys, $replacements, $scraped_data);
+		
+		curl_multi_close($this->proc);
+	}
+	
+	private function recursive_replace(&$keys, &$replacements, &$pointer){
+		
+		foreach($pointer as $key => &$value){
+			
+			if($key == "url"){
+				
+				$search =
+					array_search(
+						$value,
+						$keys
+					);
+				
+				if($search !== false){
+					
+					// replace url
+					$value = $replacements[$search];
+				}
+				
+			}elseif(is_array($value)){
+				
+				$this->recursive_replace($keys, $replacements, $value);
+			}
+		}
+	}
+	
 	public function web($get){
 		
 		if($get["npt"]){
@@ -135,6 +367,7 @@ class coccoc{
 				$this->get(
 					$proxy,
 					"https://coccoc.com/search",
+					false,
 					$query
 				);
 		}catch(Exception $error){
@@ -143,6 +376,7 @@ class coccoc{
 		}
 		//$html = file_get_contents("scraper/coccoc.html");
 		
+		$html = $this->handle_redir($proxy, $html);
 		
 		$html = explode("window.composerResponse", $html, 2);
 		
@@ -313,7 +547,7 @@ class coccoc{
 								)
 							),
 						"date" => null,
-						"url" => $sub["url"]
+						"url" => $this->redirect_add_url($sub["url"])
 					];
 				}
 			}
@@ -380,7 +614,7 @@ class coccoc{
 							$result["content"]
 						)
 					),
-				"url" => $result["url"],
+				"url" => $this->redirect_add_url($result["url"]),
 				"date" => $date,
 				"type" => "web",
 				"thumb" => $thumb,
@@ -446,6 +680,8 @@ class coccoc{
 					}
 				}
 				
+				if(isset($wiki["profiles"])){
+					
 					foreach($wiki["profiles"] as $sitename => $url){
 						
 						$sitename = explode("_", $sitename);
@@ -453,12 +689,14 @@ class coccoc{
 						
 						$sublinks[$sitename] = $url;
 					}
+				}
 				
 				$out["answer"][] = [
 					"title" =>
+						isset($wiki["title"]) ?
 							$this->titledots(
 								$wiki["title"]
-						),
+							) : null,
 					"description" => $description,
 					"url" => null,
 					"thumb" => isset($wiki["image"]["contentUrl"]) ? $wiki["image"]["contentUrl"] : null,
@@ -491,6 +729,7 @@ class coccoc{
 				);
 		}
 		
+		$this->resolve_urls($out);
 		return $out;
 	}
 	
@@ -538,6 +777,7 @@ class coccoc{
 				$this->get(
 					$proxy,
 					"https://coccoc.com/search",
+					false,
 					$query
 				);
 		}catch(Exception $error){
@@ -545,6 +785,8 @@ class coccoc{
 			throw new Exception("Failed to get search page");
 		}
 		
+		$html = $this->handle_redir($proxy, $html);
+		
 		$html = explode("window.composerResponse", $html, 2);
 		
 		if(count($html) !== 2){
@@ -637,7 +879,7 @@ class coccoc{
 						(int)$video["rich"]["data"]["duration"] : null,
 				"views" => null,
 				"thumb" => $thumb,
-				"url" => $video["url"]
+				"url" => $this->redirect_add_url($video["url"])
 			];
 		}
 		
@@ -661,6 +903,7 @@ class coccoc{
 				);
 		}
 		
+		$this->resolve_urls($out);
 		return $out;
 	}
 	

scraper/ddg.php

@@ -29,14 +29,14 @@ class ddg{
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
 		// http2 bypass
-		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
+		curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2);
 		
 		switch($reqtype){
 			case self::req_web:
 				$headers =
 					["User-Agent: " . config::USER_AGENT,
 					"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
-					"Accept-Language: en-US,en;q=0.5",
+					"Accept-Language: en-US,en;q=0.9",
 					"Accept-Encoding: gzip",
 					"DNT: 1",
 					"Sec-GPC: 1",
@@ -46,8 +46,7 @@ class ddg{
 					"Sec-Fetch-Mode: navigate",
 					"Sec-Fetch-Site: same-origin",
 					"Sec-Fetch-User: ?1",
-					"Priority: u=0, i",
-					"TE: trailers"];
+					"Priority: u=0, i"];
 				break;
 			
 			case self::req_xhr:
@@ -371,6 +370,7 @@ class ddg{
 			
 			// we have $get["s"]
 			if(
+				strpos($get["s"], "email") !== false || // contains word "email" ?? lol
 				strpos($get["s"], "\"") !== false || // contains quotes
 				strpos($get["s"], ":") !== false // contains potential site: operator or whatever the fuck
 			){
@@ -484,7 +484,7 @@ class ddg{
 		$this->fuckhtml->load($html);
 		
 		//
-		// Get next page token
+		// Detect captcha
 		//
 		$forms =
 			$this->fuckhtml
@@ -509,8 +509,9 @@ class ddg{
 			
 			$input_probe =
 				$this->fuckhtml
-				->getElementsByClassName(
-					"btn--alt",
+				->getElementsByAttributeValue(
+					"value",
+					"Next",
 					"input"
 				);
 			

scraper/fivehpx.php

@@ -19,7 +19,33 @@ class fivehpx{
 				"option" => [
 					"relevance" => "Relevance",
 					"pulse" => "Pulse",
-					"newest" => "Newest"
+					"newest" => "Newest",
+					"aqs" => "AI Score"
+				]
+			],
+			"category" => [
+				"display" => "Category",
+				"option" => [
+					"any" => "Any category",
+					"landscape,nature" => "Landscape",
+					"city" => "City",
+					"events,sports" => "Events",
+					"creative" => "Creative"
+				]
+			],/*
+			"gear" => [
+				"display" => "Gear",
+				"option" => [
+					"any" => "Any gear",
+					"camera" => "Camera",
+					"phone" => "Phone"
+				]
+			],*/
+			"downloadable" => [
+				"display" => "Downloadable",
+				"option" => [
+					"any" => "Doesn't matter",
+					"yes" => "Yes"
 				]
 			]
 		];
@@ -44,7 +70,7 @@ class fivehpx{
 				["User-Agent: " . config::USER_AGENT,
 				"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
 				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"DNT: 1",
 				"Sec-GPC: 1",
 				"Connection: keep-alive",
@@ -60,19 +86,19 @@ class fivehpx{
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
 				["User-Agent: " . config::USER_AGENT,
-				"Accept: */*",
-				"Accept-Language: en-US,en;q=0.5",
-				"Accept-Encoding: gzip",
+				"Accept: application/graphql-response+json,application/json;q=0.9",
+				"Accept-Language: en-CA,en-US;q=0.9,en;q=0.8",
+				"Accept-Encoding: gzip, deflate, br, zstd",
 				"Referer: https://500px.com/",
 				"content-type: application/json",
-				//"x-csrf-token: undefined",
-				"x-500px-source: Search",
+				"x-500px-platform: Web",
 				"Content-Length: " . strlen($post_data),
 				"Origin: https://500px.com",
 				"DNT: 1",
 				"Sec-GPC: 1",
+				"Alt-Used: api-neo.500px.com",
 				"Connection: keep-alive",
-				// "Cookie: _pin_unauth, _fbp, _sharedID, _sharedID_cst",
+				"Cookie: x-500px-device-id-prod=a7b57e48-e477-4382-8c43-70c74415eb96",
 				"Sec-Fetch-Dest: empty",
 				"Sec-Fetch-Mode: cors",
 				"Sec-Fetch-Site: same-site",
@@ -111,14 +137,12 @@ class fivehpx{
 		
 		if($get["npt"]){
 			
-			[$pagination, $proxy] =
+			[$variables, $proxy] =
 				$this->backend->get(
 					$get["npt"], "images"
 				);
 			
-			$pagination = json_decode($pagination, true);
-			$search = $pagination["search"];
-			
+			$variables = json_decode($variables, true);
 		}else{
 			
 			$search = $get["s"];
@@ -128,12 +152,28 @@ class fivehpx{
 			}
 			
 			$proxy = $this->backend->get_ip();
-			$pagination = [
+			$variables = [
+				"type" => "TEXT",
 				"sort" => strtoupper($get["sort"]),
-				"search" => $search,
-				"filters" => [],
-				"nlp" => false,
+				"first" => 20,
+				"keyword" => $search,
+				"resourceTypes" => ["PHOTO"]
 			];
+			
+			if($get["downloadable"] != "any"){
+				
+				$variables["downloadable"] = "ALLOW";
+			}
+			
+			if($get["category"] != "any"){
+				
+				$variables["categories"] = [strtoupper($get["category"])];
+			}
+			/* for some reason this filter doesnt do anything? lol
+			if($get["gear"] != "any"){
+				
+				$variables["equipments"] = [strtoupper($get["gear"])];
+			}*/
 		}
 		
 		try{
@@ -141,17 +181,18 @@ class fivehpx{
 			$json =
 				$this->get(
 					$proxy,
-					"https://api.500px.com/graphql",
+					"https://api-neo.500px.com/graphql",
 					[],
 					json_encode([
-						"operationName" => "PhotoSearchPaginationContainerQuery",
-						"variables" => $pagination,
-						"query" =>
-							'query PhotoSearchPaginationContainerQuery(' .
-							(isset($pagination["cursor"]) ? '$cursor: String, ' : "") .
-							'$sort: PhotoSort, $search: String!, $filters: [PhotoSearchFilter!], $nlp: Boolean) {  ...PhotoSearchPaginationContainer_query_1vzAZD} fragment PhotoSearchPaginationContainer_query_1vzAZD on Query { photoSearch(sort: $sort, first: 100, ' .
-							(isset($pagination["cursor"]) ? 'after: $cursor, ' : "") .
-							'search: $search, filters: $filters, nlp: $nlp) { edges { node { id legacyId canonicalPath name description width height images(sizes: [33, 36]) { size url id } } } totalCount pageInfo { endCursor hasNextPage } }}'
+						"operationName" => "searchResource",
+						"variables" => $variables,
+						"extensions" => [
+							"clientLibrary" => [
+								"name" => "@apollo/client",
+								"version" => "4.1.6"
+							]
+						],
+						"query" => "query searchResource(\$keyword: String!, \$type: SearchType = TEXT, \$first: Int!, \$after: String, \$resourceTypes: [SearchResourceType!], \$categories: [String!], \$equipments: [String!], \$downloadable: PhotoDownloadable, \$sort: SearchSortOption = RELEVANCE, \$excludeNsfw: Boolean) {\n  searchResource(\n    keyword: \$keyword\n    type: \$type\n    first: \$first\n    after: \$after\n    resourceTypes: \$resourceTypes\n    categories: \$categories\n    equipments: \$equipments\n    downloadable: \$downloadable\n    sort: \$sort\n    excludeNsfw: \$excludeNsfw\n  ) {\n    edges {\n      cursor\n      node {\n        __typename\n        ... on Photo {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          questInfo {\n            rewardQuestName\n            __typename\n          }\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          description\n          downloadable\n          licensing {\n            status\n            __typename\n          }\n          urls {\n            size_600\n            size_1024\n            __typename\n          }\n          uploader {\n            id\n            username\n            avatar\n            displayName\n            isBlockedByMe\n            isFollowedByMe\n            isFollowingMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          isNsfw\n          isLikedByMe\n          viewerHasReposted\n          width\n          height\n          dominantColorLight\n          dominantColorDark\n          hasComment\n          uploadedAt\n          __typename\n        }\n        ... on Gallery {\n          id\n          isDeleted\n          isPrivate\n          kind\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          name\n          itemCount\n          hasComment\n          isLikedByMe\n          isNsfw\n          viewerHasReposted\n          creator {\n            id\n            username\n            displayName\n            avatar\n            isFollowedByMe\n            isFollowingMe\n            isBlockedByMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          coverPhotos(first: 5) {\n            id\n            urls {\n              size_600\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on PhotoGroup {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          description\n          numCount\n          publicItemCount\n          isLikedByMe\n          isNsfw\n          viewerHasReposted\n          uploader {\n            id\n            username\n            displayName\n            avatar\n            isFollowedByMe\n            isFollowingMe\n            isBlockedByMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          coverPhotos(first: 5) {\n            id\n            urls {\n              size_600\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on User {\n          id\n          username\n          displayName\n          avatar\n          location\n          isBlockedByMe\n          isFollowedByMe\n          isFollowingMe\n          membership {\n            membership\n            __typename\n          }\n          userCoverPhotos: coverPhotos(first: 6) {\n            id\n            urls {\n              size_1024\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n        ... on Video {\n          id\n          isDeleted\n          isInReview\n          isPrivate\n          questInfo {\n            rewardQuestName\n            __typename\n          }\n          honors {\n            ...PhotoHonorFields\n            __typename\n          }\n          title\n          coverUrl\n          videoUrl\n          width\n          height\n          isNsfw\n          isLikedByMe\n          uploader {\n            id\n            username\n            avatar\n            displayName\n            isBlockedByMe\n            isFollowedByMe\n            isFollowingMe\n            membership {\n              membership\n              __typename\n            }\n            __typename\n          }\n          __typename\n        }\n      }\n      __typename\n    }\n    pageInfo {\n      hasNextPage\n      endCursor\n      __typename\n    }\n    totalCount\n    __typename\n  }\n}\n\nfragment PhotoHonorFields on PhotoHonor {\n  __typename\n  ... on PhotoHo
Line truncated
 					])
 				);
 		}catch(Exception $error){
@@ -171,7 +212,7 @@ class fivehpx{
 			throw new Exception("500px returned an API error: " . $json["errors"][0]["message"]);
 		}
 		
-		if(!isset($json["data"]["photoSearch"]["edges"])){
+		if(!isset($json["data"]["searchResource"]["edges"])){
 			
 			throw new Exception("No edges returned by API");
 		}
@@ -182,14 +223,14 @@ class fivehpx{
 			"image" => []
 		];
 		
-		foreach($json["data"]["photoSearch"]["edges"] as $image){
+		foreach($json["data"]["searchResource"]["edges"] as $image){
 			
 			$image = $image["node"];
 			$title =
 				trim(
 					$this->fuckhtml
 					->getTextContent(
-						$image["name"]
+						$image["title"]
 					) . ": " .
 					$this->fuckhtml
 					->getTextContent(
@@ -199,38 +240,62 @@ class fivehpx{
 				);
 			
 			$small = $this->image_ratio(600, $image["width"], $image["height"]);
-			$large = $this->image_ratio(2048, $image["width"], $image["height"]);
+			//$large = $this->image_ratio(2048, $image["width"], $image["height"]);
+			$large = $this->image_ratio(3840, $image["width"], $image["height"]);
+			
+			// for some fuck ass reason the invalid signature error gives me the
+			// signature key they use, lol
+			// signStrPreview:
+			// 67gfdsgf!5uiyoi3fdsphoto/37070890/0.jpg21790132400
+			// 67gfdsgf!5uiyoi3fdsphoto/211496303/0.jpg111790132400
+			
+			// hash("sha256", <-key->photo/<-url id->/0.jpg<-size-><-expiry->)
+			// https://cdn-resize-prod-com.500px.cloud/photo/211496303/0.jpg?s=1&expiry=1790132400&sig=b2873066126c294d7ca0b6f74db7f2626d79d5337a4c71add910f699734fade2
+			
+			$images = [];
+			
+			preg_match(
+				'/photo\/([0-9]+)/',
+				$image["urls"]["size_600"],
+				$image_id
+			);
+			
+			if(count($image_id) !== 0){
+				
+				$expiry = time() + 3600;
+				
+				$images[] = [
+					"url" =>
+						"https://cdn-resize-prod-com.500px.cloud/photo/{$image_id[1]}/0.jpg?s=4&expiry={$expiry}&sig=" .
+						hash("sha256", "67gfdsgf!5uiyoi3fdsphoto/{$image_id[1]}/0.jpg4{$expiry}"),
+					"width" => $large[0] > $image["width"] ? $image["width"] : $large[0],
+					"height" => $large[1] > $image["height"] ? $image["height"] : $large[1]
+				];
+			}
+			
+			// add thumbnail
+			$images[] = [
+				"url" => $image["urls"]["size_600"],
+				"width" => $small[0],
+				"height" => $small[1]
+			];
 			
 			$out["image"][] = [
 				"title" => $title,
-				"source" => [
-					[
-						"url" => $image["images"][1]["url"],
-						"width" => $large[0],
-						"height" => $large[1]
-					],
-					[
-						"url" => $image["images"][0]["url"],
-						"width" => $small[0],
-						"height" => $small[1]
-					]
-				],
-				"url" => "https://500px.com" . $image["canonicalPath"]
+				"source" => $images,
+				"url" => "https://500px.com/photo/{$image["id"]}"
 			];
 		}
 		
 		// get NPT token
-		if($json["data"]["photoSearch"]["pageInfo"]["hasNextPage"] === true){
+		
+		if($json["data"]["searchResource"]["pageInfo"]["hasNextPage"] === true){
+			
+			$variables["after"] = $json["data"]["searchResource"]["pageInfo"]["endCursor"];
 			
 			$out["npt"] =
 				$this->backend->store(
-					json_encode([
-						"cursor" => $json["data"]["photoSearch"]["pageInfo"]["endCursor"],
-						"search" => $search,
-						"sort" => $pagination["sort"],
-						"filters" => [],
-						"nlp" => false
-					]),
+					json_encode($variables),
 					"images",
 					$proxy
 				);

scraper/google.php

@@ -9,6 +9,9 @@ class google{
 		
 		include "lib/backend.php";
 		$this->backend = new backend("google");
+		
+		$this->proc = curl_multi_init();
+		$this->handles = [];
 	}
 	
 	public function getfilters($page){
@@ -523,7 +526,7 @@ class google{
 					config::FPLAY_EXTERNAL_ENDPOINT .
 					"?url=" .				urlencode($url) .
 					"&pw=" .				urlencode(config::FPLAY_PASSWORD) .
-					"&container=" .			urlencode($container) .
+					"&container=" .//		urlencode($container) .
 					"&proxy=" .				urlencode($proxy) .
 					"&url_match=" .			urlencode("&sei=") .
 					"&fail_url_match=" .	urlencode("\/sorry\/");
@@ -614,6 +617,145 @@ class google{
 		];
 	}
 	
+	private function redirect_add_url($url){
+		
+		if(
+			preg_match(
+				'/^\/goto\?url=([^&]+)/',
+				$url,
+				$slug
+			) === 0
+		){
+			
+			// not a google redirect
+			return $url;
+		}
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		//$this->backend->assign_proxy($curlproc, $proxy);
+		
+		curl_multi_add_handle($this->proc, $curlproc);
+		$this->handles[] = [
+			"replacement" => $url,
+			"handle" => $curlproc
+		];
+		
+		return $url;
+	}
+	
+	private function resolve_urls(&$scraped_data){
+		
+		do{
+			do{
+				$status = curl_multi_exec($this->proc, $active);
+			}while($status === CURLM_CALL_MULTI_PERFORM);
+
+			if($active){
+				curl_multi_select($this->proc);
+			}
+
+		}while($active && $status === CURLM_OK);
+		
+		//
+		// if we reach this, we're done downloading garbage
+		//
+		
+		// parse redirect values & create data shit
+		$keys = [];
+		$replacements = [];
+		
+		for($i=0; $i<count($this->handles); $i++){
+			
+			$this->fuckhtml->load(
+				curl_multi_getcontent(
+					$this->handles[$i]["handle"]
+				)
+			);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"a"
+				);
+			
+			if(count($a) === 0){
+				
+				throw new Exception("Failed to find link");
+			}
+			
+			$link =
+				$this->fuckhtml
+				->getTextContent(
+					$a[0]["attributes"]["href"]
+				);
+			
+			if(
+				preg_match(
+					'/^(https?:|)\/\//',
+					$link
+				)
+			){
+				
+				$keys[] = $this->handles[$i]["replacement"];
+				$replacements[] = $link;
+			}
+		}
+		
+		$this->recursive_replace($keys, $replacements, $scraped_data);
+		
+		curl_multi_close($this->proc);
+	}
+	
+	private function recursive_replace(&$keys, &$replacements, &$pointer){
+		
+		foreach($pointer as $key => &$value){
+			
+			if($key == "url"){
+				
+				$search =
+					array_search(
+						$value,
+						$keys
+					);
+				
+				if($search !== false){
+					
+					// replace url
+					$value = $replacements[$search];
+				}
+				
+			}elseif(is_array($value)){
+				
+				$this->recursive_replace($keys, $replacements, $value);
+			}
+		}
+	}
 	
 	public function web($get){
 		
@@ -731,9 +873,9 @@ class google{
 			
 			$html = &$data["data"];
 			$container = &$data["container"];
-			
 			//file_put_contents("scraper/google-jp.html", $html);
-			//$html = file_get_contents("scraper/google-jp.html");
+			//$html = file_get_contents("scraper/google.html");
+			//$container = "";
 		}
 		
 		$out = [
@@ -754,9 +896,27 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		// init
-		$this->detect_sorry();
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+		}
+		
 		$this->parsestyles();
+		
 		$this->scrape_dimg($html);
 		$this->scrape_imagearr($html);
 		
@@ -887,6 +1047,8 @@ class google{
 				$this->fuckhtml
 				->getElementsByTagName("a");
 			
+			$illegal = false;
+			
 			foreach($as as $a){
 				
 				if(!isset($a["attributes"]["href"])){ continue; }
@@ -897,14 +1059,35 @@ class google{
 						$a["attributes"]["href"]
 					);
 				
+				$link_text =
+					strtolower(
+						$this->fuckhtml
+						->getTextContent($a)
+					);
+				
 				if(
-					preg_match(
-						'/^https?:\/\/lumendatabase\.org/',
-						$link
+					str_contains(
+						$link_text,
+						"complaint"
+					) ||
+					str_contains(
+						$link_text,
+						"request"
 					)
 				){
 					
-					$notices[] = $link;
+					if(
+						str_contains(
+							$link_text,
+							"read more about the request"
+						)
+					){
+						
+						$illegal = true;
+					}
+					
+					// sometimes the link is an encrypted redirect, handle redirection securely
+					$notices[] = $this->redirect_add_url($link);
 				}
 			}
 			
@@ -916,12 +1099,23 @@ class google{
 				$i = 0;
 				$c = count($notices);
 				
+				if($illegal === false){
+					
 					$notices_dom = [
 						[
 							"type" => "text",
 							"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
 						]
 					];
+				}else{
+					
+					$notices_dom = [
+						[
+							"type" => "text",
+							"value" => "Google removed results from this page because they were illegal. You can view the takedown requests here: "
+						]
+					];
+				}
 				
 				foreach($notices as $n){
 					
@@ -1231,10 +1425,39 @@ class google{
 			
 			$sublinks = [];
 			
+			// get sublinks (rare)
+			$as =
+				$this->fuckhtml
+				->getElementsByClassName(
+					$this->getstyle([
+						"padding-right" => "6px",
+						"margin-top" => "4px",
+						"display" => "inline-block"
+					]),
+					"a"
+				);
+			
+			foreach($as as $sublink){
+				
+				$sublink_url =
+					$this->fuckhtml
+					->getTextContent(
+						$sublink["attributes"]["href"]
+					);
+				
+				$sublinks[] = [
+					"title" =>
+						$this->fuckhtml
+						->getTextContent($sublink),
+					"date" => null,
+					"url" => $this->redirect_add_url($sublink_url)
+				];
+			}
+			
 			$out["web"][] = [
 				"title" => $title,
 				"description" => $description,
-				"url" => $url,
+				"url" => $this->redirect_add_url($url),
 				"date" => $date,
 				"type" => "web",
 				"thumb" => $thumb,
@@ -1243,6 +1466,8 @@ class google{
 			];
 		}
 		
+		$this->resolve_urls($out);
+		
 		return $out;
 	}
 	
@@ -1381,7 +1606,28 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get javascript images
 		$this->scrape_imagearr($html);
@@ -1432,10 +1678,12 @@ class google{
 					),
 				"source" => $source,
 				"url" =>
+					$this->redirect_add_url(
 						$this->fuckhtml
 						->getTextContent(
 							$div["attributes"]["data-lpage"]
 						)
+					)
 			];
 		}
 		
@@ -1459,6 +1707,8 @@ class google{
 				);
 		}
 		
+		$this->resolve_urls($out);
+		
 		return $out;
 	}
 	
@@ -1544,7 +1794,6 @@ class google{
 			}
 		}
 			
-		//$html = file_get_contents("scraper/google.html");		
 		$data = $this->get(
 			$proxy,
 			"https://www.google.com/search",
@@ -1555,6 +1804,8 @@ class google{
 		$html = &$data["data"];
 		$container = &$data["container"];
 		
+		//$html = file_get_contents("scraper/google.html");
+		
 		if(!isset($params["start"])){
 			
 			$params["start"] = 0;
@@ -1563,13 +1814,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
+		// parse all <style> tags
+		$this->parsestyles();
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
+		
 		//
 		// Parse web video page
 		//
-		$this->detect_sorry();
-		
-		// parse all <style> tags
-		$this->parsestyles();
 		
 		// get javascript images
 		$this->scrape_dimg($html);
@@ -1578,6 +1848,29 @@ class google{
 		
 		$params["c"] = $container;
 		
+		// get encoded links
+		$encoded_links =
+			explode(
+				"function(){var m={",
+				$html,
+				2
+			);
+		
+		if(count($encoded_links) === 2){
+			
+			$encoded_links =
+				json_decode(
+					$this->fuckhtml
+					->extract_json(
+						"{" . $encoded_links[1]
+					),
+					true
+				);
+		}else{
+			
+			$encoded_links = [];
+		}
+		
 		$out = [
 			"status" => "ok",
 			"npt" =>
@@ -1756,6 +2049,50 @@ class google{
 				}
 			}
 			
+			$url =
+				$this->fuckhtml
+					->getTextContent(
+						$url[0]["attributes"]["href"]
+					);
+			
+			if(
+				preg_match(
+					'/^\/goto\?url/',
+					$url
+				)
+			){
+				
+				// encrypted url detected
+				// get reference div
+				$refdivs =
+					$this->fuckhtml
+					->getElementsByAttributeName(
+						"id",
+						"div"
+					);
+				
+				foreach($refdivs as $r){
+					
+					if(
+						preg_match(
+							'/^atritem-/',
+							$r["attributes"]["id"]
+						) &&
+						isset($r["attributes"]["jsdata"])
+					){
+						
+						$ref = explode(";", $r["attributes"]["jsdata"]);
+						$ref = $ref[count($ref) - 1];
+						
+						if(isset($encoded_links[$ref][32][11][7])){
+							
+							$url = $encoded_links[$ref][32][11][7];
+						}
+						break;
+					}
+				}
+			}
+			
 			$out["video"][] = [
 				"title" =>
 					$this->titledots(
@@ -1774,14 +2111,12 @@ class google{
 				"duration" => $duration,
 				"views" => null,
 				"thumb" => $thumb,
-				"url" =>
-					$this->fuckhtml
-					->getTextContent(
-						$url[0]["attributes"]["href"]
-					)
+				"url" => $this->redirect_add_url($url)
 			];
 		}
 		
+		$this->resolve_urls($out);
+		
 		return $out;
 	}
 	
@@ -1887,14 +2222,32 @@ class google{
 		
 		$this->fuckhtml->load($html);
 		
-		$this->detect_sorry();
+		// parse styles
+		$this->parsestyles();		
+		
+		// solve softcaptcha
+		$softcaptcha = $this->detect_sorry();
+		
+		if($softcaptcha !== false){
+			
+			$params["sei"] = $softcaptcha["sei"];
+			
+			$data2 = $this->get(
+				$proxy,
+				"https://www.google.com/search",
+				$params,
+				$container
+			);
+			
+			$html = &$data2["data"];
+						
+			$this->fuckhtml->load($html);
+			$this->parsestyles();
+		}
 		
 		// get images
 		$this->scrape_dimg($html);
 		
-		// parse styles
-		$this->parsestyles();
-		
 		$center_col =
 			$this->fuckhtml
 			->getElementById(
@@ -2128,6 +2481,8 @@ class google{
 			];
 		}
 		
+		$this->resolve_urls($out);
+		
 		return $out;
 	}
 
@@ -2653,6 +3008,7 @@ class google{
 	
 	private function detect_sorry(){
 		
+		// detect recaptcha
 		$captcha_form =
 			$this->fuckhtml
 			->getElementById(
@@ -2682,5 +3038,39 @@ class google{
 			
 			throw new Exception("Google returned a JS challenge");
 		}
+		
+		// bypass soft captcha
+		// "Verifying your request
+		// You'll be able to continue in a few seconds. Don't refresh this page."
+		$scripts =
+			$this->fuckhtml
+			->getElementsByTagName(
+				"script"
+			);
+		
+		foreach($scripts as $sc){
+			
+			if(
+				str_contains(
+					str_replace(" ", "", $this->fuckhtml->getloadedhtml()),
+					"setTimeout(function(){if(spinner){spinner.style.visibility='hidden';}"
+				) &&
+				preg_match(
+					'/var eid ?= ?\'([^\']*)\'/',
+					$sc["innerHTML"],
+					$matches
+				)
+			){
+				
+				sleep(10); // gotta sleep to not trigger captcha. blame google, not me
+				
+				return [
+					"sei" => $matches[1],
+					"timeout" => 10000 // set to 10000 usually
+				];
+			}
+		}
+		
+		return false; // no need to bypass
 	}
 }

scraper/mojeek.php

@@ -16,18 +16,7 @@ class mojeek{
 			
 			case "web":
 				return [
-					"focus" => [
-						"display" => "Focus",
-						"option" => [
-							"any" => "No focus",
-							"blogs" => "Blogs",
-							"Dictionary" => "Dictionary",
-							"Recipes" => "Recipes",
-							"Time" => "Time",
-							"Weather" => "Weather"
-						]
-					],
-					"lang" => [
+					"lang" => [ // lb
 						"display" => "Language",
 						"option" => [
 							"any" => "Any language",
@@ -86,7 +75,7 @@ class mojeek{
 							"zu" => "Zulu"
 						]
 					],
-					"country" => [
+					"country" => [ // rb
 						"display" => "Country",
 						"option" => [
 							"any" => "No location bias",
@@ -340,28 +329,21 @@ class mojeek{
 							"zm" => "Zambia",
 							"zw" => "Zimbabwe"
 						]
-					],/*
-					"region" => [
-						"display" => "Region",
+					],
+					"sort" => [
+						"display" => "Sort by",
 						"option" => [
-							"any" => "Any region",
-							"eu" => "European Union",
-							"de" => "Germany",
-							"fr" => "France",
-							"uk" => "United Kingdom"
-						]
-					],*/
-					"domain" => [
-						"display" => "Results per domain",
-						"option" => [
-							"1" => "1 result",
-							"2" => "2 results",
-							"3" => "3 results",
-							"4" => "4 results",
-							"5" => "5 results",
-							"10" => "10 results",
-							"0" => "Unlimited",
+							"relevance" => "Relevance",
+							"date" => "Date" // datewr=100
 						]
+					],
+					"newer" => [
+						"display" => "Newer than",
+						"option" => "_DATE"
+					],
+					"older" => [
+						"display" => "Older than",
+						"option" => "_DATE"
 					]
 				];
 				break;
@@ -371,7 +353,14 @@ class mojeek{
 		}
 	}
 	
-	private function get($proxy, $url, $get = []){
+	private function get($proxy, $url, $get = [], $is_api = false){
+		
+		if($is_api){
+						
+			$headers = [
+				"Accept: application/json"
+			];
+		}else{
 			
 			$headers = [
 				"User-Agent: " . config::USER_AGENT,
@@ -386,6 +375,7 @@ class mojeek{
 				"Sec-Fetch-Site: none",
 				"Sec-Fetch-User: ?1"
 			];
+		}
 		
 		$curlproc = curl_init();
 		
@@ -420,6 +410,154 @@ class mojeek{
 	
 	public function web($get){
 		
+		return $this->web_api($get);
+	}
+	
+	public function web_api($get){
+				
+		$key = $this->backend->get_key();
+		$proxy = $this->backend->get_ip($key["increment"]);
+		
+		if($get["npt"]){
+			
+			[$params, $proxy] = $this->backend->get($get["npt"], "web");
+			$params = json_decode($params, true);
+			$params["api_key"] = $key["key"];
+			
+		}else{
+			
+			if(strlen($get["s"]) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			$params = [
+				"api_key" => $key["key"],
+				"q" => $get["s"],
+				"date" => 1,
+				"tlen" => 127, // title length
+				"dlen" => 511, // description length
+				"fmt" => "json"
+			];
+			
+			if($get["lang"] != "any"){
+				
+				$params["lb"] = $get["lang"];
+			}
+			
+			if($get["country"] != "any"){
+				
+				$params["rb"] = $get["country"];
+			}
+			
+			if($get["sort"] == "date"){
+				
+				$params["datewr"] = 100;
+			}
+			
+			if($get["newer"] !== false){
+				
+				$params["since"] = date("Ymd", $get["newer"]);
+			}
+			
+			if($get["older"] !== false){
+				
+				$params["before"] = date("Ymd", $get["before"]);
+			}
+		}
+		
+		try{
+			$json =
+				$this->get(
+					$proxy,
+					"https://api.mojeek.com/search",
+					$params
+				);
+		}catch(Exception $error){
+			
+			throw new Exception("Failed to get JSON");
+		}
+		
+		$json = json_decode($json, true);
+		
+		if($json === null){
+			
+			throw new Exception("Failed to decode JSON");
+		}
+		
+		$out = [
+			"status" => "ok",
+			"spelling" => [
+				"type" => "no_correction",
+				"using" => null,
+				"correction" => null
+			],
+			"npt" => null,
+			"answer" => [],
+			"web" => [],
+			"image" => [],
+			"video" => [],
+			"news" => [],
+			"related" => []
+		];
+		
+		if(!isset($json["response"]["status"])){
+			
+			throw new Exception("API did not return a status field");
+		}
+		
+		if($json["response"]["status"] != "OK"){
+			
+			throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
+		}
+		
+		if(!isset($json["response"]["results"])){
+			
+			throw new Exception("Mojeek API did not return a results object");
+		}
+		
+		foreach($json["response"]["results"] as $entry){
+			
+			$out["web"][] = [
+				"title" => html_entity_decode($entry["title"]),
+				"description" => html_entity_decode($entry["desc"]),
+				"url" => $entry["url"],
+				"date" => $entry["timestamp"],
+				"type" => "web",
+				"thumb" => [
+					"url" => null,
+					"ratio" => null
+				],
+				"sublink" => [],
+				"table" => []
+			];
+		}
+		
+		// get npt
+		if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
+			
+			unset($params["api_key"]);
+			if(!isset($params["s"])){
+				
+				$params["s"] = 11;
+			}else{
+				
+				$params["s"] = $params["s"] + 10;
+			}
+			
+			$out["npt"] =
+				$this->backend->store(
+					json_encode($params),
+					"web",
+					$proxy
+				);
+		}
+		
+		return $out;
+	}
+	
+	public function web_browser($get){
+		
 		if($get["npt"]){
 			
 			[$token, $proxy] = $this->backend->get($get["npt"], "web");
@@ -457,7 +595,6 @@ class mojeek{
 				"tn" => 7, // number of news results/page
 				"date" => 1, // show date
 				"tlen" => 128, // max length of title
-				//"dlen" => 511, // max length of description
 				//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
 			];
 			
@@ -1000,7 +1137,12 @@ class mojeek{
 		return $out;
 	}
 	
-	public function news($get){
+	public function news(){
+		
+		throw new Exception("Currently unsupported");
+	}
+	
+	public function news_browser($get){
 		
 		$search = $get["s"];
 		

scraper/startpage.php

@@ -9,6 +9,10 @@ class startpage{
 		
 		include "lib/fuckhtml.php";
 		$this->fuckhtml = new fuckhtml();
+		
+		include "lib/anubis.php";
+		$this->anubis = new anubis();
+		$this->anubis_error = "all gucci";
 	}
 	
 	public function getfilters($page){
@@ -244,7 +248,7 @@ class startpage{
 		}
 	}
 	
-	private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
+	private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
 		
 		$curlproc = curl_init();
 		
@@ -259,6 +263,27 @@ class startpage{
 			$url .= "?" . $get;
 		}
 		
+		if($return_cookies){
+			
+			$cookies = [];
+			curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
+				
+				$length = strlen($header);
+				
+				$header = explode(":", $header, 2);
+				
+				if(trim(strtolower($header[0])) == "set-cookie"){
+					
+					$cookie_tmp = explode("=", trim($header[1]), 2);
+					
+					$cookies[trim($cookie_tmp[0])] =
+						explode(";", $cookie_tmp[1], 2)[0];
+				}
+				
+				return $length;
+			});
+		}
+		
 		curl_setopt($curlproc, CURLOPT_URL, $url);
 		
 		// http2 bypass
@@ -266,6 +291,15 @@ class startpage{
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
 		
+		$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
+		
+		foreach($passed_cookies as $name => $val){
+			
+			$header_cookies[] = "$name=$val";
+		}
+		
+		$header_cookies = rtrim(implode("; ", $header_cookies));
+		
 		if($is_xhr === true){
 			
 			curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -279,7 +313,7 @@ class startpage{
 				"Origin: https://www.startpage.com/",
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Sec-Fetch-Dest: empty",
 				"Sec-Fetch-Mode: cors",
 				"Sec-Fetch-Site: same-origin",
@@ -298,7 +332,7 @@ class startpage{
 				"Content-Length: " . strlen($get),
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Upgrade-Insecure-Requests: 1",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
@@ -316,7 +350,7 @@ class startpage{
 				"Accept-Encoding: gzip",
 				"DNT: 1",
 				"Connection: keep-alive",
-				"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
+				"Cookie: $header_cookies",
 				"Sec-Fetch-Dest: document",
 				"Sec-Fetch-Mode: navigate",
 				"Sec-Fetch-Site: none",
@@ -342,6 +376,15 @@ class startpage{
 		}
 		
 		curl_close($curlproc);
+		
+		if($return_cookies){
+			
+			return [
+				"cookies" => $cookies,
+				"data" => $data
+			];
+		}
+		
 		return $data;
 	}
 	
@@ -349,14 +392,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$this->anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "web");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
-					true
+					$query,
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -398,7 +449,11 @@ class startpage{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$params
+					$params,
+					false,
+					false,
+					true,
+					[]
 				);
 			}catch(Exception $error){
 				
@@ -406,12 +461,41 @@ class startpage{
 			}
 			
 			//$html = file_get_contents("scraper/startpage.html");
+			
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
+			
+			// solve anubis if need be
+			$cookies = $this->solve_anubis($proxy, $html);
+			
+			if($cookies === false){
+				
+				$cookies = [];
+				$html = $html["data"];
+			}else{
+				
+				try{
+					$html = $this->get(
+						$proxy,
+						"https://www.startpage.com/sp/search",
+						$params,
+						false,
+						false,
+						false,
+						$cookies
+					);
+				}catch(Exception $error){
+					
+					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
+				}
+			}
 		}
 		
-		//echo htmlspecialchars($html);
-		
+		// detect captcha even if we passed anubis challenge
 		$this->detect_captcha($html);
 		
+		//echo htmlspecialchars($html);
+		
 		if(
 			preg_match(
 				'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
@@ -420,14 +504,14 @@ class startpage{
 			) === 0
 		){
 			
-			throw new Exception("Failed to grep JSON object");
+			throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$json = json_decode($matches[1], true);
 		
 		if($json === null){
 			
-			throw new Exception("Failed to decode JSON");
+			throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
 		}
 		
 		//print_r($json);
@@ -449,7 +533,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "web", $proxy);
+		$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
 		
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
@@ -658,14 +742,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$this->anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "images");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
-					true
+					$query,
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -680,8 +772,6 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
-			try{
-				
 			$proxy = $this->backend->get_ip();
 			
 			$params = [
@@ -739,17 +829,45 @@ class startpage{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-						$params
+					$params,
+					false,
+					false,
+					true,
+					[]
 				);
 			}catch(Exception $error){
 				
 				throw new Exception("Failed to fetch search page");
 			}
+			
 			//$html = file_get_contents("scraper/startpage.html");
 			
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
+			
+			// solve anubis if need be
+			$cookies = $this->solve_anubis($proxy, $html);
+			
+			if($cookies === false){
+				
+				$cookies = [];
+				$html = $html["data"];
+			}else{
+				
+				try{
+					$html = $this->get(
+						$proxy,
+						"https://www.startpage.com/sp/search",
+						$params,
+						false,
+						false,
+						false,
+						$cookies
+					);
 				}catch(Exception $error){
 					
-				throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
+				}
 			}
 		}
 		
@@ -769,18 +887,18 @@ class startpage{
 			) === 0
 		){
 			
-			throw new Exception("Failed to grep JSON object");
+			throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$json = json_decode($matches[1], true);
 		
 		if($json === null){
 			
-			throw new Exception("Failed to decode JSON object");
+			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "images", $proxy);
+		$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
 		
 		// get images
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -819,14 +937,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$this->anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "videos");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
-					true
+					$query,
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -841,8 +967,6 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
-			try{
-				
 			$proxy = $this->backend->get_ip();
 			
 			$params = [
@@ -870,17 +994,45 @@ class startpage{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-						$params
+					$params,
+					false,
+					false,
+					true,
+					[]
 				);
 			}catch(Exception $error){
 				
 				throw new Exception("Failed to fetch search page");
 			}
+			
 			//$html = file_get_contents("scraper/startpage.html");
 			
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
+			
+			// solve anubis if need be
+			$cookies = $this->solve_anubis($proxy, $html);
+			
+			if($cookies === false){
+				
+				$cookies = [];
+				$html = $html["data"];
+			}else{
+				
+				try{
+					$html = $this->get(
+						$proxy,
+						"https://www.startpage.com/sp/search",
+						$params,
+						false,
+						false,
+						false,
+						$cookies
+					);
 				}catch(Exception $error){
 					
-				throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
+				}
 			}
 		}
 		
@@ -894,14 +1046,14 @@ class startpage{
 			) === 0
 		){
 			
-			throw new Exception("Failed to get JSON object");
+			throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$json = json_decode($matches[1], true);
 		
 		if($json === null){
 			
-			throw new Exception("Failed to decode JSON object");
+			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$out = [
@@ -915,7 +1067,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "video", $proxy);
+		$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
 		
 		// get results
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -971,14 +1123,22 @@ class startpage{
 		
 		if($get["npt"]){
 			
+			$this->anubis_error = "Not ran on pagination";
 			[$post, $proxy] = $this->backend->get($get["npt"], "news");
 			
+			$post = json_decode($post, true);
+			$query = $post["query"];
+			$cookies = $post["cookies"];
+			
 			try{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-					$post,
-					true
+					$query,
+					true,
+					false,
+					false,
+					$cookies
 				);
 			}catch(Exception $error){
 				
@@ -993,8 +1153,6 @@ class startpage{
 				throw new Exception("Search term is empty!");
 			}
 			
-			try{
-				
 			$proxy = $this->backend->get_ip();
 			
 			$params = [
@@ -1017,17 +1175,45 @@ class startpage{
 				$html = $this->get(
 					$proxy,
 					"https://www.startpage.com/sp/search",
-						$params
+					$params,
+					false,
+					false,
+					true,
+					[]
 				);
 			}catch(Exception $error){
 				
-					throw new Exception("Failed to fetch search page");
+				throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
 			}
+			
 			//$html = file_get_contents("scraper/startpage.html");
 			
+			// detect captcha on that page
+			$this->detect_captcha($html["data"]);
+			
+			// solve anubis if need be
+			$cookies = $this->solve_anubis($proxy, $html);
+			
+			if($cookies === false){
+				
+				$cookies = [];
+				$html = $html["data"];
+			}else{
+				
+				try{
+					$html = $this->get(
+						$proxy,
+						"https://www.startpage.com/sp/search",
+						$params,
+						false,
+						false,
+						false,
+						$cookies
+					);
 				}catch(Exception $error){
 					
-				throw new Exception("Failed to fetch search page");
+					throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
+				}
 			}
 		}
 		
@@ -1041,14 +1227,14 @@ class startpage{
 			) === 0
 		){
 			
-			throw new Exception("Failed to get JSON object");
+			throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$json = json_decode($matches[1], true);
 		
 		if($json === null){
 			
-			throw new Exception("Failed to decode JSON object");
+			throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
 		}
 		
 		$out = [
@@ -1058,7 +1244,7 @@ class startpage{
 		];
 		
 		// get npt
-		$out["npt"] = $this->parse_npt($json, "news", $proxy);
+		$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
 		
 		foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
 			
@@ -1101,7 +1287,7 @@ class startpage{
 		return $out;
 	}
 	
-	private function parse_npt($json, $pagetype, $proxy){
+	private function parse_npt($json, $pagetype, $proxy, $cookies){
 		
 		foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
 			
@@ -1109,6 +1295,8 @@ class startpage{
 				
 				return
 					$this->backend->store(
+						json_encode([
+							"query" =>
 								http_build_query(
 									[
 										"lui" => "english",
@@ -1116,15 +1304,18 @@ class startpage{
 										"sc" => $json["render"]["search_sc"],
 										"t" => "device",
 										"cat" => $pagetype,
-								"segment" => "organic",
+										"segment" => "startpage.udog", // "original" sometimes
 										"abd" => "0",
 										"abe" => "0",
 										"query" => rawurldecode($json["render"]["query"]),
 										"page" => $page["number"],
 										"qsr" => "all",
-								"qadf" => "none" // @ todo fix (??)
+										"qadf" => "none", // @ todo fix (??)
+										...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
 									]
 								),
+							"cookies" => $cookies
+						]),
 						$pagetype,
 						$proxy
 					);
@@ -1270,4 +1461,56 @@ class startpage{
 			throw new Exception("Startpage redirected the scraper to an unhandled page");
 		}
 	}
+	
+	private function solve_anubis($proxy, $html){
+		
+		//
+		// solve anubis challenge, if theres one
+		//
+		$res = null;
+		$cookies = [];
+		
+		try{
+			$res = $this->anubis->scrape($html["data"]);
+			
+		}catch(Exception $error){
+			
+			// do nothang
+			$this->anubis_error = $error->getMessage();
+		}
+		
+		if($res !== null){
+			
+			// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
+			// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
+			// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
+			// &nonce=3098
+			// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
+			// &elapsedTime=111
+			
+			$anubis_response =
+				$this->get(
+					$proxy,
+					"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
+					[
+						"id" => $res["id"],
+						"response" => $res["response"],
+						"nonce" => $res["nonce"],
+						"redir" => "https://www.startpage.com/sp/search",
+						"elapsedTime" => $res["elapsedTime"]
+					],
+					false,
+					false,
+					true,
+					$html["cookies"]
+				);
+			
+			// re-fetch search page AGAIN with the anubis bullshit
+			$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
+			
+			return $cookies;
+		}
+		
+		return false;
+	}
 }

settings.php

@@ -87,10 +87,6 @@ $settings = [
 						"value" => "kagi",
 						"text" => "Kagi"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -145,14 +141,6 @@ $settings = [
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
-					[
-						"value" => "ghostery",
-						"text" => "Ghostery"
-					],
 					[
 						"value" => "yep",
 						"text" => "Yep"
@@ -231,10 +219,6 @@ $settings = [
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
 					[
 						"value" => "naver",
 						"text" => "Naver"
@@ -251,10 +235,6 @@ $settings = [
 						"value" => "pinterest",
 						"text" => "Pinterest"
 					],
-					[
-						"value" => "cara",
-						"text" => "Cara"
-					],
 					[
 						"value" => "flickr",
 						"text" => "Flickr"
@@ -329,10 +309,6 @@ $settings = [
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
 					[
 						"value" => "naver",
 						"text" => "Naver"
@@ -379,14 +355,10 @@ $settings = [
 						"value" => "startpage",
 						"text" => "Startpage"
 					],
-					[
-						"value" => "qwant",
-						"text" => "Qwant"
-					],
-					[
+					/*[
 						"value" => "mojeek",
 						"text" => "Mojeek"
-					],
+					],*/
 					[
 						"value" => "baidu",
 						"text" => "Baidu"

static/serverping.js

@@ -22,7 +22,7 @@ var list = [];
 var pinged_list = [];
 var reqs = 0;
 var errors = 0;
-var sort = 6; // highest version first
+var sort = 12;
 
 // check for instance redirect stuff
 var redir = [];
@@ -68,8 +68,8 @@ table.innerHTML =
 		'<th class="extend">Server</th>' +
 		'<th>Address</th>' +
 		'<th>Bot protection</th>' +
-		'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
-		'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
+		'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
+		'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
 		'<th>API</th>' +
 		'<th><div class="arrow up"></div>Version</th>' +
 	'</tr>' +

template/donate.html

@@ -1,9 +1,18 @@
 <a href="/" class="link">&lt; Go back</a>
 
 <h1>Donate to the project</h1>
-This project takes up most of my free time and costs money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
+This project takes up most of my free time and costs (a lot of) money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
 
-To run smoothly, 4get requires 20$/month for the server fees and 15-25$ for residential proxies. I also have plans to build my own index to provide better search results; funds raised here will go directly towards better hardware for these purposes. According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> serves between 800-1400 users every day, or around 7000-10000 searches!
+To run smoothly, 4get requires:
+<ul>
+	<li>$350/year for server hosting at OVH</li>
+	<li>$60-80/month for proxies</li>
+	<li>$54/month for home internet</li>
+</ul>
+
+I also bought a dedicated machine for browser rendering hosted at home coming at around $380.<br><br>
+
+According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> currently serves between <a href="https://4get.ca/ami4get">2000-2500 users every day</a>, and serves around <a href="https://4get.ca/api/v2/graph">20,000-22,000 searches</a>!
 
 <h2>Ko-fi</h2>
 This is the most convenient way to donate. Supports PayPal.
@@ -17,4 +26,4 @@ Due to popular demand, I have added an XMR addy. I'm planning to reward users wh
 	<li><a href="monero:85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB?tx_description=4get+donation">85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB</a></li>
 </ul>
 
-Thank you all for your generous support!!
+I'm immensely grateful for every donation. Thank you all for your generous support!!

template/header.html

@@ -15,7 +15,7 @@
 		<div class="navigation">
 			<a href="/">Home</a>
 			<a href="/settings">Settings</a>
-			<a href="https://git.lolcat.ca/lolcat/4get_news" target="_BLANK">News</a>
+			<a href="https://bsky.app/profile/4getisdown.bsky.social" target="_BLANK">News</a>
 		</div>
 		<form method="GET" autocomplete="off">
 			<div class="searchbox">

template/home.html

@@ -28,7 +28,7 @@
 					<div class="autocomplete"></div>
 				</div>
 			</form>
-			<a href="settings">Settings</a> • <a href="instances">Instances</a> • <a href="https://git.lolcat.ca/lolcat/4get_news">News</a> • <a href="api.txt">API</a> • <a href="about">About</a> • <a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
+			<a href="settings">Settings</a> • <a href="instances">Instances</a> • <a href="https://bsky.app/profile/4getisdown.bsky.social">News</a> • <a href="api.txt">API</a> • <a href="about">About</a> • <a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
 			<a href="/donate"><b>Donate to the project</b></a>
 			<div class="subtext">
 				<a href="https://4get.ca">Clearnet</a> • <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">Tor</a> • <a href="https://lolcat.ca">Report a problem</a><br>

videos.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->video($get);
 	

web.php

@@ -26,6 +26,9 @@ $payload = [
 	"left" => ""
 ];
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	$results = $scraper->web($get);
 	
@@ -34,35 +37,6 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "web", $payload["timetaken"]);
 }
 
-/*
-	Prepend Oracle output, if applicable
-*/
-include("oracles/encoder.php");
-include("oracles/calc.php");
-include("oracles/time.php");
-include("oracles/numerics.php");
-$oracles = [new calculator(), new encoder(), new time(), new numerics()];
-$fortune = "";
-foreach ($oracles as $oracle) {
-	if ($oracle->check_query($_GET["s"])) {
-		$resp = $oracle->generate_response($_GET["s"]);
-		if ($resp != "") {
-			$fortune .= "<div class=\"infobox\">";
-			foreach ($resp as $title => $r) {
-				if ($title) {
-					$fortune .= "<h3>".htmlspecialchars($title)."</h3><div class=\"code\">".htmlspecialchars($r)."</div>";
-				}
-				else {
-					$fortune .= "<i>".$r."</i><br>";
-				}
-			}
-			$fortune .= "<small>Answer provided by oracle: ".$oracle->info["name"]."</small></div>";
-		}
-		break;
-	}
-}
-$payload["left"] = $fortune;
-
 $answerlen = 0;
 
 /*