rotate ips on url resolution

Reviewed-on: #98

api/v1/ac.php

@@ -163,6 +163,8 @@ class autocomplete{
 				// if it respects the openSearch protocol
 				$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
 				
+				if(!isset($json[1])){ $json[1] = []; }
+				
 				echo json_encode(
 					[
 						$_GET["s"],

data/config.php

@@ -168,6 +168,7 @@ class config{
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
 	const PROXY_GOOGLE = false;
+	const PROXY_GOOGLE_ALT = false; // alt proxies for url resolution
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
 	const PROXY_STARTPAGE = false;

docs/apache2-example.md

@@ -1,31 +1,16 @@
 # Sample Apache2 configuration
 This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
 
+Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
+
 Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
 
 ```xml
-<VirtualHost *:443>
+<VirtualHost *:8080>
-	ServerName www.4get.ca
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://4get.ca$1
-</VirtualHost>
-
-<VirtualHost *:443>
 	ServerName 4get.ca
+	ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
 
-	ServerAdmin will@lolcat.ca
+	# compress
-	DocumentRoot /var/www/4get
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
 	AddOutputFilterByType DEFLATE application/json
 	AddOutputFilterByType DEFLATE application/javascript
 	AddOutputFilterByType DEFLATE application/x-javascript
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	AddOutputFilterByType DEFLATE text/plain
 	AddOutputFilterByType DEFLATE text/css
 	
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
+	DocumentRoot /var/www/4get
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
+	ErrorLog ${APACHE_LOG_DIR}/error.log
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	CustomLog /dev/null combined
+	#CustomLog /var/log/apache2/access.log combined
 	
+	<FilesMatch \.php$>
+		SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
+	</FilesMatch>
+
 	<Directory /var/www/4get>
 		Options -MultiViews
 		AllowOverride All
 		Require all granted
 		
+		# prevent 404s from hitting php-fpm
 		RewriteEngine On
+		RewriteCond %{REQUEST_URI} \.php$ [NC]
+		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteRule ^ - [R=404,L]
+		
+		# prevent /index hits
+		RewriteRule ^index$ - [R=404,L]
+		
+		# 404 direct .php file access
+		RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
+		RewriteRule ^ - [R=404,L]
+		
+		# internally redirect to non-php extension
 		RewriteCond %{REQUEST_FILENAME} !-d
 		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteCond %{REQUEST_FILENAME}.php -f
 		RewriteRule ^([^\.]+)$ $1.php [NC,L]
 	</Directory>
 
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
 		Require all denied
 	</DirectoryMatch>
-
+	
-	# remove directory listing from /icons
 	<Directory /var/www/4get/icons>
 		Options -Indexes
 	</Directory>
 </VirtualHost>
-
+```
-<VirtualHost *:443>
+
-	ServerName www.lolcat.ca
+# Haproxy configuration
-	
+
-	SSLEngine On
+```cfg
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
+global
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
+	#log /dev/log	local0
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	log /dev/log	local0 warning
-	
+	log /dev/log	local1 notice
-	RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
+	chroot /var/lib/haproxy
-</VirtualHost>
+	stats socket /run/haproxy/admin.sock mode 660 level admin
-
+	stats timeout 30s
-<VirtualHost *:443>
+	user haproxy
-	ServerName lolcat.ca
+	group haproxy
-
+	daemon
-	ServerAdmin will@lolcat.ca
+
-	DocumentRoot /var/www/lolcat
+	# Default SSL material locations
-	
+	ca-base /etc/ssl/certs
-	SSLEngine On
+	crt-base /etc/ssl/private
-	SSLOptions +StdEnvVars	
+
-	
+	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
+	#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
-	
+	ssl-default-bind-ciphers HIGH:!aNULL
-	AddOutputFilterByType DEFLATE application/json
+	ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
-	AddOutputFilterByType DEFLATE application/javascript
+	ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
-	AddOutputFilterByType DEFLATE application/x-javascript
+	
-	AddOutputFilterByType DEFLATE text/html
+	# ja4 settings
-	AddOutputFilterByType DEFLATE text/plain
+	tune.lua.bool-sample-conversion normal
-	AddOutputFilterByType DEFLATE text/css
+	tune.ssl.capture-buffer-size 190
-
+	lua-load /etc/haproxy/lua/ja4.lua
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
+defaults
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
+	log	global
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	mode	http
-
+	#option	httplog
-	<Directory /var/www/lolcat>
+	option	dontlognull
-		Options -MultiViews
+	timeout connect 5000
-		AllowOverride All
+	timeout client  50000
-		Require all granted
+	timeout server  50000
-		
+	errorfile 400 /etc/haproxy/errors/400.http
-		RewriteEngine On
+	errorfile 403 /etc/haproxy/errors/403.http
-		RewriteCond %{REQUEST_FILENAME} !-d
+	errorfile 408 /etc/haproxy/errors/408.http
-		RewriteCond %{REQUEST_FILENAME} !-f
+	errorfile 500 /etc/haproxy/errors/500.http
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
+	errorfile 502 /etc/haproxy/errors/502.http
-	</Directory>
+	errorfile 503 /etc/haproxy/errors/503.http
-</VirtualHost>
+	errorfile 504 /etc/haproxy/errors/504.http
-
+
-<VirtualHost *:443>
+frontend http_80_in
-	ServerName www.nyym.co
+	# bind tcp ip4/6
-	
+	bind *:80
-	SSLEngine On
+	bind [::]:80
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
+	
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
+	mode http
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
+	
-	
+	# handle ip.lolcat.ca+onion addy on plain HTTP
-	RedirectMatch 301 ^(.*)$ https://nyym.co$1
+	acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
-</VirtualHost>
+		
-
+	# redirect to https
-<VirtualHost *:443>
+	http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
-	ServerName nyym.co
+	http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
-
+	http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
-	ServerAdmin will@lolcat.ca
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
-	DocumentRoot /var/www/nyym
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
-	
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
-	SSLEngine On
+	
-	SSLOptions +StdEnvVars	
+	# redirect www.lolcat.ca to lolcat.ca
-	
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
+	
-	
+	# everything else redirects to 4get.ca
-	AddOutputFilterByType DEFLATE application/json
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
-	AddOutputFilterByType DEFLATE application/javascript
+	
-	AddOutputFilterByType DEFLATE application/x-javascript
+	# ... unless its an ip host, or the onion domain
-	AddOutputFilterByType DEFLATE text/html
+	use_backend apache if is_ip_host
-	AddOutputFilterByType DEFLATE text/plain
+
-	AddOutputFilterByType DEFLATE text/css
+frontend https_443_in
-
+	# bind tcp ip4/6
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
+	bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
+	bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
+	
-	
+	# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
-	<Directory /var/www/nyym>
+	# bind udp ip4/6
-		Options -MultiViews
+	#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
-		AllowOverride All
+	#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
-		Require all granted
+	
-		
+	# tell client we support http3 now
-		RewriteEngine On
+	#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
-		RewriteCond %{REQUEST_FILENAME} !-d
+	
-		RewriteCond %{REQUEST_FILENAME} !-f
+	mode http
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
+	
-	</Directory>
+	# redirect all unknown domains to 4get.ca
-</VirtualHost>
+	acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
-
+	
-<VirtualHost *:443>
+	# add onion domain to 4get.ca
-	ServerName git.lolcat.ca
+	acl is_4get hdr(host) -i 4get.ca
-
+	http-request set-var(txn.is_4get) bool(true) if is_4get
-	SSLEngine On
+	http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
-	SSLOptions +StdEnvVars	
+
-	
+	# git.lolcat.ca -> gitea service
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
+	acl acl_gitea hdr(host) -i git.lolcat.ca
-	
+	
-	AddOutputFilterByType DEFLATE application/json
+	# redirects
-	AddOutputFilterByType DEFLATE application/javascript
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
-	AddOutputFilterByType DEFLATE application/x-javascript
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
-	AddOutputFilterByType DEFLATE text/html
+	
-	AddOutputFilterByType DEFLATE text/plain
+	# add security headers
-	AddOutputFilterByType DEFLATE text/css
+	http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
-
+	http-response set-header X-Content-Type-Options "nosniff"
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
+	http-response set-header X-Frame-Options "SAMEORIGIN"
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
+	
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	# route
-
+	use_backend gitea if acl_gitea
-	ProxyPreserveHost On
+	default_backend apache
-	ProxyRequests off
+
-	AllowEncodedSlashes NoDecode
+backend apache
-	ProxyPass / http://localhost:3000/ nocanon
+	# pass ja4
-</VirtualHost>
+	http-request lua.fingerprint_ja4 if { ssl_fc }
-
+	http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
-<VirtualHost *:443>
+	
-	ServerName live.lolcat.ca
+	# delete server header
-
+	http-response del-header Server
-	ServerAdmin will@lolcat.ca
+	
-	DocumentRoot /var/www/live
+	# pass remote IP
-	
+	option forwardfor
-	SSLEngine On
+	server apache_server 127.0.0.1:8080 check
-	SSLOptions +StdEnvVars	
+
-	
+backend gitea
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
+	server gitea_server 127.0.0.1:3000 check
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-</VirtualHost>
 ```

lib/backend.php

@@ -12,9 +12,9 @@ class backend{
 	/*
 		Proxy stuff
 	*/
-	public function get_ip($proxy_index_raw = null){
+	public function get_ip($proxy_index_raw = null, $use_alt_list = false){
 		
-		$pool = constant("config::PROXY_" . strtoupper($this->scraper));
+		$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
 		if($pool === false){
 			
 			// we don't want a proxy, fuck off!

scraper/brave.php

@@ -460,7 +460,7 @@ class brave{
 				){
 					
 					$thumb = [
-						"ratio" => $result["thumbnail"]["logo"] == "false" ? "16:9" : "1:1",
+						"ratio" => (isset($result["thumbnail"]["logo"]) && $result["thumbnail"]["logo"] == "true") ? "1:1" : "16:9",
 						"url" => $result["thumbnail"]["original"]
 					];
 				}else{
@@ -994,6 +994,15 @@ class brave{
 				
 				foreach($data["videos"]["results"] as $video){
 					
+					if(
+						$video["title"] == "" &&
+						$video["description"] == ""
+					){
+						
+						// ignore result
+						continue;
+					}
+					
 					// get thumbnails
 					$thumbs = [];
 					
@@ -1111,141 +1120,6 @@ class brave{
 		return $out;
 	}
 	
-	public function news($get){
-		
-		if($get["npt"]){
-			
-			[$req, $proxy] = $this->backend->get($get["npt"], "news");
-			
-			$req = json_decode($req, true);
-			
-			$search = $req["q"];
-			$country = $req["country"];
-			$nsfw = $req["nsfw"];
-			$offset = $req["offset"];
-			$spellcheck = $req["spellcheck"];
-			
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"offset" => $offset,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-			
-		}else{
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			if(strlen($search) > 2048){
-				
-				throw new Exception("Search term is too long!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$country = $get["country"];
-			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
-			
-			/*
-			$handle = fopen("scraper/brave-news.html", "r");
-			$html = fread($handle, filesize("scraper/brave-news.html"));
-			fclose($handle);*/
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		// load html
-		$this->fuckhtml->load($html);
-		
-		$this->detect_captcha($html);
-		
-		// get npt
-		$out["npt"] =
-			$this->generatenextpagetoken(
-				$search,
-				$nsfw,
-				$country,
-				$spellcheck,
-				"news",
-				$proxy
-			);
-		
-		$this->fuckhtml->load($html);
-		$jsons = $this->get_js();
-		
-		foreach($jsons as $json){
-			foreach(
-				$json["data"]["response"]["news"]["results"]
-				as $news
-			){
-				
-				if(
-					!isset($news["thumbnail"]["src"]) ||
-					$news["thumbnail"]["src"] == "void 0"
-				){
-									
-					$thumb = [
-						"url" => null,
-						"ratio" => null
-					];
-				}else{
-									
-					$thumb = [
-						"url" => $this->unshiturl($news["thumbnail"]["src"]),
-						"ratio" => "16:9"
-					];
-				}
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"author" => null,
-					"description" => $news["description"],
-					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
-					"thumb" => $thumb,
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
 	public function image($get){
 		
 		$search = $get["s"];
@@ -1298,6 +1172,12 @@ class brave{
 		$jsons = $this->get_js();
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1325,7 +1205,7 @@ class brave{
 				];
 			}
 		}
-			
+		
 		return $out;
 	}
 	
@@ -1431,8 +1311,15 @@ class brave{
 		
 		$this->fuckhtml->load($html);
 		$jsons = $this->get_js();
+		$error = false;
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1488,6 +1375,147 @@ class brave{
 		return $out;
 	}
 	
+	public function news($get){
+		
+		if($get["npt"]){
+			
+			[$req, $proxy] = $this->backend->get($get["npt"], "news");
+			
+			$req = json_decode($req, true);
+			
+			$search = $req["q"];
+			$country = $req["country"];
+			$nsfw = $req["nsfw"];
+			$offset = $req["offset"];
+			$spellcheck = $req["spellcheck"];
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"offset" => $offset,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+			
+		}else{
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			if(strlen($search) > 2048){
+				
+				throw new Exception("Search term is too long!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			$nsfw = $get["nsfw"];
+			$country = $get["country"];
+			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
+			
+			/*
+			$handle = fopen("scraper/brave-news.html", "r");
+			$html = fread($handle, filesize("scraper/brave-news.html"));
+			fclose($handle);*/
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"news" => []
+		];
+		
+		// load html
+		$this->fuckhtml->load($html);
+		
+		$this->detect_captcha($html);
+		
+		// get npt
+		$out["npt"] =
+			$this->generatenextpagetoken(
+				$search,
+				$nsfw,
+				$country,
+				$spellcheck,
+				"news",
+				$proxy
+			);
+		
+		$this->fuckhtml->load($html);
+		$jsons = $this->get_js();
+		
+		foreach($jsons as $json){
+						
+			if(!isset($json["data"]["response"]["news"]["results"])){
+				
+				continue;
+			}
+			
+			foreach(
+				$json["data"]["response"]["news"]["results"]
+				as $news
+			){
+				
+				if(
+					!isset($news["thumbnail"]["src"]) ||
+					$news["thumbnail"]["src"] == "void 0"
+				){
+									
+					$thumb = [
+						"url" => null,
+						"ratio" => null
+					];
+				}else{
+									
+					$thumb = [
+						"url" => $this->unshiturl($news["thumbnail"]["src"]),
+						"ratio" => "16:9"
+					];
+				}
+				
+				$out["news"][] = [
+					"title" => $news["title"],
+					"author" => null,
+					"description" => $news["description"],
+					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
+					"thumb" => $thumb,
+					"url" => $news["url"]
+				];
+			}
+		}
+		
+		return $out;
+	}
+	
 	private function stackoverflow_parse($html){
 		
 		$i = 0;

scraper/google.php

@@ -658,7 +658,7 @@ class google{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		//$this->backend->assign_proxy($curlproc, $proxy);
+		$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
 		
 		curl_multi_add_handle($this->proc, $curlproc);
 		$this->handles[] = [
@@ -706,20 +706,29 @@ class google{
 			
 			if(count($a) === 0){
 				
-				throw new Exception("Failed to find link");
+				// failed to find link, will serve a resolver target as fallback
+				$fallback = true;
+				$slug = explode("url=", $this->handles[$i]["replacement"]);
+				$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
+			}else{
+				
+				$fallback = false;
+				$link =
+					$this->fuckhtml
+					->getTextContent(
+						$a[0]["attributes"]["href"]
+					);
 			}
 			
-			$link =
-				$this->fuckhtml
-				->getTextContent(
-					$a[0]["attributes"]["href"]
-				);
-			
 			if(
-				preg_match(
+				(
-					'/^(https?:|)\/\//',
+					$fallback === false &&
-					$link
+					preg_match(
-				)
+						'/^(https?:|)\/\//',
+						$link
+					)
+				) ||
+				$fallback === true
 			){
 				
 				$keys[] = $this->handles[$i]["replacement"];
@@ -817,7 +826,7 @@ class google{
 			// country
 			if($get["country"] != "any"){
 				
-				$params["gl"] = $country;
+				$params["gl"] = $get["country"];
 			}
 			
 			// nsfw
@@ -826,15 +835,15 @@ class google{
 			// language
 			if($get["lang"] != "any"){
 				
-				$params["lr"] = "lang_" . $lang;
+				$params["lr"] = "lang_" . $get["lang"];
 			}
 			
 			// generate tbs
 			$tbs = [];
 			
 			// get date
-			$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
+			$get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
-			$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
+			$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
 			
 			if(
 				$get["older"] !== null ||
@@ -1478,15 +1487,15 @@ class google{
 		// generate parameters
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$params_json, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"images"
 				);
 			
-			$params = json_decode($params, true);
+			$params_json = json_decode($params_json, true);
-			$container = $params["c"];
+			$container = &$params_json["c"];
-			unset($params["c"]);
+			$params = &$params_json["p"];
 		}else{
 			
 			$search = $get["s"];
@@ -1505,6 +1514,7 @@ class google{
 			$type = $get["type"];
 			$format = $get["format"];
 			$rights = $get["rights"];
+			$container = null;
 			
 			$params = [
 				"q" => $search,
@@ -1598,7 +1608,8 @@ class google{
 		$data = $this->get(
 			$proxy,
 			"https://www.google.com/search",
-			$params
+			$params,
+			$container
 		);
 		
 		$html = &$data["data"];
@@ -1701,7 +1712,10 @@ class google{
 			$out["npt"] =
 				$this->backend
 				->store(
-					json_encode($params),
+					json_encode([
+						"p" => $params,
+						"c" => $container
+					]),
 					"image",
 					$proxy
 				);

scraper/google_api.php

@@ -567,6 +567,8 @@ class google_api{
 		
 		foreach($json["items"] as $result){
 			
+			$date = null;
+			
 			//
 			// probe for thumbnail
 			//

scraper/google_cse.php

@@ -725,18 +725,27 @@ class google_cse{
 				// calculate rounded ratio
 				if($found_size){
 					
-					$aspect_ratio = $width / $height;
+					if(
-					
+						$width === 0 ||
-					if($aspect_ratio >= 1.5){
+						$height === 0
-						
+					){
-						$thumb["ratio"] = "16:9";
-					}
-					elseif($aspect_ratio >= 0.8){
 						
 						$thumb["ratio"] = "1:1";
 					}else{
 						
-						$thumb["ratio"] = "9:16";
+						$aspect_ratio = $width / $height;
+						
+						if($aspect_ratio >= 1.5){
+							
+							$thumb["ratio"] = "16:9";
+						}
+						elseif($aspect_ratio >= 0.8){
+							
+							$thumb["ratio"] = "1:1";
+						}else{
+							
+							$thumb["ratio"] = "9:16";
+						}
 					}
 				}
 			}

scraper/naver.php

@@ -261,9 +261,10 @@ class naver{
 			
 			$filters = [
 				"nso" => "",
+				"page" => 2, // page 2 is repeat of page 1
 				"query" => $search,
 				"sm" => "tab_pge",
-				"where" => "web",
+				"ssc" => "tab.ur.all",
 				"start" => 1 // increment by number of results each time (16??)
 			];
 			
@@ -304,13 +305,13 @@ class naver{
 		
 		$this->fuckhtml->load($html);
 		
-		$results =
+		$jsons =
 			preg_split(
 				'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
 				$html
 			);
 		
-		if(count($results) !== 2){
+		if(count($jsons) <= 1){
 			
 			// this is thrown when no results are found
 			
@@ -342,119 +343,169 @@ class naver{
 			//throw new Exception("Failed to grep results entrypoint");
 		}
 		
-		$json =
+		foreach($jsons as $json){
-			json_decode(
-				$this->fuckhtml
-				->extract_json(
-					$results[1]
-				),
-				true
-			);
-		
-		if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){
 			
-			throw new Exception("Failed to access nested children");
+			$json =
-		}
+				json_decode(
-		
+					$this->fuckhtml
-		foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){
+					->extract_json(
+						$json
+					),
+					true
+				);
 			
-			if(
+			if($json === null){
-				!isset($result["templateId"]) ||
-				$result["templateId"] != "webItem"
-			){
 				
-				// should not happen
 				continue;
 			}
 			
-			$result = $result["props"];
+			if(!isset($json["data"]["state"]["items"])){
-			
-			// get sublinks
-			$sublinks = [];
-			
-			if(isset($result["subLinks"])){
 				
-				foreach($result["subLinks"] as $s){
+				//throw new Exception("Failed to access nested children");
+				continue;
+			}
+			
+			foreach($json["data"]["state"]["items"] as $result){
+				
+				// get sublinks
+				$sublinks = [];
+				
+				if(isset($result["subLinks"])){
 					
-					$sublinks[] = [
+					foreach($result["subLinks"] as $s){
-						"title" => $s["text"],
+						
-						"description" => null,
+						$sublinks[] = [
-						"url" => $s["href"],
+							"title" => $s["text"],
-						"date" => null
+							"description" => null,
+							"url" => $s["href"],
+							"date" => null
+						];
+					}
+				}
+				
+				if(isset($result["linkBtns"])){
+					
+					foreach($result["linkBtns"] as $s){
+						
+						$sublinks[] = [
+							"title" => $s["text"],
+							"description" => null,
+							"url" => $s["href"],
+							"date" => null
+						];
+					}
+				}
+				
+				// get image (thumbnail, i guess)
+				if(isset($result["images"][0]["imageSrc"])){
+					
+					$thumb = [
+						"ratio" => "16:9",
+						"url" => $this->unshit_thumb($result["images"][0]["imageSrc"])
+					];
+				}else{
+					
+					$thumb = [
+						"ratio" => null,
+						"url" => null
 					];
 				}
-			}
-			
-			if(isset($result["linkBtns"])){
 				
-				foreach($result["linkBtns"] as $s){
+				// get table elements
+				$table = [];
+				
+				if(isset($result["keyValue"]["contents"])){
 					
-					$sublinks[] = [
+					foreach($result["keyValue"]["contents"] as $s){
-						"title" => $s["text"],
+						
-						"description" => null,
+						if(!isset($s["valueData"]["text"])){ continue; }
-						"url" => $s["href"],
+						
-						"date" => null
+						$table[$s["key"]] = $s["valueData"]["text"];
-					];
+					}
 				}
-			}
-			
-			// get image (thumbnail, i guess)
-			if(isset($result["images"][0]["imageSrc"])){
 				
-				$thumb = [
+				// get date
-					"ratio" => "16:9",
+				$time = null;
-					"url" => $this->unshit_thumb($result["images"][0]["imageSrc"])
-				];
-			}else{
 				
-				$thumb = [
+				if(isset($result["profile"]["createdDate"])){
-					"ratio" => null,
+					
-					"url" => null
+					$time = $result["profile"]["createdDate"];
+				}elseif(isset($result["sourceProfile"]["createdDate"])){
+					
+					$time = $result["sourceProfile"]["createdDate"];
+				}
+				
+				if($time !== null){
+					$date =
+						strtotime(
+							substr(
+								$time,
+								-1
+							)
+						);
+					
+					if($date !== false){
+						
+						$time = $date;
+					}else{
+						
+						$time = null;
+					}
+				}
+				
+				// get title
+				if(isset($result["title"])){
+					
+					$title = $result["title"];
+				}elseif(isset($result["questionTitle"])){
+					
+					$title = $result["questionTitle"];
+				}else{
+					
+					continue;
+				}
+				
+				// get url
+				if(isset($result["titleHref"])){
+					
+					$href = $result["titleHref"];
+				}elseif(isset($result["href"])){
+					
+					$href = $result["href"];
+				}elseif(isset($result["answerHref"])){
+					
+					$href = $result["answerHref"];
+				}else{
+					
+					continue;
+				}
+				
+				// get description
+				if(isset($result["bodyText"])){
+					
+					$description = $result["bodyText"];
+				}elseif(isset($result["content"])){
+					
+					$description = $result["content"];
+				}elseif(isset($result["questionContent"])){
+					
+					$description = $result["questionContent"];
+				}else{
+					
+					$description = null;
+				}
+				
+				$out["web"][] = [
+					"title" => $this->decode_html($title),
+					"description" => $this->decode_html($description),
+					"url" => $href,
+					"date" => $time,
+					"type" => "web",
+					"thumb" => $thumb,
+					"sublink" => $sublinks,
+					"table" => $table
 				];
 			}
-			
-			// get table elements
-			$table = [];
-			
-			if(isset($result["keyValue"]["contents"])){
-				
-				foreach($result["keyValue"]["contents"] as $s){
-					
-					if(!isset($s["valueData"]["text"])){ continue; }
-					
-					$table[$s["key"]] = $s["valueData"]["text"];
-				}
-			}
-			
-			// get date
-			$time = null;
-			
-			if(isset($result["bodyPrefixes"][0]["text"])){
-				
-				$date =
-					strtotime(
-						substr(
-							$result["bodyPrefixes"][0]["text"],
-							-1
-						)
-					);
-				
-				if($date !== false){
-					
-					$time = $date;
-				}
-			}
-			
-			$out["web"][] = [
-				"title" => $this->decode_html($result["title"]),
-				"description" => $this->decode_html($result["bodyText"]),
-				"url" => $result["href"],
-				"date" => $time,
-				"type" => "web",
-				"thumb" => $thumb,
-				"sublink" => $sublinks,
-				"table" => $table
-			];
 		}
 		
 		// get next page

scraper/yt.php

@@ -678,6 +678,15 @@ class yt{
 							$video["longBylineText"]
 						),
 					"url" =>
+						isset(
+							$video["ownerText"]
+							["runs"]
+							[0]
+							["navigationEndpoint"]
+							["commandMetadata"]
+							["webCommandMetadata"]
+							["url"]
+						) ?
 						"https://www.youtube.com" .
 						$video["ownerText"]
 						["runs"]
@@ -685,7 +694,7 @@ class yt{
 						["navigationEndpoint"]
 						["commandMetadata"]
 						["webCommandMetadata"]
-						["url"],
+						["url"] : null,
 					"avatar" =>
 						$video["channelThumbnailSupportedRenderers"]
 						["channelThumbnailWithLinkRenderer"]

web.php

@@ -349,8 +349,8 @@ if(count($results["answer"]) !== 0){
 				
 				$right["answer"] .=
 					'<tr>' . 
-						'<td>' . $info . '</td>' .
+						'<td>' . htmlspecialchars($info) . '</td>' .
-						'<td>' . $value . '</td>' .
+						'<td>' . htmlspecialchars($value) . '</td>' .
 					'</tr>';
 			}