fix google news

Reviewed-on: #98

api/v1/ac.php

@@ -163,6 +163,8 @@ class autocomplete{
 				// if it respects the openSearch protocol
 				$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
 				
+				if(!isset($json[1])){ $json[1] = []; }
+				
 				echo json_encode(
 					[
 						$_GET["s"],

data/config.php

@@ -168,6 +168,7 @@ class config{
 	const PROXY_YAHOO_JAPAN = false;
 	const PROXY_BRAVE = false;
 	const PROXY_GOOGLE = false;
+	const PROXY_GOOGLE_ALT = false; // alt proxies for url resolution
 	const PROXY_GOOGLE_API = false;
 	const PROXY_GOOGLE_CSE = false;
 	const PROXY_STARTPAGE = false;

docs/apache2-example.md

@@ -1,31 +1,16 @@
 # Sample Apache2 configuration
 This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
 
+Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
+
 Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
 
 ```xml
-<VirtualHost *:443>
-	ServerName www.4get.ca
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://4get.ca$1
-</VirtualHost>
-
-<VirtualHost *:443>
+<VirtualHost *:8080>
 	ServerName 4get.ca
+	ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
 
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/4get
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
+	# compress
 	AddOutputFilterByType DEFLATE application/json
 	AddOutputFilterByType DEFLATE application/javascript
 	AddOutputFilterByType DEFLATE application/x-javascript
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
 	AddOutputFilterByType DEFLATE text/plain
 	AddOutputFilterByType DEFLATE text/css
 	
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
+	DocumentRoot /var/www/4get
+	ErrorLog ${APACHE_LOG_DIR}/error.log
+	CustomLog /dev/null combined
+	#CustomLog /var/log/apache2/access.log combined
+	
+	<FilesMatch \.php$>
+		SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
+	</FilesMatch>
 
 	<Directory /var/www/4get>
 		Options -MultiViews
 		AllowOverride All
 		Require all granted
 		
+		# prevent 404s from hitting php-fpm
 		RewriteEngine On
+		RewriteCond %{REQUEST_URI} \.php$ [NC]
+		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteRule ^ - [R=404,L]
+		
+		# prevent /index hits
+		RewriteRule ^index$ - [R=404,L]
+		
+		# 404 direct .php file access
+		RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
+		RewriteRule ^ - [R=404,L]
+		
+		# internally redirect to non-php extension
 		RewriteCond %{REQUEST_FILENAME} !-d
 		RewriteCond %{REQUEST_FILENAME} !-f
+		RewriteCond %{REQUEST_FILENAME}.php -f
 		RewriteRule ^([^\.]+)$ $1.php [NC,L]
 	</Directory>
 
 	# deny access to private resources
-	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
+	<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
 		Require all denied
 	</DirectoryMatch>
 	
-	# remove directory listing from /icons
 	<Directory /var/www/4get/icons>
 		Options -Indexes
 	</Directory>
 </VirtualHost>
-
-<VirtualHost *:443>
-	ServerName www.lolcat.ca
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName lolcat.ca
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/lolcat
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-
-	<Directory /var/www/lolcat>
-		Options -MultiViews
-		AllowOverride All
-		Require all granted
-		
-		RewriteEngine On
-		RewriteCond %{REQUEST_FILENAME} !-d
-		RewriteCond %{REQUEST_FILENAME} !-f
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
-	</Directory>
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName www.nyym.co
-	
-	SSLEngine On
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
-	
-	RedirectMatch 301 ^(.*)$ https://nyym.co$1
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName nyym.co
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/nyym
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
-	
-	<Directory /var/www/nyym>
-		Options -MultiViews
-		AllowOverride All
-		Require all granted
-		
-		RewriteEngine On
-		RewriteCond %{REQUEST_FILENAME} !-d
-		RewriteCond %{REQUEST_FILENAME} !-f
-		RewriteRule ^([^\.]+)$ $1.php [NC,L]
-	</Directory>
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName git.lolcat.ca
-
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-
-	ProxyPreserveHost On
-	ProxyRequests off
-	AllowEncodedSlashes NoDecode
-	ProxyPass / http://localhost:3000/ nocanon
-</VirtualHost>
-
-<VirtualHost *:443>
-	ServerName live.lolcat.ca
-
-	ServerAdmin will@lolcat.ca
-	DocumentRoot /var/www/live
-	
-	SSLEngine On
-	SSLOptions +StdEnvVars	
-	
-	#ErrorLog ${APACHE_LOG_DIR}/error.log
-	
-	AddOutputFilterByType DEFLATE application/json
-	AddOutputFilterByType DEFLATE application/javascript
-	AddOutputFilterByType DEFLATE application/x-javascript
-	AddOutputFilterByType DEFLATE text/html
-	AddOutputFilterByType DEFLATE text/plain
-	AddOutputFilterByType DEFLATE text/css
-
-	SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
-	SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
-	SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
-</VirtualHost>
+```
+
+# Haproxy configuration
+
+```cfg
+global
+	#log /dev/log	local0
+	log /dev/log	local0 warning
+	log /dev/log	local1 notice
+	chroot /var/lib/haproxy
+	stats socket /run/haproxy/admin.sock mode 660 level admin
+	stats timeout 30s
+	user haproxy
+	group haproxy
+	daemon
+
+	# Default SSL material locations
+	ca-base /etc/ssl/certs
+	crt-base /etc/ssl/private
+
+	# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
+	#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
+	ssl-default-bind-ciphers HIGH:!aNULL
+	ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
+	ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
+	
+	# ja4 settings
+	tune.lua.bool-sample-conversion normal
+	tune.ssl.capture-buffer-size 190
+	lua-load /etc/haproxy/lua/ja4.lua
+defaults
+	log	global
+	mode	http
+	#option	httplog
+	option	dontlognull
+	timeout connect 5000
+	timeout client  50000
+	timeout server  50000
+	errorfile 400 /etc/haproxy/errors/400.http
+	errorfile 403 /etc/haproxy/errors/403.http
+	errorfile 408 /etc/haproxy/errors/408.http
+	errorfile 500 /etc/haproxy/errors/500.http
+	errorfile 502 /etc/haproxy/errors/502.http
+	errorfile 503 /etc/haproxy/errors/503.http
+	errorfile 504 /etc/haproxy/errors/504.http
+
+frontend http_80_in
+	# bind tcp ip4/6
+	bind *:80
+	bind [::]:80
+	
+	mode http
+	
+	# handle ip.lolcat.ca+onion addy on plain HTTP
+	acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
+		
+	# redirect to https
+	http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
+	http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
+	
+	# redirect www.lolcat.ca to lolcat.ca
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
+	
+	# everything else redirects to 4get.ca
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
+	
+	# ... unless its an ip host, or the onion domain
+	use_backend apache if is_ip_host
+
+frontend https_443_in
+	# bind tcp ip4/6
+	bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
+	bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
+	
+	# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
+	# bind udp ip4/6
+	#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
+	#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
+	
+	# tell client we support http3 now
+	#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
+	
+	mode http
+	
+	# redirect all unknown domains to 4get.ca
+	acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
+	
+	# add onion domain to 4get.ca
+	acl is_4get hdr(host) -i 4get.ca
+	http-request set-var(txn.is_4get) bool(true) if is_4get
+	http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
+
+	# git.lolcat.ca -> gitea service
+	acl acl_gitea hdr(host) -i git.lolcat.ca
+	
+	# redirects
+	http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
+	http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
+	
+	# add security headers
+	http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
+	http-response set-header X-Content-Type-Options "nosniff"
+	http-response set-header X-Frame-Options "SAMEORIGIN"
+	
+	# route
+	use_backend gitea if acl_gitea
+	default_backend apache
+
+backend apache
+	# pass ja4
+	http-request lua.fingerprint_ja4 if { ssl_fc }
+	http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
+	
+	# delete server header
+	http-response del-header Server
+	
+	# pass remote IP
+	option forwardfor
+	server apache_server 127.0.0.1:8080 check
+
+backend gitea
+	server gitea_server 127.0.0.1:3000 check
 ```

lib/backend.php

@@ -12,9 +12,9 @@ class backend{
 	/*
 		Proxy stuff
 	*/
-	public function get_ip($proxy_index_raw = null){
+	public function get_ip($proxy_index_raw = null, $use_alt_list = false){
 		
-		$pool = constant("config::PROXY_" . strtoupper($this->scraper));
+		$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
 		if($pool === false){
 			
 			// we don't want a proxy, fuck off!

scraper/brave.php

@@ -460,7 +460,7 @@ class brave{
 				){
 					
 					$thumb = [
-						"ratio" => $result["thumbnail"]["logo"] == "false" ? "16:9" : "1:1",
+						"ratio" => (isset($result["thumbnail"]["logo"]) && $result["thumbnail"]["logo"] == "true") ? "1:1" : "16:9",
 						"url" => $result["thumbnail"]["original"]
 					];
 				}else{
@@ -994,6 +994,15 @@ class brave{
 				
 				foreach($data["videos"]["results"] as $video){
 					
+					if(
+						$video["title"] == "" &&
+						$video["description"] == ""
+					){
+						
+						// ignore result
+						continue;
+					}
+					
 					// get thumbnails
 					$thumbs = [];
 					
@@ -1111,141 +1120,6 @@ class brave{
 		return $out;
 	}
 	
-	public function news($get){
-		
-		if($get["npt"]){
-			
-			[$req, $proxy] = $this->backend->get($get["npt"], "news");
-			
-			$req = json_decode($req, true);
-			
-			$search = $req["q"];
-			$country = $req["country"];
-			$nsfw = $req["nsfw"];
-			$offset = $req["offset"];
-			$spellcheck = $req["spellcheck"];
-			
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"offset" => $offset,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-			
-		}else{
-			$search = $get["s"];
-			if(strlen($search) === 0){
-				
-				throw new Exception("Search term is empty!");
-			}
-			
-			if(strlen($search) > 2048){
-				
-				throw new Exception("Search term is too long!");
-			}
-			
-			$proxy = $this->backend->get_ip();
-			$nsfw = $get["nsfw"];
-			$country = $get["country"];
-			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
-			
-			/*
-			$handle = fopen("scraper/brave-news.html", "r");
-			$html = fread($handle, filesize("scraper/brave-news.html"));
-			fclose($handle);*/
-			try{
-				$html =
-					$this->get(
-						$proxy,
-						"https://search.brave.com/news",
-						[
-							"q" => $search,
-							"spellcheck" => $spellcheck
-						],
-						$nsfw,
-						$country
-					);
-				
-			}catch(Exception $error){
-				
-				throw new Exception("Could not fetch search page");
-			}
-		}
-		
-		$out = [
-			"status" => "ok",
-			"npt" => null,
-			"news" => []
-		];
-		
-		// load html
-		$this->fuckhtml->load($html);
-		
-		$this->detect_captcha($html);
-		
-		// get npt
-		$out["npt"] =
-			$this->generatenextpagetoken(
-				$search,
-				$nsfw,
-				$country,
-				$spellcheck,
-				"news",
-				$proxy
-			);
-		
-		$this->fuckhtml->load($html);
-		$jsons = $this->get_js();
-		
-		foreach($jsons as $json){
-			foreach(
-				$json["data"]["response"]["news"]["results"]
-				as $news
-			){
-				
-				if(
-					!isset($news["thumbnail"]["src"]) ||
-					$news["thumbnail"]["src"] == "void 0"
-				){
-									
-					$thumb = [
-						"url" => null,
-						"ratio" => null
-					];
-				}else{
-									
-					$thumb = [
-						"url" => $this->unshiturl($news["thumbnail"]["src"]),
-						"ratio" => "16:9"
-					];
-				}
-				
-				$out["news"][] = [
-					"title" => $news["title"],
-					"author" => null,
-					"description" => $news["description"],
-					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
-					"thumb" => $thumb,
-					"url" => $news["url"]
-				];
-			}
-		}
-		
-		return $out;
-	}
-	
 	public function image($get){
 		
 		$search = $get["s"];
@@ -1298,6 +1172,12 @@ class brave{
 		$jsons = $this->get_js();
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1431,8 +1311,15 @@ class brave{
 		
 		$this->fuckhtml->load($html);
 		$jsons = $this->get_js();
+		$error = false;
 		
 		foreach($jsons as $json){
+			
+			if(!isset($json["data"]["body"]["response"]["results"])){
+				
+				continue;
+			}
+			
 			foreach(
 				$json
 				["data"]
@@ -1488,6 +1375,147 @@ class brave{
 		return $out;
 	}
 	
+	public function news($get){
+		
+		if($get["npt"]){
+			
+			[$req, $proxy] = $this->backend->get($get["npt"], "news");
+			
+			$req = json_decode($req, true);
+			
+			$search = $req["q"];
+			$country = $req["country"];
+			$nsfw = $req["nsfw"];
+			$offset = $req["offset"];
+			$spellcheck = $req["spellcheck"];
+			
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"offset" => $offset,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+			
+		}else{
+			$search = $get["s"];
+			if(strlen($search) === 0){
+				
+				throw new Exception("Search term is empty!");
+			}
+			
+			if(strlen($search) > 2048){
+				
+				throw new Exception("Search term is too long!");
+			}
+			
+			$proxy = $this->backend->get_ip();
+			$nsfw = $get["nsfw"];
+			$country = $get["country"];
+			$spellcheck = $get["spellcheck"] == "yes" ? "1" : "0";
+			
+			/*
+			$handle = fopen("scraper/brave-news.html", "r");
+			$html = fread($handle, filesize("scraper/brave-news.html"));
+			fclose($handle);*/
+			try{
+				$html =
+					$this->get(
+						$proxy,
+						"https://search.brave.com/news",
+						[
+							"q" => $search,
+							"spellcheck" => $spellcheck
+						],
+						$nsfw,
+						$country
+					);
+				
+			}catch(Exception $error){
+				
+				throw new Exception("Could not fetch search page");
+			}
+		}
+		
+		$out = [
+			"status" => "ok",
+			"npt" => null,
+			"news" => []
+		];
+		
+		// load html
+		$this->fuckhtml->load($html);
+		
+		$this->detect_captcha($html);
+		
+		// get npt
+		$out["npt"] =
+			$this->generatenextpagetoken(
+				$search,
+				$nsfw,
+				$country,
+				$spellcheck,
+				"news",
+				$proxy
+			);
+		
+		$this->fuckhtml->load($html);
+		$jsons = $this->get_js();
+		
+		foreach($jsons as $json){
+						
+			if(!isset($json["data"]["response"]["news"]["results"])){
+				
+				continue;
+			}
+			
+			foreach(
+				$json["data"]["response"]["news"]["results"]
+				as $news
+			){
+				
+				if(
+					!isset($news["thumbnail"]["src"]) ||
+					$news["thumbnail"]["src"] == "void 0"
+				){
+									
+					$thumb = [
+						"url" => null,
+						"ratio" => null
+					];
+				}else{
+									
+					$thumb = [
+						"url" => $this->unshiturl($news["thumbnail"]["src"]),
+						"ratio" => "16:9"
+					];
+				}
+				
+				$out["news"][] = [
+					"title" => $news["title"],
+					"author" => null,
+					"description" => $news["description"],
+					"date" => !isset($news["age"]) || $news["age"] == "void 0" || $news["age"] == "null" ? null : strtotime($news["age"]),
+					"thumb" => $thumb,
+					"url" => $news["url"]
+				];
+			}
+		}
+		
+		return $out;
+	}
+	
 	private function stackoverflow_parse($html){
 		
 		$i = 0;

scraper/google.php

@@ -10,8 +10,9 @@ class google{
 		include "lib/backend.php";
 		$this->backend = new backend("google");
 		
-		$this->proc = curl_multi_init();
-		$this->handles = [];
+		$this->urls_to_resolve = [];
+		
+		$this->init_multi_curl();
 	}
 	
 	public function getfilters($page){
@@ -617,23 +618,57 @@ class google{
 		];
 	}
 	
-	private function redirect_add_url($url){
+	private function init_multi_curl(){
+		
+		$this->proc = curl_multi_init();
+		$this->handles = [];
+	}
+	
+	private function redirect_add_url($url, $replace = null){
 		
 		if(
 			preg_match(
 				'/^\/goto\?url=([^&]+)/',
 				$url,
 				$slug
-			) === 0
+			) === 1
 		){
 			
-			// not a google redirect
+			// normal goto
+			$this->urls_to_resolve[] = [
+				"replace" => $url,
+				"slug" => $slug[1],
+				"type" => "goto"
+			];
+		}
+		
+		elseif(
+			preg_match(
+				'/https?:\/\/share\.google\/([^"]+)+/',
+				$url,
+				$link_regex
+			) === 1
+		){
+			
+			// share.google
+			$this->urls_to_resolve[] = [
+				"replace" => $replace,
+				"slug" => $link_regex[1],
+				"type" => "share"
+			];
+		}
+		
+		// rest is ignored, since its not a redirect
+		
+		// always return url
 		return $url;
 	}
 	
+	private function urlfuck_goto($url, $slug){
+		
 		$curlproc = curl_init();
 		
-		curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
+		curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
 		
 		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
 		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -658,7 +693,7 @@ class google{
 		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
 		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
 		
-		//$this->backend->assign_proxy($curlproc, $proxy);
+		//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
 		
 		curl_multi_add_handle($this->proc, $curlproc);
 		$this->handles[] = [
@@ -669,7 +704,92 @@ class google{
 		return $url;
 	}
 	
-	private function resolve_urls(&$scraped_data){
+	private function urlfuck_shareservice($url, $slug){
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
+		
+		curl_multi_add_handle($this->proc, $curlproc);
+		$this->handles[] = [
+			"replacement" => $url,
+			"handle" => $curlproc
+		];
+		
+		return $url;
+	}
+	
+	private function urlfuck_share($url, $slug){
+		
+		$curlproc = curl_init();
+		
+		curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
+		
+		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
+		curl_setopt($curlproc, CURLOPT_HTTPHEADER,
+			["User-Agent: " . config::USER_AGENT,
+			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+			"Accept-Language: en-US,en;q=0.5",
+			"Accept-Encoding: gzip, deflate, br, zstd",
+			"DNT: 1",
+			"Sec-GPC: 1",
+			"Connection: keep-alive",
+			"Upgrade-Insecure-Requests: 1",
+			"Sec-Fetch-Dest: document",
+			"Sec-Fetch-Mode: navigate",
+			"Sec-Fetch-Site: none",
+			"Sec-Fetch-User: ?1",
+			"Priority: u=0, i"]
+		);
+		
+		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
+		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
+		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
+		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
+		
+		//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
+		
+		curl_multi_add_handle($this->proc, $curlproc);
+		$this->handles[] = [
+			"replacement" => $url,
+			"handle" => $curlproc
+		];
+		
+		return $url;
+	}
+	
+	private function resolve_gotos(&$scraped_data){
+		
+		foreach($this->urls_to_resolve as $item){
+			
+			$this->urlfuck_goto($item["replace"], $item["slug"]);
+		}
 		
 		do{
 			do{
@@ -706,20 +826,163 @@ class google{
 			
 			if(count($a) === 0){
 				
-				throw new Exception("Failed to find link");
-			}
+				// failed to find link, will serve a resolver target as fallback
+				$fallback = true;
+				$slug = explode("url=", $this->handles[$i]["replacement"]);
+				$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
+			}else{
 				
+				$fallback = false;
 				$link =
 					$this->fuckhtml
 					->getTextContent(
 						$a[0]["attributes"]["href"]
 					);
+			}
 			
 			if(
+				(
+					$fallback === false &&
 					preg_match(
 						'/^(https?:|)\/\//',
 						$link
 					)
+				) ||
+				$fallback === true
+			){
+				
+				$keys[] = $this->handles[$i]["replacement"];
+				$replacements[] = $link;
+			}
+		}
+		
+		$this->recursive_replace($keys, $replacements, $scraped_data);
+		
+		curl_multi_close($this->proc);
+	}
+	
+	private function resolve_share(&$scraped_data){
+		
+		foreach($this->urls_to_resolve as $item){
+			
+			$this->urlfuck_shareservice($item["replace"], $item["slug"]);
+		}
+		
+		do{
+			do{
+				$status = curl_multi_exec($this->proc, $active);
+			}while($status === CURLM_CALL_MULTI_PERFORM);
+
+			if($active){
+				curl_multi_select($this->proc);
+			}
+
+		}while($active && $status === CURLM_OK);
+		
+		//
+		// if we reach this, we're done downloading garbage
+		//
+		
+		// get the share links
+		$this->urls_to_resolve = [];
+		
+		for($i=0; $i<count($this->handles); $i++){
+			
+			$html = curl_multi_getcontent($this->handles[$i]["handle"]);
+			
+			if(
+				preg_match(
+					'/https?:\/\/share\.google\/(?:[^"]+)/',
+					$html,
+					$link_regex
+				) !== 1
+			){
+				
+				// failed to find link, will serve a resolver target as fallback
+				$fallback = true;
+				$slug = explode("url=", $this->handles[$i]["replacement"]);
+				$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
+			}else{
+				
+				$fallback = false;
+				$link = $link_regex[0];
+			}
+			
+			if(
+				(
+					$fallback === false &&
+					preg_match(
+						'/^(https?:|)\/\//',
+						$link
+					)
+				) ||
+				$fallback === true
+			){
+				
+				$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
+			}
+		}
+		
+		// resolve shareable urls
+		$this->init_multi_curl();
+		
+		foreach($this->urls_to_resolve as $item){
+			
+			$this->urlfuck_share($item["replace"], $item["slug"]);
+		}
+		
+		do{
+			do{
+				$status = curl_multi_exec($this->proc, $active);
+			}while($status === CURLM_CALL_MULTI_PERFORM);
+
+			if($active){
+				curl_multi_select($this->proc);
+			}
+
+		}while($active && $status === CURLM_OK);
+		
+		// parse downloaded garbage
+		$keys = [];
+		$replacements = [];
+		
+		for($i=0; $i<count($this->handles); $i++){
+			
+			$html = curl_multi_getcontent($this->handles[$i]["handle"]);
+			
+			$this->fuckhtml->load($html);
+			
+			$a =
+				$this->fuckhtml
+				->getElementsByTagName(
+					"a"
+				);
+			
+			if(count($a) === 0){
+				
+				// failed to find link, will serve a resolver target as fallback
+				$fallback = true;
+				$slug = explode("url=", $this->handles[$i]["replacement"]);
+				$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
+			}else{
+				
+				$fallback = false;
+				$link =
+					$this->fuckhtml
+					->getTextContent(
+						$a[0]["attributes"]["href"]
+					);
+			}
+			
+			if(
+				(
+					$fallback === false &&
+					preg_match(
+						'/^(https?:|)\/\//',
+						$link
+					)
+				) ||
+				$fallback === true
 			){
 				
 				$keys[] = $this->handles[$i]["replacement"];
@@ -785,22 +1048,6 @@ class google{
 				
 				throw new Exception("Search term is empty!");
 			}
-			/*
-			$session = $this->backend->session_get();
-			
-			if(config::FPLAY_ENABLE_API === false){
-				
-				throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
-			}
-			
-			if($session === false){
-				
-				throw new Exception("Session database is empty");
-			}
-			
-			$proxy = &$session["proxy"];
-			$headers = &$session["headers"];
-			*/
 			
 			$proxy = $this->backend->get_ip();
 			
@@ -817,7 +1064,7 @@ class google{
 			// country
 			if($get["country"] != "any"){
 				
-				$params["gl"] = $country;
+				$params["gl"] = $get["country"];
 			}
 			
 			// nsfw
@@ -826,15 +1073,15 @@ class google{
 			// language
 			if($get["lang"] != "any"){
 				
-				$params["lr"] = "lang_" . $lang;
+				$params["lr"] = "lang_" . $get["lang"];
 			}
 			
 			// generate tbs
 			$tbs = [];
 			
 			// get date
-			$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
-			$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
+			$get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
+			$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
 			
 			if(
 				$get["older"] !== null ||
@@ -1466,7 +1713,7 @@ class google{
 			];
 		}
 		
-		$this->resolve_urls($out);
+		$this->resolve_share($out);
 		
 		return $out;
 	}
@@ -1478,15 +1725,15 @@ class google{
 		// generate parameters
 		if($get["npt"]){
 			
-			[$params, $proxy] =
+			[$params_json, $proxy] =
 				$this->backend->get(
 					$get["npt"],
 					"images"
 				);
 			
-			$params = json_decode($params, true);
-			$container = $params["c"];
-			unset($params["c"]);
+			$params_json = json_decode($params_json, true);
+			$container = &$params_json["c"];
+			$params = &$params_json["p"];
 		}else{
 			
 			$search = $get["s"];
@@ -1505,6 +1752,7 @@ class google{
 			$type = $get["type"];
 			$format = $get["format"];
 			$rights = $get["rights"];
+			$container = null;
 			
 			$params = [
 				"q" => $search,
@@ -1593,17 +1841,18 @@ class google{
 			}
 		}
 		
-		//$html = file_get_contents("scraper/google.html");
-				
 		$data = $this->get(
 			$proxy,
 			"https://www.google.com/search",
-			$params
+			$params,
+			$container
 		);
 		
 		$html = &$data["data"];
 		$container = &$data["container"];
 		
+		//$html = file_get_contents("scraper/google.html");
+		
 		$this->fuckhtml->load($html);
 		
 		// parse all <style> tags
@@ -1701,13 +1950,16 @@ class google{
 			$out["npt"] =
 				$this->backend
 				->store(
-					json_encode($params),
+					json_encode([
+						"p" => $params,
+						"c" => $container
+					]),
 					"image",
 					$proxy
 				);
 		}
 		
-		$this->resolve_urls($out);
+		$this->resolve_gotos($out);
 		
 		return $out;
 	}
@@ -2115,7 +2367,7 @@ class google{
 			];
 		}
 		
-		$this->resolve_urls($out);
+		$this->resolve_share($out);
 		
 		return $out;
 	}
@@ -2474,14 +2726,14 @@ class google{
 				"date" => $date,
 				"thumb" => $thumb,
 				"url" =>
-					$this->unshiturl(
+					$this->redirect_add_url(
 						$a["attributes"]
 						["href"]
 					)
 			];
 		}
 		
-		$this->resolve_urls($out);
+		$this->resolve_share($out);
 		
 		return $out;
 	}

scraper/google_api.php

@@ -567,6 +567,8 @@ class google_api{
 		
 		foreach($json["items"] as $result){
 			
+			$date = null;
+			
 			//
 			// probe for thumbnail
 			//

scraper/google_cse.php

@@ -725,6 +725,14 @@ class google_cse{
 				// calculate rounded ratio
 				if($found_size){
 					
+					if(
+						$width === 0 ||
+						$height === 0
+					){
+						
+						$thumb["ratio"] = "1:1";
+					}else{
+						
 						$aspect_ratio = $width / $height;
 						
 						if($aspect_ratio >= 1.5){
@@ -740,6 +748,7 @@ class google_cse{
 						}
 					}
 				}
+			}
 			
 			$out["web"][] = [
 				"title" => rtrim($result["titleNoFormatting"], " ."),

scraper/naver.php

@@ -261,9 +261,10 @@ class naver{
 			
 			$filters = [
 				"nso" => "",
+				"page" => 2, // page 2 is repeat of page 1
 				"query" => $search,
 				"sm" => "tab_pge",
-				"where" => "web",
+				"ssc" => "tab.ur.all",
 				"start" => 1 // increment by number of results each time (16??)
 			];
 			
@@ -304,13 +305,13 @@ class naver{
 		
 		$this->fuckhtml->load($html);
 		
-		$results =
+		$jsons =
 			preg_split(
 				'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
 				$html
 			);
 		
-		if(count($results) !== 2){
+		if(count($jsons) <= 1){
 			
 			// this is thrown when no results are found
 			
@@ -342,32 +343,29 @@ class naver{
 			//throw new Exception("Failed to grep results entrypoint");
 		}
 		
+		foreach($jsons as $json){
+			
 			$json =
 				json_decode(
 					$this->fuckhtml
 					->extract_json(
-					$results[1]
+						$json
 					),
 					true
 				);
 			
-		if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){
+			if($json === null){
 				
-			throw new Exception("Failed to access nested children");
-		}
-		
-		foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){
-			
-			if(
-				!isset($result["templateId"]) ||
-				$result["templateId"] != "webItem"
-			){
-				
-				// should not happen
 				continue;
 			}
 			
-			$result = $result["props"];
+			if(!isset($json["data"]["state"]["items"])){
+				
+				//throw new Exception("Failed to access nested children");
+				continue;
+			}
+			
+			foreach($json["data"]["state"]["items"] as $result){
 				
 				// get sublinks
 				$sublinks = [];
@@ -429,12 +427,19 @@ class naver{
 				// get date
 				$time = null;
 				
-			if(isset($result["bodyPrefixes"][0]["text"])){
+				if(isset($result["profile"]["createdDate"])){
 					
+					$time = $result["profile"]["createdDate"];
+				}elseif(isset($result["sourceProfile"]["createdDate"])){
+					
+					$time = $result["sourceProfile"]["createdDate"];
+				}
+				
+				if($time !== null){
 					$date =
 						strtotime(
 							substr(
-							$result["bodyPrefixes"][0]["text"],
+								$time,
 								-1
 							)
 						);
@@ -442,13 +447,58 @@ class naver{
 					if($date !== false){
 						
 						$time = $date;
+					}else{
+						
+						$time = null;
 					}
 				}
 				
+				// get title
+				if(isset($result["title"])){
+					
+					$title = $result["title"];
+				}elseif(isset($result["questionTitle"])){
+					
+					$title = $result["questionTitle"];
+				}else{
+					
+					continue;
+				}
+				
+				// get url
+				if(isset($result["titleHref"])){
+					
+					$href = $result["titleHref"];
+				}elseif(isset($result["href"])){
+					
+					$href = $result["href"];
+				}elseif(isset($result["answerHref"])){
+					
+					$href = $result["answerHref"];
+				}else{
+					
+					continue;
+				}
+				
+				// get description
+				if(isset($result["bodyText"])){
+					
+					$description = $result["bodyText"];
+				}elseif(isset($result["content"])){
+					
+					$description = $result["content"];
+				}elseif(isset($result["questionContent"])){
+					
+					$description = $result["questionContent"];
+				}else{
+					
+					$description = null;
+				}
+				
 				$out["web"][] = [
-				"title" => $this->decode_html($result["title"]),
-				"description" => $this->decode_html($result["bodyText"]),
-				"url" => $result["href"],
+					"title" => $this->decode_html($title),
+					"description" => $this->decode_html($description),
+					"url" => $href,
 					"date" => $time,
 					"type" => "web",
 					"thumb" => $thumb,
@@ -456,6 +506,7 @@ class naver{
 					"table" => $table
 				];
 			}
+		}
 		
 		// get next page
 		$npt =

scraper/yt.php

@@ -678,6 +678,15 @@ class yt{
 							$video["longBylineText"]
 						),
 					"url" =>
+						isset(
+							$video["ownerText"]
+							["runs"]
+							[0]
+							["navigationEndpoint"]
+							["commandMetadata"]
+							["webCommandMetadata"]
+							["url"]
+						) ?
 						"https://www.youtube.com" .
 						$video["ownerText"]
 						["runs"]
@@ -685,7 +694,7 @@ class yt{
 						["navigationEndpoint"]
 						["commandMetadata"]
 						["webCommandMetadata"]
-						["url"],
+						["url"] : null,
 					"avatar" =>
 						$video["channelThumbnailSupportedRenderers"]
 						["channelThumbnailWithLinkRenderer"]

web.php

@@ -349,8 +349,8 @@ if(count($results["answer"]) !== 0){
 				
 				$right["answer"] .=
 					'<tr>' . 
-						'<td>' . $info . '</td>' .
-						'<td>' . $value . '</td>' .
+						'<td>' . htmlspecialchars($info) . '</td>' .
+						'<td>' . htmlspecialchars($value) . '</td>' .
 					'</tr>';
 			}