Compare commits

...
25 Commits
Author SHA1 Message Date
lolcat 4c7c4ea072 forgot settings woopsies tehehe 2026-11-15 19:41:29 -05:00
lolcat 761a1e91cd added coccoc and naver autocomplete, added google aliases 2026-11-15 19:14:37 -05:00
lolcat c5311bc399 autocomplete fixes 2026-11-15 18:44:14 -05:00
lolcat 4536cef88f how is it fucking 4am again 2026-11-13 03:57:07 -05:00
lolcat 0c3844d578 added brave api support since theyre bragging about adding goto tracking links soon wtf are they even thinking 2026-11-13 03:56:13 -05:00
lolcat 30cd3c8492 brave fix fuck you brave 2026-11-12 11:06:28 -05:00
lolcat aa818ac925 im starting to understand chinese 2026-11-11 00:08:00 -05:00
lolcat 5967462b6e baidu captchawalls all requests, removing 2026-11-10 20:26:00 -05:00
lolcat b014a3199d ok this should work now, make sure to enable browser.search.suggest.enabled in ff aaaaaaaaaa 2026-11-08 16:50:37 -05:00
lolcat 24130effe6 autocomplete fixes 2026-11-08 16:33:02 -05:00
lolcat 5d8c9a5798 startpage fix 2026-11-08 15:55:20 -05:00
lolcat 6bbb37d86e yep fic 2026-11-07 02:09:26 -05:00
lolcat c0324d7d61 Merge pull request 'fix ami4get 500 error' (#101) from docker_fix_ami4get into master
Reviewed-on: #101
2026-11-01 13:03:03 -05:00
throwaway 2218fd6f04 change variable 2026-11-01 00:27:32 -07:00
lolcat be91ec1806 fix google news 2026-10-31 14:11:20 -04:00
lolcat 57a72e5b5e url redirect bypass 2026-10-31 14:01:52 -04:00
lolcat 9f0437d949 rotate ips on url resolution 2026-10-30 00:18:06 -04:00
lolcat d91834f86f fix filter bugs on google scraper 2026-10-29 23:56:12 -04:00
lolcat 42d0252e2f handle failed redirect fetches 2026-10-28 22:59:45 -04:00
lolcat a27a8ed01a fix potential xss with scraper supplied data in table, ty cynic 2026-10-27 01:12:32 -04:00
lolcat 9db95c79d2 update example apache2 config 2026-10-24 05:24:27 -04:00
lolcat 25d4f991c2 fix some php warnings 2026-10-24 04:03:00 -04:00
lolcat e1d0ab121f fix google cse crash 2026-10-24 03:24:52 -04:00
lolcat 33ce0a062b Merge branch 'master' of https://git.lolcat.ca/lolcat/4get 2026-10-24 03:19:13 -04:00
lolcat 3b3c759ce5 hyung..sunbae.. i fixed naver 2026-10-24 03:18:07 -04:00
19 changed files with 2320 additions and 1656 deletions

No files matched your search

+1 -1
View File
@@ -31,7 +31,7 @@ echo json_encode(
"real_requests" => $real_requests, "real_requests" => $real_requests,
"bot_requests" => $bot_requests, "bot_requests" => $bot_requests,
"api_enabled" => config::API_ENABLED, "api_enabled" => config::API_ENABLED,
"alt_addresses" => config::ALT_ADDRESSES, "alt_addresses" => config::ADDRESSES,
"version" => config::VERSION "version" => config::VERSION
], ],
"instances" => config::INSTANCES "instances" => config::INSTANCES
+116 -24
View File
@@ -7,37 +7,51 @@ class autocomplete{
public function __construct(){ public function __construct(){
header("Content-Type: application/json"); header("Content-Type: application/x-suggestions+json");
$this->scrapers = [ $this->scrapers = [
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}", "brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
"bing" => "https://www.bing.com/osjson.aspx?query={searchTerms}",
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list", "ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015", "ecosia" => "https://ac.ecosia.org/autocomplete?q={searchTerms}&type=list",
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}", "yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3",
"google" => "https://www.google.com/complete/search?client=chrome&q={searchTerms}",
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch", "qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}", "marginalia" => "https://marginalia-search.com/suggest/?partial={searchTerms}",
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}", "yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
"sc" => "", "sc" => "",
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english", "kagi" => "https://kagisuggest.com/api/autosuggest?q={searchTerms}",
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}" "startpage" => "https://www.startpage.com/osuggestions?q={searchTerms}",
"yahoo" => "https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=sd1&command={searchTerms}",
"purili" => "https://puri.li/api/suggest?q={searchTerms}",
"coccoc" => "https://coccoc.com/composer/autocomplete?q={searchTerms}",
"naver" => "https://ac.search.naver.com/nx/ac?q={searchTerms}&st=100"
]; ];
// add duplicates
$this->scrapers["google_api"] = &$this->scrapers["google"];
$this->scrapers["google_cse"] = &$this->scrapers["google"];
$this->scrapers["yahoo_japan"] = &$this->scrapers["yahoo"];
/* /*
Sanitize input Sanitize input
*/ */
if(!isset($_GET["s"])){ if(!isset($_GET["s"])){
$this->do404("Missing search(s) parameter"); // Missing search(s) parameter
$this->doempty();
} }
if(is_string($_GET["s"]) === false){ if(is_string($_GET["s"]) === false){
$this->do404("Invalid search(s) parameter"); // Invalid search(s) parameter
$this->doempty();
} }
if(strlen($_GET["s"]) > 500){ if(strlen($_GET["s"]) > 500){
$this->do404("Search(s) exceeds the 500 char length"); // Search(s) exceeds the 500 char length
$this->doempty();
} }
/* /*
@@ -72,9 +86,8 @@ class autocomplete{
// return results // return results
switch($scraper){ switch($scraper){
case "google":
case "yt": case "yt":
// handle google cause they want to be a special snowflake :( // handle youtube cause it wants to be a special snowflake :(
$js = $this->get($this->scrapers[$scraper], $_GET["s"]); $js = $this->get($this->scrapers[$scraper], $_GET["s"]);
preg_match( preg_match(
@@ -141,12 +154,12 @@ class autocomplete{
break; break;
case "marginalia": case "marginalia":
case "purili":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]); $json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true); $json = json_decode($json, true);
if($json === null){ if($json === null){
$this->doempty(); $this->doempty();
} }
@@ -159,10 +172,98 @@ class autocomplete{
); );
break; break;
case "yahoo":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["r"][0]["k"])
){
$this->doempty();
}
$payload = [];
foreach($json["r"] as $term){
if(isset($term["k"])){
$payload[] = $term["k"];
}
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "naver":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["items"][0][0])
){
$this->doempty();
}
$payload = [];
foreach($json["items"][0] as $term){
$payload[] = $term[0];
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "coccoc":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["suggestions"])
){
$this->doempty();
}
echo json_encode(
[
$_GET["s"],
$json["suggestions"]
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
default: default:
// if it respects the openSearch protocol // if it respects the openSearch protocol
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true); $json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
if(
!isset($json[1]) ||
!is_array($json[1])
){
$json[1] = [];
}
echo json_encode( echo json_encode(
[ [
$_GET["s"], $_GET["s"],
@@ -185,10 +286,10 @@ class autocomplete{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER, curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0", ["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/javascript, */*; q=0.01", "Accept: application/json, text/javascript, */*; q=0.01",
"Accept-Language: en-US,en;q=0.5", "Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip", "Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1", "DNT: 1",
"Connection: keep-alive", "Connection: keep-alive",
"Sec-Fetch-Dest: empty", "Sec-Fetch-Dest: empty",
@@ -214,19 +315,10 @@ class autocomplete{
}catch(Exception $error){ }catch(Exception $error){
do404("Curl error: " . $error->getMessage()); doempty();
} }
} }
private function do404($error){
echo json_encode(
["error" => $error],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
die();
}
private function doempty(){ private function doempty(){
echo json_encode( echo json_encode(
+1
View File
@@ -0,0 +1 @@
+3 -2
View File
@@ -154,7 +154,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages // Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things. // Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0"; const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:160.0) Gecko/20100101 Firefox/160.0";
// User agent to use with 4get-friendly APIs // User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)"; const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
@@ -225,6 +225,7 @@ class config{
// use the API (Eg: "public"). API has less filters. // use the API (Eg: "public"). API has less filters.
const MARGINALIA_API_KEY = null; const MARGINALIA_API_KEY = null;
// Yep // Scrapers to use the API on
const YEP_USE_API = false; const YEP_USE_API = false;
const BRAVE_USE_API = false;
} }
+154 -163
View File
@@ -1,31 +1,16 @@
# Sample Apache2 configuration # Sample Apache2 configuration
This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you! This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>. Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
```xml ```xml
<VirtualHost *:443> <VirtualHost *:8080>
ServerName www.4get.ca
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
RedirectMatch 301 ^(.*)$ https://4get.ca$1
</VirtualHost>
<VirtualHost *:443>
ServerName 4get.ca ServerName 4get.ca
ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
ServerAdmin will@lolcat.ca # compress
DocumentRoot /var/www/4get
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
AddOutputFilterByType DEFLATE application/json AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript AddOutputFilterByType DEFLATE application/x-javascript
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
AddOutputFilterByType DEFLATE text/plain AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem DocumentRoot /var/www/4get
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem ErrorLog ${APACHE_LOG_DIR}/error.log
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem CustomLog /dev/null combined
#CustomLog /var/log/apache2/access.log combined
<FilesMatch \.php$>
SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
</FilesMatch>
<Directory /var/www/4get> <Directory /var/www/4get>
Options -MultiViews Options -MultiViews
AllowOverride All AllowOverride All
Require all granted Require all granted
# prevent 404s from hitting php-fpm
RewriteEngine On RewriteEngine On
RewriteCond %{REQUEST_URI} \.php$ [NC]
RewriteCond %{REQUEST_FILENAME} !-f
RewriteRule ^ - [R=404,L]
# prevent /index hits
RewriteRule ^index$ - [R=404,L]
# 404 direct .php file access
RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
RewriteRule ^ - [R=404,L]
# internally redirect to non-php extension
RewriteCond %{REQUEST_FILENAME} !-d RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-f RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME}.php -f
RewriteRule ^([^\.]+)$ $1.php [NC,L] RewriteRule ^([^\.]+)$ $1.php [NC,L]
</Directory> </Directory>
# deny access to private resources # deny access to private resources
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/"> <DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
Require all denied Require all denied
</DirectoryMatch> </DirectoryMatch>
# remove directory listing from /icons
<Directory /var/www/4get/icons> <Directory /var/www/4get/icons>
Options -Indexes Options -Indexes
</Directory> </Directory>
</VirtualHost> </VirtualHost>
```
<VirtualHost *:443>
ServerName www.lolcat.ca # Haproxy configuration
SSLEngine On ```cfg
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem global
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem #log /dev/log local0
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem log /dev/log local0 warning
log /dev/log local1 notice
RedirectMatch 301 ^(.*)$ https://lolcat.ca$1 chroot /var/lib/haproxy
</VirtualHost> stats socket /run/haproxy/admin.sock mode 660 level admin
stats timeout 30s
<VirtualHost *:443> user haproxy
ServerName lolcat.ca group haproxy
daemon
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/lolcat # Default SSL material locations
ca-base /etc/ssl/certs
SSLEngine On crt-base /etc/ssl/private
SSLOptions +StdEnvVars
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
#ErrorLog ${APACHE_LOG_DIR}/error.log #ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl-default-bind-ciphers HIGH:!aNULL
AddOutputFilterByType DEFLATE application/json ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
AddOutputFilterByType DEFLATE application/javascript ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html # ja4 settings
AddOutputFilterByType DEFLATE text/plain tune.lua.bool-sample-conversion normal
AddOutputFilterByType DEFLATE text/css tune.ssl.capture-buffer-size 190
lua-load /etc/haproxy/lua/ja4.lua
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem defaults
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem log global
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem mode http
#option httplog
<Directory /var/www/lolcat> option dontlognull
Options -MultiViews timeout connect 5000
AllowOverride All timeout client 50000
Require all granted timeout server 50000
errorfile 400 /etc/haproxy/errors/400.http
RewriteEngine On errorfile 403 /etc/haproxy/errors/403.http
RewriteCond %{REQUEST_FILENAME} !-d errorfile 408 /etc/haproxy/errors/408.http
RewriteCond %{REQUEST_FILENAME} !-f errorfile 500 /etc/haproxy/errors/500.http
RewriteRule ^([^\.]+)$ $1.php [NC,L] errorfile 502 /etc/haproxy/errors/502.http
</Directory> errorfile 503 /etc/haproxy/errors/503.http
</VirtualHost> errorfile 504 /etc/haproxy/errors/504.http
<VirtualHost *:443> frontend http_80_in
ServerName www.nyym.co # bind tcp ip4/6
bind *:80
SSLEngine On bind [::]:80
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem mode http
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
# handle ip.lolcat.ca+onion addy on plain HTTP
RedirectMatch 301 ^(.*)$ https://nyym.co$1 acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
</VirtualHost>
# redirect to https
<VirtualHost *:443> http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
ServerName nyym.co http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
ServerAdmin will@lolcat.ca http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
DocumentRoot /var/www/nyym http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
SSLEngine On
SSLOptions +StdEnvVars # redirect www.lolcat.ca to lolcat.ca
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
#ErrorLog ${APACHE_LOG_DIR}/error.log
# everything else redirects to 4get.ca
AddOutputFilterByType DEFLATE application/json http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript # ... unless its an ip host, or the onion domain
AddOutputFilterByType DEFLATE text/html use_backend apache if is_ip_host
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css frontend https_443_in
# bind tcp ip4/6
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
<Directory /var/www/nyym> # bind udp ip4/6
Options -MultiViews #bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
AllowOverride All #bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
Require all granted
# tell client we support http3 now
RewriteEngine On #http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-f mode http
RewriteRule ^([^\.]+)$ $1.php [NC,L]
</Directory> # redirect all unknown domains to 4get.ca
</VirtualHost> acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
<VirtualHost *:443> # add onion domain to 4get.ca
ServerName git.lolcat.ca acl is_4get hdr(host) -i 4get.ca
http-request set-var(txn.is_4get) bool(true) if is_4get
SSLEngine On http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
SSLOptions +StdEnvVars
# git.lolcat.ca -> gitea service
#ErrorLog ${APACHE_LOG_DIR}/error.log acl acl_gitea hdr(host) -i git.lolcat.ca
AddOutputFilterByType DEFLATE application/json # redirects
AddOutputFilterByType DEFLATE application/javascript http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
AddOutputFilterByType DEFLATE application/x-javascript http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain # add security headers
AddOutputFilterByType DEFLATE text/css http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
http-response set-header X-Content-Type-Options "nosniff"
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem http-response set-header X-Frame-Options "SAMEORIGIN"
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem # route
use_backend gitea if acl_gitea
ProxyPreserveHost On default_backend apache
ProxyRequests off
AllowEncodedSlashes NoDecode backend apache
ProxyPass / http://localhost:3000/ nocanon # pass ja4
</VirtualHost> http-request lua.fingerprint_ja4 if { ssl_fc }
http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
<VirtualHost *:443>
ServerName live.lolcat.ca # delete server header
http-response del-header Server
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/live # pass remote IP
option forwardfor
SSLEngine On server apache_server 127.0.0.1:8080 check
SSLOptions +StdEnvVars
backend gitea
#ErrorLog ${APACHE_LOG_DIR}/error.log server gitea_server 127.0.0.1:3000 check
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
</VirtualHost>
``` ```
+2 -2
View File
@@ -12,9 +12,9 @@ class backend{
/* /*
Proxy stuff Proxy stuff
*/ */
public function get_ip($proxy_index_raw = null){ public function get_ip($proxy_index_raw = null, $use_alt_list = false){
$pool = constant("config::PROXY_" . strtoupper($this->scraper)); $pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
if($pool === false){ if($pool === false){
// we don't want a proxy, fuck off! // we don't want a proxy, fuck off!
+1 -2
View File
@@ -1119,7 +1119,6 @@ class frontend{
$filters["scraper"] = [ $filters["scraper"] = [
"display" => "Scraper", "display" => "Scraper",
"option" => [ "option" => [
//"fget" => "fget",
"ddg" => "DuckDuckGo", "ddg" => "DuckDuckGo",
//"yahoo" => "Yahoo!", //"yahoo" => "Yahoo!",
"brave" => "Brave", "brave" => "Brave",
@@ -1207,7 +1206,7 @@ class frontend{
"yahoo_japan" => "Yahoo! JAPAN", "yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage", "startpage" => "Startpage",
//"mojeek" => "Mojeek", //"mojeek" => "Mojeek",
"baidu" => "Baidu" //"baidu" => "Baidu"
] ]
]; ];
break; break;
+26 -13
View File
@@ -3,32 +3,45 @@
header("Content-Type: application/xml"); header("Content-Type: application/xml");
include "data/config.php"; include "data/config.php";
$domain = $host = strtolower(explode(":", $_SERVER["HTTP_HOST"], 2)[0]);
htmlspecialchars(
(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on" ? "https" : "http") .
'://' . $_SERVER["HTTP_HOST"]
);
// detect if 4get should serve the HTTPS endpoint
if( if(
preg_match( preg_match(
'/\.onion$/', '/\.(onion|i2p)$/',
$domain strtolower($host),
$match
) )
){ ){
$onion = true; $ssl = false;
$src = $match[1];
}elseif(
is_string(
filter_var(
$host,
FILTER_VALIDATE_IP
)
) ||
$host == "localhost"
){
$ssl = false;
$src = "localhost";
}else{ }else{
$onion = false; $ssl = true;
} }
$domain = ($ssl ? "https" : "http") . "://" . $_SERVER["HTTP_HOST"];
echo echo
'<?xml version="1.0" encoding="UTF-8"?>' .
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' . '<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($onion ? " (onion)" : "") . '</ShortName>' . '<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($ssl === false ? " ({$src})" : "") . '</ShortName>' .
'<Description>' . htmlspecialchars(config::SERVER_SHORT_DESCRIPTION) . '</Description>' .
'<InputEncoding>UTF-8</InputEncoding>' . '<InputEncoding>UTF-8</InputEncoding>' .
'<Image width="16" height="16">' . $domain . '/favicon.ico</Image>' . '<Image width="32" height="32" type="image/x-icon">' . htmlspecialchars($domain) . '/favicon.ico</Image>' .
'<Url type="text/html" method="GET" template="' . $domain . '/web?s={searchTerms}"/>'; '<Url rel="results" type="text/html" method="GET" template="' . htmlspecialchars($domain) . '/web?s={searchTerms}"/>';
if( if(
isset($_GET["ac"]) && isset($_GET["ac"]) &&
+261 -1044
View File
File diff suppressed because it is too large. Load diff
+1047 -147
View File
File diff suppressed because it is too large. Load diff
+303 -51
View File
@@ -10,8 +10,9 @@ class google{
include "lib/backend.php"; include "lib/backend.php";
$this->backend = new backend("google"); $this->backend = new backend("google");
$this->proc = curl_multi_init(); $this->urls_to_resolve = [];
$this->handles = [];
$this->init_multi_curl();
} }
public function getfilters($page){ public function getfilters($page){
@@ -617,23 +618,57 @@ class google{
]; ];
} }
private function redirect_add_url($url){ private function init_multi_curl(){
$this->proc = curl_multi_init();
$this->handles = [];
}
private function redirect_add_url($url, $replace = null){
if( if(
preg_match( preg_match(
'/^\/goto\?url=([^&]+)/', '/^\/goto\?url=([^&]+)/',
$url, $url,
$slug $slug
) === 0 ) === 1
){ ){
// not a google redirect // normal goto
return $url; $this->urls_to_resolve[] = [
"replace" => $url,
"slug" => $slug[1],
"type" => "goto"
];
} }
elseif(
preg_match(
'/https?:\/\/share\.google\/([^"]+)+/',
$url,
$link_regex
) === 1
){
// share.google
$this->urls_to_resolve[] = [
"replace" => $replace,
"slug" => $link_regex[1],
"type" => "share"
];
}
// rest is ignored, since its not a redirect
// always return url
return $url;
}
private function urlfuck_goto($url, $slug){
$curlproc = curl_init(); $curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1])); curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER, curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -658,7 +693,7 @@ class google{
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $proxy); //$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc); curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [ $this->handles[] = [
@@ -669,7 +704,92 @@ class google{
return $url; return $url;
} }
private function resolve_urls(&$scraped_data){ private function urlfuck_shareservice($url, $slug){
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
"replacement" => $url,
"handle" => $curlproc
];
return $url;
}
private function urlfuck_share($url, $slug){
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
"replacement" => $url,
"handle" => $curlproc
];
return $url;
}
private function resolve_gotos(&$scraped_data){
foreach($this->urls_to_resolve as $item){
$this->urlfuck_goto($item["replace"], $item["slug"]);
}
do{ do{
do{ do{
@@ -706,20 +826,163 @@ class google{
if(count($a) === 0){ if(count($a) === 0){
throw new Exception("Failed to find link"); // failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
} }
$link = if(
$this->fuckhtml (
->getTextContent( $fallback === false &&
$a[0]["attributes"]["href"] preg_match(
); '/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){
$keys[] = $this->handles[$i]["replacement"];
$replacements[] = $link;
}
}
$this->recursive_replace($keys, $replacements, $scraped_data);
curl_multi_close($this->proc);
}
private function resolve_share(&$scraped_data){
foreach($this->urls_to_resolve as $item){
$this->urlfuck_shareservice($item["replace"], $item["slug"]);
}
do{
do{
$status = curl_multi_exec($this->proc, $active);
}while($status === CURLM_CALL_MULTI_PERFORM);
if($active){
curl_multi_select($this->proc);
}
}while($active && $status === CURLM_OK);
//
// if we reach this, we're done downloading garbage
//
// get the share links
$this->urls_to_resolve = [];
for($i=0; $i<count($this->handles); $i++){
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
if( if(
preg_match( preg_match(
'/^(https?:|)\/\//', '/https?:\/\/share\.google\/(?:[^"]+)/',
$link $html,
) $link_regex
) !== 1
){
// failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link = $link_regex[0];
}
if(
(
$fallback === false &&
preg_match(
'/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){
$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
}
}
// resolve shareable urls
$this->init_multi_curl();
foreach($this->urls_to_resolve as $item){
$this->urlfuck_share($item["replace"], $item["slug"]);
}
do{
do{
$status = curl_multi_exec($this->proc, $active);
}while($status === CURLM_CALL_MULTI_PERFORM);
if($active){
curl_multi_select($this->proc);
}
}while($active && $status === CURLM_OK);
// parse downloaded garbage
$keys = [];
$replacements = [];
for($i=0; $i<count($this->handles); $i++){
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
$this->fuckhtml->load($html);
$a =
$this->fuckhtml
->getElementsByTagName(
"a"
);
if(count($a) === 0){
// failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
}
if(
(
$fallback === false &&
preg_match(
'/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){ ){
$keys[] = $this->handles[$i]["replacement"]; $keys[] = $this->handles[$i]["replacement"];
@@ -785,22 +1048,6 @@ class google{
throw new Exception("Search term is empty!"); throw new Exception("Search term is empty!");
} }
/*
$session = $this->backend->session_get();
if(config::FPLAY_ENABLE_API === false){
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
}
if($session === false){
throw new Exception("Session database is empty");
}
$proxy = &$session["proxy"];
$headers = &$session["headers"];
*/
$proxy = $this->backend->get_ip(); $proxy = $this->backend->get_ip();
@@ -817,7 +1064,7 @@ class google{
// country // country
if($get["country"] != "any"){ if($get["country"] != "any"){
$params["gl"] = $country; $params["gl"] = $get["country"];
} }
// nsfw // nsfw
@@ -826,15 +1073,15 @@ class google{
// language // language
if($get["lang"] != "any"){ if($get["lang"] != "any"){
$params["lr"] = "lang_" . $lang; $params["lr"] = "lang_" . $get["lang"];
} }
// generate tbs // generate tbs
$tbs = []; $tbs = [];
// get date // get date
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older); $get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer); $get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
if( if(
$get["older"] !== null || $get["older"] !== null ||
@@ -1466,7 +1713,7 @@ class google{
]; ];
} }
$this->resolve_urls($out); $this->resolve_share($out);
return $out; return $out;
} }
@@ -1478,15 +1725,15 @@ class google{
// generate parameters // generate parameters
if($get["npt"]){ if($get["npt"]){
[$params, $proxy] = [$params_json, $proxy] =
$this->backend->get( $this->backend->get(
$get["npt"], $get["npt"],
"images" "images"
); );
$params = json_decode($params, true); $params_json = json_decode($params_json, true);
$container = $params["c"]; $container = &$params_json["c"];
unset($params["c"]); $params = &$params_json["p"];
}else{ }else{
$search = $get["s"]; $search = $get["s"];
@@ -1505,6 +1752,7 @@ class google{
$type = $get["type"]; $type = $get["type"];
$format = $get["format"]; $format = $get["format"];
$rights = $get["rights"]; $rights = $get["rights"];
$container = null;
$params = [ $params = [
"q" => $search, "q" => $search,
@@ -1593,17 +1841,18 @@ class google{
} }
} }
//$html = file_get_contents("scraper/google.html");
$data = $this->get( $data = $this->get(
$proxy, $proxy,
"https://www.google.com/search", "https://www.google.com/search",
$params $params,
$container
); );
$html = &$data["data"]; $html = &$data["data"];
$container = &$data["container"]; $container = &$data["container"];
//$html = file_get_contents("scraper/google.html");
$this->fuckhtml->load($html); $this->fuckhtml->load($html);
// parse all <style> tags // parse all <style> tags
@@ -1701,13 +1950,16 @@ class google{
$out["npt"] = $out["npt"] =
$this->backend $this->backend
->store( ->store(
json_encode($params), json_encode([
"p" => $params,
"c" => $container
]),
"image", "image",
$proxy $proxy
); );
} }
$this->resolve_urls($out); $this->resolve_gotos($out);
return $out; return $out;
} }
@@ -2115,7 +2367,7 @@ class google{
]; ];
} }
$this->resolve_urls($out); $this->resolve_share($out);
return $out; return $out;
} }
@@ -2474,14 +2726,14 @@ class google{
"date" => $date, "date" => $date,
"thumb" => $thumb, "thumb" => $thumb,
"url" => "url" =>
$this->unshiturl( $this->redirect_add_url(
$a["attributes"] $a["attributes"]
["href"] ["href"]
) )
]; ];
} }
$this->resolve_urls($out); $this->resolve_share($out);
return $out; return $out;
} }
+2
View File
@@ -567,6 +567,8 @@ class google_api{
foreach($json["items"] as $result){ foreach($json["items"] as $result){
$date = null;
// //
// probe for thumbnail // probe for thumbnail
// //
+17 -8
View File
@@ -725,18 +725,27 @@ class google_cse{
// calculate rounded ratio // calculate rounded ratio
if($found_size){ if($found_size){
$aspect_ratio = $width / $height; if(
$width === 0 ||
if($aspect_ratio >= 1.5){ $height === 0
){
$thumb["ratio"] = "16:9";
}
elseif($aspect_ratio >= 0.8){
$thumb["ratio"] = "1:1"; $thumb["ratio"] = "1:1";
}else{ }else{
$thumb["ratio"] = "9:16"; $aspect_ratio = $width / $height;
if($aspect_ratio >= 1.5){
$thumb["ratio"] = "16:9";
}
elseif($aspect_ratio >= 0.8){
$thumb["ratio"] = "1:1";
}else{
$thumb["ratio"] = "9:16";
}
} }
} }
} }
+146 -95
View File
@@ -261,9 +261,10 @@ class naver{
$filters = [ $filters = [
"nso" => "", "nso" => "",
"page" => 2, // page 2 is repeat of page 1
"query" => $search, "query" => $search,
"sm" => "tab_pge", "sm" => "tab_pge",
"where" => "web", "ssc" => "tab.ur.all",
"start" => 1 // increment by number of results each time (16??) "start" => 1 // increment by number of results each time (16??)
]; ];
@@ -304,13 +305,13 @@ class naver{
$this->fuckhtml->load($html); $this->fuckhtml->load($html);
$results = $jsons =
preg_split( preg_split(
'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/', '/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
$html $html
); );
if(count($results) !== 2){ if(count($jsons) <= 1){
// this is thrown when no results are found // this is thrown when no results are found
@@ -342,119 +343,169 @@ class naver{
//throw new Exception("Failed to grep results entrypoint"); //throw new Exception("Failed to grep results entrypoint");
} }
$json = foreach($jsons as $json){
json_decode(
$this->fuckhtml
->extract_json(
$results[1]
),
true
);
if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){ $json =
json_decode(
$this->fuckhtml
->extract_json(
$json
),
true
);
throw new Exception("Failed to access nested children"); if($json === null){
}
foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){
if(
!isset($result["templateId"]) ||
$result["templateId"] != "webItem"
){
// should not happen
continue; continue;
} }
$result = $result["props"]; if(!isset($json["data"]["state"]["items"])){
// get sublinks //throw new Exception("Failed to access nested children");
$sublinks = []; continue;
}
if(isset($result["subLinks"])){ foreach($json["data"]["state"]["items"] as $result){
foreach($result["subLinks"] as $s){ // get sublinks
$sublinks = [];
$sublinks[] = [ if(isset($result["subLinks"])){
"title" => $s["text"],
"description" => null, foreach($result["subLinks"] as $s){
"url" => $s["href"],
"date" => null $sublinks[] = [
"title" => $s["text"],
"description" => null,
"url" => $s["href"],
"date" => null
];
}
}
if(isset($result["linkBtns"])){
foreach($result["linkBtns"] as $s){
$sublinks[] = [
"title" => $s["text"],
"description" => null,
"url" => $s["href"],
"date" => null
];
}
}
// get image (thumbnail, i guess)
if(isset($result["images"][0]["imageSrc"])){
$thumb = [
"ratio" => "16:9",
"url" => $this->unshit_thumb($result["images"][0]["imageSrc"])
];
}else{
$thumb = [
"ratio" => null,
"url" => null
]; ];
} }
}
if(isset($result["linkBtns"])){ // get table elements
$table = [];
foreach($result["linkBtns"] as $s){ if(isset($result["keyValue"]["contents"])){
$sublinks[] = [ foreach($result["keyValue"]["contents"] as $s){
"title" => $s["text"],
"description" => null, if(!isset($s["valueData"]["text"])){ continue; }
"url" => $s["href"],
"date" => null $table[$s["key"]] = $s["valueData"]["text"];
]; }
} }
}
// get image (thumbnail, i guess) // get date
if(isset($result["images"][0]["imageSrc"])){ $time = null;
$thumb = [ if(isset($result["profile"]["createdDate"])){
"ratio" => "16:9",
"url" => $this->unshit_thumb($result["images"][0]["imageSrc"])
];
}else{
$thumb = [ $time = $result["profile"]["createdDate"];
"ratio" => null, }elseif(isset($result["sourceProfile"]["createdDate"])){
"url" => null
$time = $result["sourceProfile"]["createdDate"];
}
if($time !== null){
$date =
strtotime(
substr(
$time,
-1
)
);
if($date !== false){
$time = $date;
}else{
$time = null;
}
}
// get title
if(isset($result["title"])){
$title = $result["title"];
}elseif(isset($result["questionTitle"])){
$title = $result["questionTitle"];
}else{
continue;
}
// get url
if(isset($result["titleHref"])){
$href = $result["titleHref"];
}elseif(isset($result["href"])){
$href = $result["href"];
}elseif(isset($result["answerHref"])){
$href = $result["answerHref"];
}else{
continue;
}
// get description
if(isset($result["bodyText"])){
$description = $result["bodyText"];
}elseif(isset($result["content"])){
$description = $result["content"];
}elseif(isset($result["questionContent"])){
$description = $result["questionContent"];
}else{
$description = null;
}
$out["web"][] = [
"title" => $this->decode_html($title),
"description" => $this->decode_html($description),
"url" => $href,
"date" => $time,
"type" => "web",
"thumb" => $thumb,
"sublink" => $sublinks,
"table" => $table
]; ];
} }
// get table elements
$table = [];
if(isset($result["keyValue"]["contents"])){
foreach($result["keyValue"]["contents"] as $s){
if(!isset($s["valueData"]["text"])){ continue; }
$table[$s["key"]] = $s["valueData"]["text"];
}
}
// get date
$time = null;
if(isset($result["bodyPrefixes"][0]["text"])){
$date =
strtotime(
substr(
$result["bodyPrefixes"][0]["text"],
-1
)
);
if($date !== false){
$time = $date;
}
}
$out["web"][] = [
"title" => $this->decode_html($result["title"]),
"description" => $this->decode_html($result["bodyText"]),
"url" => $result["href"],
"date" => $time,
"type" => "web",
"thumb" => $thumb,
"sublink" => $sublinks,
"table" => $table
];
} }
// get next page // get next page
+169 -27
View File
@@ -545,17 +545,20 @@ class startpage{
switch($category["display_type"]){ switch($category["display_type"]){
case "web-google": case "web-google":
case "web-bing":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
$sublinks = []; $sublinks = [];
foreach($result["siteLinks"] as $sublink){ if(isset($result["siteLinks"])){
foreach($result["siteLinks"] as $sublink){
$sublinks[] = [ $sublinks[] = [
"title" => $sublink["title"], "title" => $sublink["title"],
"description" => null, "description" => null,
"url" => $sublink["clickUrl"] "url" => $sublink["clickUrl"]
]; ];
}
} }
$description = $description =
@@ -596,15 +599,17 @@ class startpage{
$out["web"][] = [ $out["web"][] = [
"title" => "title" =>
$this->titledots( $this->remove_penguins(
html_entity_decode( $this->titledots(
$this->fuckhtml html_entity_decode(
->getTextContent( $this->fuckhtml
$result["title"] ->getTextContent(
$result["title"]
)
) )
) )
), ),
"description" => $description, "description" => $this->remove_penguins($description),
"url" => $result["clickUrl"], "url" => $result["clickUrl"],
"date" => $date, "date" => $date,
"type" => "web", "type" => "web",
@@ -619,8 +624,43 @@ class startpage{
break; break;
case "images-qi-top": case "images-qi-top":
case "images-bing-top":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
if(isset($result["mdThumbnailWidth"])){
$thumb_width = $result["mdThumbnailWidth"];
}elseif(isset($result["thumbnailWidth"])){
$thumb_width = $result["thumbnailWidth"];
}else{
$thumb_width = null;
}
if(isset($result["mdThumbnailHeight"])){
$thumb_height = $result["mdThumbnailHeight"];
}elseif(isset($result["thumbnailHeight"])){
$thumb_height = $result["thumbnailHeight"];
}else{
$thumb_height = null;
}
if(isset($result["mdThumbnailUrl"])){
$thumb_source = $result["mdThumbnailUrl"];
}elseif(isset($result["thumbnailUrl"])){
$thumb_source = $result["thumbnailUrl"];
}else{
// should not happen
continue;
}
$out["image"][] = [ $out["image"][] = [
"title" => "title" =>
$this->titledots( $this->titledots(
@@ -638,19 +678,17 @@ class startpage{
"height" => (int)$result["height"] "height" => (int)$result["height"]
], ],
[ [
"url" => $this->unshitimage($result["mdThumbnailUrl"]), "url" => $this->unshitimage($thumb_source),
"width" => (int)$result["mdThumbnailWidth"], "width" => (int)$thumb_width,
"height" => (int)$result["mdThumbnailHeight"] "height" => (int)$thumb_height
] ]
], ],
"url" => "url" => $result["altClickUrl"]
$result["altClickUrl"]
]; ];
} }
break; break;
case "spellsuggest-google": case "spellsuggest-google":
if(isset($category["results"][0]["query"])){ if(isset($category["results"][0]["query"])){
$out["spelling"] = $out["spelling"] =
@@ -662,6 +700,60 @@ class startpage{
} }
break; break;
case "spelladjust-bing":
if(isset($category["results"][0]["alterationDisplayQuery"])){
$out["spelling"] =
[
"type" => "including",
"using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]),
"correction" => $category["results"][0]["originalQuery"]
];
}
break;
case "aylf-bing-top":
if(isset($category["results"][0]["query"])){
foreach($category["results"] as $ddm){
$out["related"][] = $ddm["query"];
}
}
break;
case "video-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $video){
if(isset($video["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $this->unshitimage($video["thumbnailUrl"])
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["video"][] = [
"title" => $video["title"],
"description" => $this->remove_penguins($video["description"]),
"date" => isset($video["date"]) ? strtotime($video["date"]) : null,
"duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null,
"views" => $video["viewCount"],
"thumb" => $thumb,
"url" => $video["clickUrl"]
];
}
}
break;
case "dictionary-qi": case "dictionary-qi":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
@@ -732,6 +824,36 @@ class startpage{
$out["answer"][] = $answer; $out["answer"][] = $answer;
} }
break; break;
case "news-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $news){
if(isset($news["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $news["thumbnailUrl"]
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["news"][] = [
"title" => $this->remove_penguins($news["title"]),
"description" => $news["description"],
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb,
"url" => $news["clickUrl"],
];
}
}
break;
} }
} }
@@ -1246,14 +1368,10 @@ class startpage{
// get npt // get npt
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies); $out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
//print_r($json);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){ foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
if($category["display_type"] != "news-bing"){
// unsupported category
continue;
}
foreach($category["results"] as $news){ foreach($category["results"] as $news){
if( if(
@@ -1276,10 +1394,10 @@ class startpage{
$out["news"][] = [ $out["news"][] = [
"title" => $this->titledots($this->remove_penguins($news["title"])), "title" => $this->titledots($this->remove_penguins($news["title"])),
"author" => $news["source"], "author" => $news["source"],
"description" => $this->titledots($this->remove_penguins($news["description"])), "description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null,
"date" => (int)substr((string)$news["date"], 0, -3), "date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb, "thumb" => $thumb,
"url" => $news["clickUrl"] "url" => $this->remove_tracking_link($news["clickUrl"])
]; ];
} }
} }
@@ -1327,6 +1445,30 @@ class startpage{
return null; return null;
} }
private function remove_tracking_link($url){
// https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog
$query = parse_url($url);
if(
!isset($query["host"]) ||
$query["host"] != "api.airfind.com" ||
!isset($query["query"])
){
return $url;
}
parse_str($query["query"], $res);
if(isset($res["redirectURL"])){
return $res["redirectURL"];
}
return $url;
}
private function unshitimage($url){ private function unshitimage($url){
$query = parse_url($url, PHP_URL_QUERY); $query = parse_url($url, PHP_URL_QUERY);
+31 -70
View File
@@ -205,13 +205,6 @@ class yep{
"zh-tw" => "Chinese (Traditional)", "zh-tw" => "Chinese (Traditional)",
"zu" => "Zulu" "zu" => "Zulu"
] ]
],
"nsfw" => [
"display" => "NSFW",
"option" => [
"yes" => "Yes",
"no" => "No"
]
] ]
]; ];
} }
@@ -317,20 +310,23 @@ class yep{
// parse filters // parse filters
$filters = [ $filters = [
"limit" => 100, // wwwwwwwwwwwwwww "q" => $search,
"query" => $search, "limit" => 10 // no fun allowed
]; ];
if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; } if($get["lang"] != "any"){
if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
$filters["language"] = $get["lang"];
$filters["lang"] = $get["lang"];
}
try{ try{
// https://api.yep.com/search?limit=20&query=asmr // https://yep.com/web/v1/search?q=asmr&limit=10
$json = $json =
$this->get( $this->get(
$this->backend->get_ip(), $this->backend->get_ip(),
"https://api.yep.com/search", "https://yep.com/web/v1/search",
$filters $filters
); );
@@ -343,71 +339,37 @@ class yep{
$json = json_decode($json, true); $json = json_decode($json, true);
//$json = json_decode(file_get_contents("scraper/yep.json"), true); //$json = json_decode(file_get_contents("scraper/yep.json"), true);
if($json === null){ if($json === null){
throw new Exception("Failed to decode JSON"); throw new Exception("Failed to decode JSON");
} }
if(isset($json[1]["correction"])){ if(isset($json["code"])){
$out["spelling"] = [ if(isset($json["reason"])){
"type" => "not_many",
"using" => $search, throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
"correction" => $json[1]["correction"][1] }
];
throw new Exception("Yep returned an unknown error");
} }
if(isset($json[1]["results"])){ if(isset($json["results"])){
foreach($json[1]["results"] as $item){ foreach($json["results"] as $item){
switch(strtolower($item["type"])){ $out["web"][] = [
"title" => $item["title"],
case "organic": "description" => str_replace("\n", " ", $item["description"]),
$sublinks = []; "url" => $item["url"],
"date" => null,
if(isset($item["sitelinks"]["full"])){ "type" => "web",
"thumb" => [
foreach($item["sitelinks"]["full"] as $link){ "url" => null,
"ratio" => null
$sublinks[] = [ ],
"title" => $link["title"], "sublink" => [],
"date" => null, "table" => []
"description" => ];
$this->titledots(
strip_tags(
html_entity_decode(
$link["snippet"]
)
)
),
"url" => $link["url"]
];
}
}
$out["web"][] = [
"title" => $item["title"],
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$item["snippet"]
)
)
),
"url" => $item["url"],
"date" => null,
"type" => "web",
"thumb" => [
"url" => null,
"ratio" => null
],
"sublink" => $sublinks,
"table" => []
];
break;
}
} }
} }
@@ -445,7 +407,6 @@ class yep{
"limit" => 99 // for some reason they dont show anything sometimes at 100 results "limit" => 99 // for some reason they dont show anything sometimes at 100 results
]; ];
if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; } if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
// add api key // add api key
+10 -1
View File
@@ -678,6 +678,15 @@ class yt{
$video["longBylineText"] $video["longBylineText"]
), ),
"url" => "url" =>
isset(
$video["ownerText"]
["runs"]
[0]
["navigationEndpoint"]
["commandMetadata"]
["webCommandMetadata"]
["url"]
) ?
"https://www.youtube.com" . "https://www.youtube.com" .
$video["ownerText"] $video["ownerText"]
["runs"] ["runs"]
@@ -685,7 +694,7 @@ class yt{
["navigationEndpoint"] ["navigationEndpoint"]
["commandMetadata"] ["commandMetadata"]
["webCommandMetadata"] ["webCommandMetadata"]
["url"], ["url"] : null,
"avatar" => "avatar" =>
$video["channelThumbnailSupportedRenderers"] $video["channelThumbnailSupportedRenderers"]
["channelThumbnailWithLinkRenderer"] ["channelThumbnailWithLinkRenderer"]
+28 -4
View File
@@ -67,6 +67,10 @@ $settings = [
"value" => "brave", "value" => "brave",
"text" => "Brave" "text" => "Brave"
], ],
[
"value" => "bing",
"text" => "Bing"
],
[ [
"value" => "ddg", "value" => "ddg",
"text" => "DuckDuckGo" "text" => "DuckDuckGo"
@@ -79,17 +83,25 @@ $settings = [
"value" => "google", "value" => "google",
"text" => "Google" "text" => "Google"
], ],
[
"value" => "yahoo",
"text" => "Yahoo"
],
[ [
"value" => "startpage", "value" => "startpage",
"text" => "Startpage" "text" => "Startpage"
], ],
[ [
"value" => "kagi", "value" => "qwant",
"text" => "Kagi" "text" => "Qwant"
], ],
[ [
"value" => "yep", "value" => "ecosia",
"text" => "Yep" "text" => "Ecosia"
],
[
"value" => "kagi",
"text" => "Kagi"
], ],
[ [
"value" => "marginalia", "value" => "marginalia",
@@ -102,6 +114,18 @@ $settings = [
[ [
"value" => "sc", "value" => "sc",
"text" => "SoundCloud" "text" => "SoundCloud"
],
[
"value" => "coccoc",
"text" => "Coccoc"
],
[
"value" => "naver",
"text" => "Naver"
],
[
"value" => "purili",
"text" => "Purili"
] ]
] ]
], ],
+2 -2
View File
@@ -349,8 +349,8 @@ if(count($results["answer"]) !== 0){
$right["answer"] .= $right["answer"] .=
'<tr>' . '<tr>' .
'<td>' . $info . '</td>' . '<td>' . htmlspecialchars($info) . '</td>' .
'<td>' . $value . '</td>' . '<td>' . htmlspecialchars($value) . '</td>' .
'</tr>'; '</tr>';
} }