Compare commits

...
25 Commits
Author SHA1 Message Date
lolcat 4c7c4ea072 forgot settings woopsies tehehe 2026-11-15 19:41:29 -05:00
lolcat 761a1e91cd added coccoc and naver autocomplete, added google aliases 2026-11-15 19:14:37 -05:00
lolcat c5311bc399 autocomplete fixes 2026-11-15 18:44:14 -05:00
lolcat 4536cef88f how is it fucking 4am again 2026-11-13 03:57:07 -05:00
lolcat 0c3844d578 added brave api support since theyre bragging about adding goto tracking links soon wtf are they even thinking 2026-11-13 03:56:13 -05:00
lolcat 30cd3c8492 brave fix fuck you brave 2026-11-12 11:06:28 -05:00
lolcat aa818ac925 im starting to understand chinese 2026-11-11 00:08:00 -05:00
lolcat 5967462b6e baidu captchawalls all requests, removing 2026-11-10 20:26:00 -05:00
lolcat b014a3199d ok this should work now, make sure to enable browser.search.suggest.enabled in ff aaaaaaaaaa 2026-11-08 16:50:37 -05:00
lolcat 24130effe6 autocomplete fixes 2026-11-08 16:33:02 -05:00
lolcat 5d8c9a5798 startpage fix 2026-11-08 15:55:20 -05:00
lolcat 6bbb37d86e yep fic 2026-11-07 02:09:26 -05:00
lolcat c0324d7d61 Merge pull request 'fix ami4get 500 error' (#101) from docker_fix_ami4get into master
Reviewed-on: #101
2026-11-01 13:03:03 -05:00
throwaway 2218fd6f04 change variable 2026-11-01 00:27:32 -07:00
lolcat be91ec1806 fix google news 2026-10-31 14:11:20 -04:00
lolcat 57a72e5b5e url redirect bypass 2026-10-31 14:01:52 -04:00
lolcat 9f0437d949 rotate ips on url resolution 2026-10-30 00:18:06 -04:00
lolcat d91834f86f fix filter bugs on google scraper 2026-10-29 23:56:12 -04:00
lolcat 42d0252e2f handle failed redirect fetches 2026-10-28 22:59:45 -04:00
lolcat a27a8ed01a fix potential xss with scraper supplied data in table, ty cynic 2026-10-27 01:12:32 -04:00
lolcat 9db95c79d2 update example apache2 config 2026-10-24 05:24:27 -04:00
lolcat 25d4f991c2 fix some php warnings 2026-10-24 04:03:00 -04:00
lolcat e1d0ab121f fix google cse crash 2026-10-24 03:24:52 -04:00
lolcat 33ce0a062b Merge branch 'master' of https://git.lolcat.ca/lolcat/4get 2026-10-24 03:19:13 -04:00
lolcat 3b3c759ce5 hyung..sunbae.. i fixed naver 2026-10-24 03:18:07 -04:00
19 changed files with 2170 additions and 1506 deletions

No files matched your search

+1 -1
View File
@@ -31,7 +31,7 @@ echo json_encode(
"real_requests" => $real_requests,
"bot_requests" => $bot_requests,
"api_enabled" => config::API_ENABLED,
"alt_addresses" => config::ALT_ADDRESSES,
"alt_addresses" => config::ADDRESSES,
"version" => config::VERSION
],
"instances" => config::INSTANCES
+116 -24
View File
@@ -7,37 +7,51 @@ class autocomplete{
public function __construct(){
header("Content-Type: application/json");
header("Content-Type: application/x-suggestions+json");
$this->scrapers = [
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
"bing" => "https://www.bing.com/osjson.aspx?query={searchTerms}",
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
"ecosia" => "https://ac.ecosia.org/autocomplete?q={searchTerms}&type=list",
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3",
"google" => "https://www.google.com/complete/search?client=chrome&q={searchTerms}",
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
"marginalia" => "https://marginalia-search.com/suggest/?partial={searchTerms}",
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
"sc" => "",
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
"kagi" => "https://kagisuggest.com/api/autosuggest?q={searchTerms}",
"startpage" => "https://www.startpage.com/osuggestions?q={searchTerms}",
"yahoo" => "https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=sd1&command={searchTerms}",
"purili" => "https://puri.li/api/suggest?q={searchTerms}",
"coccoc" => "https://coccoc.com/composer/autocomplete?q={searchTerms}",
"naver" => "https://ac.search.naver.com/nx/ac?q={searchTerms}&st=100"
];
// add duplicates
$this->scrapers["google_api"] = &$this->scrapers["google"];
$this->scrapers["google_cse"] = &$this->scrapers["google"];
$this->scrapers["yahoo_japan"] = &$this->scrapers["yahoo"];
/*
Sanitize input
*/
if(!isset($_GET["s"])){
$this->do404("Missing search(s) parameter");
// Missing search(s) parameter
$this->doempty();
}
if(is_string($_GET["s"]) === false){
$this->do404("Invalid search(s) parameter");
// Invalid search(s) parameter
$this->doempty();
}
if(strlen($_GET["s"]) > 500){
$this->do404("Search(s) exceeds the 500 char length");
// Search(s) exceeds the 500 char length
$this->doempty();
}
/*
@@ -72,9 +86,8 @@ class autocomplete{
// return results
switch($scraper){
case "google":
case "yt":
// handle google cause they want to be a special snowflake :(
// handle youtube cause it wants to be a special snowflake :(
$js = $this->get($this->scrapers[$scraper], $_GET["s"]);
preg_match(
@@ -141,12 +154,12 @@ class autocomplete{
break;
case "marginalia":
case "purili":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if($json === null){
$this->doempty();
}
@@ -159,10 +172,98 @@ class autocomplete{
);
break;
case "yahoo":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["r"][0]["k"])
){
$this->doempty();
}
$payload = [];
foreach($json["r"] as $term){
if(isset($term["k"])){
$payload[] = $term["k"];
}
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "naver":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["items"][0][0])
){
$this->doempty();
}
$payload = [];
foreach($json["items"][0] as $term){
$payload[] = $term[0];
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "coccoc":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["suggestions"])
){
$this->doempty();
}
echo json_encode(
[
$_GET["s"],
$json["suggestions"]
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
default:
// if it respects the openSearch protocol
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
if(
!isset($json[1]) ||
!is_array($json[1])
){
$json[1] = [];
}
echo json_encode(
[
$_GET["s"],
@@ -185,10 +286,10 @@ class autocomplete{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/javascript, */*; q=0.01",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
@@ -214,19 +315,10 @@ class autocomplete{
}catch(Exception $error){
do404("Curl error: " . $error->getMessage());
doempty();
}
}
private function do404($error){
echo json_encode(
["error" => $error],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
die();
}
private function doempty(){
echo json_encode(
+1
View File
@@ -0,0 +1 @@
+3 -2
View File
@@ -154,7 +154,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:160.0) Gecko/20100101 Firefox/160.0";
// User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
@@ -225,6 +225,7 @@ class config{
// use the API (Eg: "public"). API has less filters.
const MARGINALIA_API_KEY = null;
// Yep
// Scrapers to use the API on
const YEP_USE_API = false;
const BRAVE_USE_API = false;
}
+154 -163
View File
@@ -1,31 +1,16 @@
# Sample Apache2 configuration
This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
```xml
<VirtualHost *:443>
ServerName www.4get.ca
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
RedirectMatch 301 ^(.*)$ https://4get.ca$1
</VirtualHost>
<VirtualHost *:443>
<VirtualHost *:8080>
ServerName 4get.ca
ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/4get
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
# compress
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
DocumentRoot /var/www/4get
ErrorLog ${APACHE_LOG_DIR}/error.log
CustomLog /dev/null combined
#CustomLog /var/log/apache2/access.log combined
<FilesMatch \.php$>
SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
</FilesMatch>
<Directory /var/www/4get>
Options -MultiViews
AllowOverride All
Require all granted
# prevent 404s from hitting php-fpm
RewriteEngine On
RewriteCond %{REQUEST_URI} \.php$ [NC]
RewriteCond %{REQUEST_FILENAME} !-f
RewriteRule ^ - [R=404,L]
# prevent /index hits
RewriteRule ^index$ - [R=404,L]
# 404 direct .php file access
RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
RewriteRule ^ - [R=404,L]
# internally redirect to non-php extension
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-f
RewriteCond %{REQUEST_FILENAME}.php -f
RewriteRule ^([^\.]+)$ $1.php [NC,L]
</Directory>
# deny access to private resources
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
Require all denied
</DirectoryMatch>
# remove directory listing from /icons
<Directory /var/www/4get/icons>
Options -Indexes
</Directory>
</VirtualHost>
<VirtualHost *:443>
ServerName www.lolcat.ca
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
</VirtualHost>
<VirtualHost *:443>
ServerName lolcat.ca
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/lolcat
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
<Directory /var/www/lolcat>
Options -MultiViews
AllowOverride All
Require all granted
RewriteEngine On
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-f
RewriteRule ^([^\.]+)$ $1.php [NC,L]
</Directory>
</VirtualHost>
<VirtualHost *:443>
ServerName www.nyym.co
SSLEngine On
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
RedirectMatch 301 ^(.*)$ https://nyym.co$1
</VirtualHost>
<VirtualHost *:443>
ServerName nyym.co
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/nyym
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
<Directory /var/www/nyym>
Options -MultiViews
AllowOverride All
Require all granted
RewriteEngine On
RewriteCond %{REQUEST_FILENAME} !-d
RewriteCond %{REQUEST_FILENAME} !-f
RewriteRule ^([^\.]+)$ $1.php [NC,L]
</Directory>
</VirtualHost>
<VirtualHost *:443>
ServerName git.lolcat.ca
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
ProxyPreserveHost On
ProxyRequests off
AllowEncodedSlashes NoDecode
ProxyPass / http://localhost:3000/ nocanon
</VirtualHost>
<VirtualHost *:443>
ServerName live.lolcat.ca
ServerAdmin will@lolcat.ca
DocumentRoot /var/www/live
SSLEngine On
SSLOptions +StdEnvVars
#ErrorLog ${APACHE_LOG_DIR}/error.log
AddOutputFilterByType DEFLATE application/json
AddOutputFilterByType DEFLATE application/javascript
AddOutputFilterByType DEFLATE application/x-javascript
AddOutputFilterByType DEFLATE text/html
AddOutputFilterByType DEFLATE text/plain
AddOutputFilterByType DEFLATE text/css
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
</VirtualHost>
```
# Haproxy configuration
```cfg
global
#log /dev/log local0
log /dev/log local0 warning
log /dev/log local1 notice
chroot /var/lib/haproxy
stats socket /run/haproxy/admin.sock mode 660 level admin
stats timeout 30s
user haproxy
group haproxy
daemon
# Default SSL material locations
ca-base /etc/ssl/certs
crt-base /etc/ssl/private
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
ssl-default-bind-ciphers HIGH:!aNULL
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
# ja4 settings
tune.lua.bool-sample-conversion normal
tune.ssl.capture-buffer-size 190
lua-load /etc/haproxy/lua/ja4.lua
defaults
log global
mode http
#option httplog
option dontlognull
timeout connect 5000
timeout client 50000
timeout server 50000
errorfile 400 /etc/haproxy/errors/400.http
errorfile 403 /etc/haproxy/errors/403.http
errorfile 408 /etc/haproxy/errors/408.http
errorfile 500 /etc/haproxy/errors/500.http
errorfile 502 /etc/haproxy/errors/502.http
errorfile 503 /etc/haproxy/errors/503.http
errorfile 504 /etc/haproxy/errors/504.http
frontend http_80_in
# bind tcp ip4/6
bind *:80
bind [::]:80
mode http
# handle ip.lolcat.ca+onion addy on plain HTTP
acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
# redirect to https
http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
# redirect www.lolcat.ca to lolcat.ca
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
# everything else redirects to 4get.ca
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
# ... unless its an ip host, or the onion domain
use_backend apache if is_ip_host
frontend https_443_in
# bind tcp ip4/6
bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
# bind udp ip4/6
#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
# tell client we support http3 now
#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
mode http
# redirect all unknown domains to 4get.ca
acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
# add onion domain to 4get.ca
acl is_4get hdr(host) -i 4get.ca
http-request set-var(txn.is_4get) bool(true) if is_4get
http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
# git.lolcat.ca -> gitea service
acl acl_gitea hdr(host) -i git.lolcat.ca
# redirects
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
# add security headers
http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
http-response set-header X-Content-Type-Options "nosniff"
http-response set-header X-Frame-Options "SAMEORIGIN"
# route
use_backend gitea if acl_gitea
default_backend apache
backend apache
# pass ja4
http-request lua.fingerprint_ja4 if { ssl_fc }
http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
# delete server header
http-response del-header Server
# pass remote IP
option forwardfor
server apache_server 127.0.0.1:8080 check
backend gitea
server gitea_server 127.0.0.1:3000 check
```
+2 -2
View File
@@ -12,9 +12,9 @@ class backend{
/*
Proxy stuff
*/
public function get_ip($proxy_index_raw = null){
public function get_ip($proxy_index_raw = null, $use_alt_list = false){
$pool = constant("config::PROXY_" . strtoupper($this->scraper));
$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
if($pool === false){
// we don't want a proxy, fuck off!
+1 -2
View File
@@ -1119,7 +1119,6 @@ class frontend{
$filters["scraper"] = [
"display" => "Scraper",
"option" => [
//"fget" => "fget",
"ddg" => "DuckDuckGo",
//"yahoo" => "Yahoo!",
"brave" => "Brave",
@@ -1207,7 +1206,7 @@ class frontend{
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
//"mojeek" => "Mojeek",
"baidu" => "Baidu"
//"baidu" => "Baidu"
]
];
break;
+26 -13
View File
@@ -3,32 +3,45 @@
header("Content-Type: application/xml");
include "data/config.php";
$domain =
htmlspecialchars(
(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on" ? "https" : "http") .
'://' . $_SERVER["HTTP_HOST"]
);
$host = strtolower(explode(":", $_SERVER["HTTP_HOST"], 2)[0]);
// detect if 4get should serve the HTTPS endpoint
if(
preg_match(
'/\.onion$/',
$domain
'/\.(onion|i2p)$/',
strtolower($host),
$match
)
){
$onion = true;
$ssl = false;
$src = $match[1];
}elseif(
is_string(
filter_var(
$host,
FILTER_VALIDATE_IP
)
) ||
$host == "localhost"
){
$ssl = false;
$src = "localhost";
}else{
$onion = false;
$ssl = true;
}
$domain = ($ssl ? "https" : "http") . "://" . $_SERVER["HTTP_HOST"];
echo
'<?xml version="1.0" encoding="UTF-8"?>' .
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($onion ? " (onion)" : "") . '</ShortName>' .
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($ssl === false ? " ({$src})" : "") . '</ShortName>' .
'<Description>' . htmlspecialchars(config::SERVER_SHORT_DESCRIPTION) . '</Description>' .
'<InputEncoding>UTF-8</InputEncoding>' .
'<Image width="16" height="16">' . $domain . '/favicon.ico</Image>' .
'<Url type="text/html" method="GET" template="' . $domain . '/web?s={searchTerms}"/>';
'<Image width="32" height="32" type="image/x-icon">' . htmlspecialchars($domain) . '/favicon.ico</Image>' .
'<Url rel="results" type="text/html" method="GET" template="' . htmlspecialchars($domain) . '/web?s={searchTerms}"/>';
if(
isset($_GET["ac"]) &&
+223 -1006
View File
File diff suppressed because it is too large. Load diff
+1047 -147
View File
File diff suppressed because it is too large. Load diff
+295 -43
View File
@@ -10,8 +10,9 @@ class google{
include "lib/backend.php";
$this->backend = new backend("google");
$this->proc = curl_multi_init();
$this->handles = [];
$this->urls_to_resolve = [];
$this->init_multi_curl();
}
public function getfilters($page){
@@ -617,23 +618,57 @@ class google{
];
}
private function redirect_add_url($url){
private function init_multi_curl(){
$this->proc = curl_multi_init();
$this->handles = [];
}
private function redirect_add_url($url, $replace = null){
if(
preg_match(
'/^\/goto\?url=([^&]+)/',
$url,
$slug
) === 0
) === 1
){
// not a google redirect
// normal goto
$this->urls_to_resolve[] = [
"replace" => $url,
"slug" => $slug[1],
"type" => "goto"
];
}
elseif(
preg_match(
'/https?:\/\/share\.google\/([^"]+)+/',
$url,
$link_regex
) === 1
){
// share.google
$this->urls_to_resolve[] = [
"replace" => $replace,
"slug" => $link_regex[1],
"type" => "share"
];
}
// rest is ignored, since its not a redirect
// always return url
return $url;
}
private function urlfuck_goto($url, $slug){
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -658,7 +693,7 @@ class google{
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $proxy);
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
@@ -669,7 +704,92 @@ class google{
return $url;
}
private function resolve_urls(&$scraped_data){
private function urlfuck_shareservice($url, $slug){
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
"replacement" => $url,
"handle" => $curlproc
];
return $url;
}
private function urlfuck_share($url, $slug){
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
"replacement" => $url,
"handle" => $curlproc
];
return $url;
}
private function resolve_gotos(&$scraped_data){
foreach($this->urls_to_resolve as $item){
$this->urlfuck_goto($item["replace"], $item["slug"]);
}
do{
do{
@@ -706,20 +826,163 @@ class google{
if(count($a) === 0){
throw new Exception("Failed to find link");
}
// failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
}
if(
(
$fallback === false &&
preg_match(
'/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){
$keys[] = $this->handles[$i]["replacement"];
$replacements[] = $link;
}
}
$this->recursive_replace($keys, $replacements, $scraped_data);
curl_multi_close($this->proc);
}
private function resolve_share(&$scraped_data){
foreach($this->urls_to_resolve as $item){
$this->urlfuck_shareservice($item["replace"], $item["slug"]);
}
do{
do{
$status = curl_multi_exec($this->proc, $active);
}while($status === CURLM_CALL_MULTI_PERFORM);
if($active){
curl_multi_select($this->proc);
}
}while($active && $status === CURLM_OK);
//
// if we reach this, we're done downloading garbage
//
// get the share links
$this->urls_to_resolve = [];
for($i=0; $i<count($this->handles); $i++){
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
if(
preg_match(
'/https?:\/\/share\.google\/(?:[^"]+)/',
$html,
$link_regex
) !== 1
){
// failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link = $link_regex[0];
}
if(
(
$fallback === false &&
preg_match(
'/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){
$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
}
}
// resolve shareable urls
$this->init_multi_curl();
foreach($this->urls_to_resolve as $item){
$this->urlfuck_share($item["replace"], $item["slug"]);
}
do{
do{
$status = curl_multi_exec($this->proc, $active);
}while($status === CURLM_CALL_MULTI_PERFORM);
if($active){
curl_multi_select($this->proc);
}
}while($active && $status === CURLM_OK);
// parse downloaded garbage
$keys = [];
$replacements = [];
for($i=0; $i<count($this->handles); $i++){
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
$this->fuckhtml->load($html);
$a =
$this->fuckhtml
->getElementsByTagName(
"a"
);
if(count($a) === 0){
// failed to find link, will serve a resolver target as fallback
$fallback = true;
$slug = explode("url=", $this->handles[$i]["replacement"]);
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
}else{
$fallback = false;
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
}
if(
(
$fallback === false &&
preg_match(
'/^(https?:|)\/\//',
$link
)
) ||
$fallback === true
){
$keys[] = $this->handles[$i]["replacement"];
@@ -785,22 +1048,6 @@ class google{
throw new Exception("Search term is empty!");
}
/*
$session = $this->backend->session_get();
if(config::FPLAY_ENABLE_API === false){
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
}
if($session === false){
throw new Exception("Session database is empty");
}
$proxy = &$session["proxy"];
$headers = &$session["headers"];
*/
$proxy = $this->backend->get_ip();
@@ -817,7 +1064,7 @@ class google{
// country
if($get["country"] != "any"){
$params["gl"] = $country;
$params["gl"] = $get["country"];
}
// nsfw
@@ -826,15 +1073,15 @@ class google{
// language
if($get["lang"] != "any"){
$params["lr"] = "lang_" . $lang;
$params["lr"] = "lang_" . $get["lang"];
}
// generate tbs
$tbs = [];
// get date
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
if(
$get["older"] !== null ||
@@ -1466,7 +1713,7 @@ class google{
];
}
$this->resolve_urls($out);
$this->resolve_share($out);
return $out;
}
@@ -1478,15 +1725,15 @@ class google{
// generate parameters
if($get["npt"]){
[$params, $proxy] =
[$params_json, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$params = json_decode($params, true);
$container = $params["c"];
unset($params["c"]);
$params_json = json_decode($params_json, true);
$container = &$params_json["c"];
$params = &$params_json["p"];
}else{
$search = $get["s"];
@@ -1505,6 +1752,7 @@ class google{
$type = $get["type"];
$format = $get["format"];
$rights = $get["rights"];
$container = null;
$params = [
"q" => $search,
@@ -1593,17 +1841,18 @@ class google{
}
}
//$html = file_get_contents("scraper/google.html");
$data = $this->get(
$proxy,
"https://www.google.com/search",
$params
$params,
$container
);
$html = &$data["data"];
$container = &$data["container"];
//$html = file_get_contents("scraper/google.html");
$this->fuckhtml->load($html);
// parse all <style> tags
@@ -1701,13 +1950,16 @@ class google{
$out["npt"] =
$this->backend
->store(
json_encode($params),
json_encode([
"p" => $params,
"c" => $container
]),
"image",
$proxy
);
}
$this->resolve_urls($out);
$this->resolve_gotos($out);
return $out;
}
@@ -2115,7 +2367,7 @@ class google{
];
}
$this->resolve_urls($out);
$this->resolve_share($out);
return $out;
}
@@ -2474,14 +2726,14 @@ class google{
"date" => $date,
"thumb" => $thumb,
"url" =>
$this->unshiturl(
$this->redirect_add_url(
$a["attributes"]
["href"]
)
];
}
$this->resolve_urls($out);
$this->resolve_share($out);
return $out;
}
+2
View File
@@ -567,6 +567,8 @@ class google_api{
foreach($json["items"] as $result){
$date = null;
//
// probe for thumbnail
//
+9
View File
@@ -725,6 +725,14 @@ class google_cse{
// calculate rounded ratio
if($found_size){
if(
$width === 0 ||
$height === 0
){
$thumb["ratio"] = "1:1";
}else{
$aspect_ratio = $width / $height;
if($aspect_ratio >= 1.5){
@@ -740,6 +748,7 @@ class google_cse{
}
}
}
}
$out["web"][] = [
"title" => rtrim($result["titleNoFormatting"], " ."),
+73 -22
View File
@@ -261,9 +261,10 @@ class naver{
$filters = [
"nso" => "",
"page" => 2, // page 2 is repeat of page 1
"query" => $search,
"sm" => "tab_pge",
"where" => "web",
"ssc" => "tab.ur.all",
"start" => 1 // increment by number of results each time (16??)
];
@@ -304,13 +305,13 @@ class naver{
$this->fuckhtml->load($html);
$results =
$jsons =
preg_split(
'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
$html
);
if(count($results) !== 2){
if(count($jsons) <= 1){
// this is thrown when no results are found
@@ -342,32 +343,29 @@ class naver{
//throw new Exception("Failed to grep results entrypoint");
}
foreach($jsons as $json){
$json =
json_decode(
$this->fuckhtml
->extract_json(
$results[1]
$json
),
true
);
if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){
if($json === null){
throw new Exception("Failed to access nested children");
}
foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){
if(
!isset($result["templateId"]) ||
$result["templateId"] != "webItem"
){
// should not happen
continue;
}
$result = $result["props"];
if(!isset($json["data"]["state"]["items"])){
//throw new Exception("Failed to access nested children");
continue;
}
foreach($json["data"]["state"]["items"] as $result){
// get sublinks
$sublinks = [];
@@ -429,12 +427,19 @@ class naver{
// get date
$time = null;
if(isset($result["bodyPrefixes"][0]["text"])){
if(isset($result["profile"]["createdDate"])){
$time = $result["profile"]["createdDate"];
}elseif(isset($result["sourceProfile"]["createdDate"])){
$time = $result["sourceProfile"]["createdDate"];
}
if($time !== null){
$date =
strtotime(
substr(
$result["bodyPrefixes"][0]["text"],
$time,
-1
)
);
@@ -442,13 +447,58 @@ class naver{
if($date !== false){
$time = $date;
}else{
$time = null;
}
}
// get title
if(isset($result["title"])){
$title = $result["title"];
}elseif(isset($result["questionTitle"])){
$title = $result["questionTitle"];
}else{
continue;
}
// get url
if(isset($result["titleHref"])){
$href = $result["titleHref"];
}elseif(isset($result["href"])){
$href = $result["href"];
}elseif(isset($result["answerHref"])){
$href = $result["answerHref"];
}else{
continue;
}
// get description
if(isset($result["bodyText"])){
$description = $result["bodyText"];
}elseif(isset($result["content"])){
$description = $result["content"];
}elseif(isset($result["questionContent"])){
$description = $result["questionContent"];
}else{
$description = null;
}
$out["web"][] = [
"title" => $this->decode_html($result["title"]),
"description" => $this->decode_html($result["bodyText"]),
"url" => $result["href"],
"title" => $this->decode_html($title),
"description" => $this->decode_html($description),
"url" => $href,
"date" => $time,
"type" => "web",
"thumb" => $thumb,
@@ -456,6 +506,7 @@ class naver{
"table" => $table
];
}
}
// get next page
$npt =
+158 -16
View File
@@ -545,10 +545,12 @@ class startpage{
switch($category["display_type"]){
case "web-google":
case "web-bing":
foreach($category["results"] as $result){
$sublinks = [];
if(isset($result["siteLinks"])){
foreach($result["siteLinks"] as $sublink){
$sublinks[] = [
@@ -557,6 +559,7 @@ class startpage{
"url" => $sublink["clickUrl"]
];
}
}
$description =
explode(
@@ -596,6 +599,7 @@ class startpage{
$out["web"][] = [
"title" =>
$this->remove_penguins(
$this->titledots(
html_entity_decode(
$this->fuckhtml
@@ -603,8 +607,9 @@ class startpage{
$result["title"]
)
)
)
),
"description" => $description,
"description" => $this->remove_penguins($description),
"url" => $result["clickUrl"],
"date" => $date,
"type" => "web",
@@ -619,8 +624,43 @@ class startpage{
break;
case "images-qi-top":
case "images-bing-top":
foreach($category["results"] as $result){
if(isset($result["mdThumbnailWidth"])){
$thumb_width = $result["mdThumbnailWidth"];
}elseif(isset($result["thumbnailWidth"])){
$thumb_width = $result["thumbnailWidth"];
}else{
$thumb_width = null;
}
if(isset($result["mdThumbnailHeight"])){
$thumb_height = $result["mdThumbnailHeight"];
}elseif(isset($result["thumbnailHeight"])){
$thumb_height = $result["thumbnailHeight"];
}else{
$thumb_height = null;
}
if(isset($result["mdThumbnailUrl"])){
$thumb_source = $result["mdThumbnailUrl"];
}elseif(isset($result["thumbnailUrl"])){
$thumb_source = $result["thumbnailUrl"];
}else{
// should not happen
continue;
}
$out["image"][] = [
"title" =>
$this->titledots(
@@ -638,19 +678,17 @@ class startpage{
"height" => (int)$result["height"]
],
[
"url" => $this->unshitimage($result["mdThumbnailUrl"]),
"width" => (int)$result["mdThumbnailWidth"],
"height" => (int)$result["mdThumbnailHeight"]
"url" => $this->unshitimage($thumb_source),
"width" => (int)$thumb_width,
"height" => (int)$thumb_height
]
],
"url" =>
$result["altClickUrl"]
"url" => $result["altClickUrl"]
];
}
break;
case "spellsuggest-google":
if(isset($category["results"][0]["query"])){
$out["spelling"] =
@@ -662,6 +700,60 @@ class startpage{
}
break;
case "spelladjust-bing":
if(isset($category["results"][0]["alterationDisplayQuery"])){
$out["spelling"] =
[
"type" => "including",
"using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]),
"correction" => $category["results"][0]["originalQuery"]
];
}
break;
case "aylf-bing-top":
if(isset($category["results"][0]["query"])){
foreach($category["results"] as $ddm){
$out["related"][] = $ddm["query"];
}
}
break;
case "video-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $video){
if(isset($video["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $this->unshitimage($video["thumbnailUrl"])
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["video"][] = [
"title" => $video["title"],
"description" => $this->remove_penguins($video["description"]),
"date" => isset($video["date"]) ? strtotime($video["date"]) : null,
"duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null,
"views" => $video["viewCount"],
"thumb" => $thumb,
"url" => $video["clickUrl"]
];
}
}
break;
case "dictionary-qi":
foreach($category["results"] as $result){
@@ -732,6 +824,36 @@ class startpage{
$out["answer"][] = $answer;
}
break;
case "news-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $news){
if(isset($news["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $news["thumbnailUrl"]
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["news"][] = [
"title" => $this->remove_penguins($news["title"]),
"description" => $news["description"],
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb,
"url" => $news["clickUrl"],
];
}
}
break;
}
}
@@ -1246,14 +1368,10 @@ class startpage{
// get npt
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
//print_r($json);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
if($category["display_type"] != "news-bing"){
// unsupported category
continue;
}
foreach($category["results"] as $news){
if(
@@ -1276,10 +1394,10 @@ class startpage{
$out["news"][] = [
"title" => $this->titledots($this->remove_penguins($news["title"])),
"author" => $news["source"],
"description" => $this->titledots($this->remove_penguins($news["description"])),
"date" => (int)substr((string)$news["date"], 0, -3),
"description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null,
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb,
"url" => $news["clickUrl"]
"url" => $this->remove_tracking_link($news["clickUrl"])
];
}
}
@@ -1327,6 +1445,30 @@ class startpage{
return null;
}
private function remove_tracking_link($url){
// https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog
$query = parse_url($url);
if(
!isset($query["host"]) ||
$query["host"] != "api.airfind.com" ||
!isset($query["query"])
){
return $url;
}
parse_str($query["query"], $res);
if(isset($res["redirectURL"])){
return $res["redirectURL"];
}
return $url;
}
private function unshitimage($url){
$query = parse_url($url, PHP_URL_QUERY);
+19 -58
View File
@@ -205,13 +205,6 @@ class yep{
"zh-tw" => "Chinese (Traditional)",
"zu" => "Zulu"
]
],
"nsfw" => [
"display" => "NSFW",
"option" => [
"yes" => "Yes",
"no" => "No"
]
]
];
}
@@ -317,20 +310,23 @@ class yep{
// parse filters
$filters = [
"limit" => 100, // wwwwwwwwwwwwwww
"query" => $search,
"q" => $search,
"limit" => 10 // no fun allowed
];
if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
if($get["lang"] != "any"){
$filters["language"] = $get["lang"];
$filters["lang"] = $get["lang"];
}
try{
// https://api.yep.com/search?limit=20&query=asmr
// https://yep.com/web/v1/search?q=asmr&limit=10
$json =
$this->get(
$this->backend->get_ip(),
"https://api.yep.com/search",
"https://yep.com/web/v1/search",
$filters
);
@@ -343,59 +339,27 @@ class yep{
$json = json_decode($json, true);
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json[1]["correction"])){
if(isset($json["code"])){
$out["spelling"] = [
"type" => "not_many",
"using" => $search,
"correction" => $json[1]["correction"][1]
];
if(isset($json["reason"])){
throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
}
if(isset($json[1]["results"])){
foreach($json[1]["results"] as $item){
switch(strtolower($item["type"])){
case "organic":
$sublinks = [];
if(isset($item["sitelinks"]["full"])){
foreach($item["sitelinks"]["full"] as $link){
$sublinks[] = [
"title" => $link["title"],
"date" => null,
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$link["snippet"]
)
)
),
"url" => $link["url"]
];
}
throw new Exception("Yep returned an unknown error");
}
if(isset($json["results"])){
foreach($json["results"] as $item){
$out["web"][] = [
"title" => $item["title"],
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$item["snippet"]
)
)
),
"description" => str_replace("\n", " ", $item["description"]),
"url" => $item["url"],
"date" => null,
"type" => "web",
@@ -403,11 +367,9 @@ class yep{
"url" => null,
"ratio" => null
],
"sublink" => $sublinks,
"sublink" => [],
"table" => []
];
break;
}
}
}
@@ -445,7 +407,6 @@ class yep{
"limit" => 99 // for some reason they dont show anything sometimes at 100 results
];
if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
// add api key
+10 -1
View File
@@ -678,6 +678,15 @@ class yt{
$video["longBylineText"]
),
"url" =>
isset(
$video["ownerText"]
["runs"]
[0]
["navigationEndpoint"]
["commandMetadata"]
["webCommandMetadata"]
["url"]
) ?
"https://www.youtube.com" .
$video["ownerText"]
["runs"]
@@ -685,7 +694,7 @@ class yt{
["navigationEndpoint"]
["commandMetadata"]
["webCommandMetadata"]
["url"],
["url"] : null,
"avatar" =>
$video["channelThumbnailSupportedRenderers"]
["channelThumbnailWithLinkRenderer"]
+28 -4
View File
@@ -67,6 +67,10 @@ $settings = [
"value" => "brave",
"text" => "Brave"
],
[
"value" => "bing",
"text" => "Bing"
],
[
"value" => "ddg",
"text" => "DuckDuckGo"
@@ -79,17 +83,25 @@ $settings = [
"value" => "google",
"text" => "Google"
],
[
"value" => "yahoo",
"text" => "Yahoo"
],
[
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "kagi",
"text" => "Kagi"
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "yep",
"text" => "Yep"
"value" => "ecosia",
"text" => "Ecosia"
],
[
"value" => "kagi",
"text" => "Kagi"
],
[
"value" => "marginalia",
@@ -102,6 +114,18 @@ $settings = [
[
"value" => "sc",
"text" => "SoundCloud"
],
[
"value" => "coccoc",
"text" => "Coccoc"
],
[
"value" => "naver",
"text" => "Naver"
],
[
"value" => "purili",
"text" => "Purili"
]
]
],
+2 -2
View File
@@ -349,8 +349,8 @@ if(count($results["answer"]) !== 0){
$right["answer"] .=
'<tr>' .
'<td>' . $info . '</td>' .
'<td>' . $value . '</td>' .
'<td>' . htmlspecialchars($info) . '</td>' .
'<td>' . htmlspecialchars($value) . '</td>' .
'</tr>';
}