Compare commits
23
Commits
33ce0a062b
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c7c4ea072 | ||
|
|
761a1e91cd | ||
|
|
c5311bc399 | ||
|
|
4536cef88f | ||
|
|
0c3844d578 | ||
|
|
30cd3c8492 | ||
|
|
aa818ac925 | ||
|
|
5967462b6e | ||
|
|
b014a3199d | ||
|
|
24130effe6 | ||
|
|
5d8c9a5798 | ||
|
|
6bbb37d86e | ||
|
|
c0324d7d61 | ||
|
|
2218fd6f04 | ||
|
|
be91ec1806 | ||
|
|
57a72e5b5e | ||
|
|
9f0437d949 | ||
|
|
d91834f86f | ||
|
|
42d0252e2f | ||
|
|
a27a8ed01a | ||
|
|
9db95c79d2 | ||
|
|
25d4f991c2 | ||
|
|
e1d0ab121f |
No files matched your search
+1
-1
@@ -31,7 +31,7 @@ echo json_encode(
|
||||
"real_requests" => $real_requests,
|
||||
"bot_requests" => $bot_requests,
|
||||
"api_enabled" => config::API_ENABLED,
|
||||
"alt_addresses" => config::ALT_ADDRESSES,
|
||||
"alt_addresses" => config::ADDRESSES,
|
||||
"version" => config::VERSION
|
||||
],
|
||||
"instances" => config::INSTANCES
|
||||
|
||||
+116
-24
@@ -7,37 +7,51 @@ class autocomplete{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
header("Content-Type: application/json");
|
||||
header("Content-Type: application/x-suggestions+json");
|
||||
|
||||
$this->scrapers = [
|
||||
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
|
||||
"bing" => "https://www.bing.com/osjson.aspx?query={searchTerms}",
|
||||
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
|
||||
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
|
||||
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
|
||||
"ecosia" => "https://ac.ecosia.org/autocomplete?q={searchTerms}&type=list",
|
||||
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3",
|
||||
"google" => "https://www.google.com/complete/search?client=chrome&q={searchTerms}",
|
||||
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
|
||||
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
|
||||
"marginalia" => "https://marginalia-search.com/suggest/?partial={searchTerms}",
|
||||
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
|
||||
"sc" => "",
|
||||
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
|
||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
|
||||
"kagi" => "https://kagisuggest.com/api/autosuggest?q={searchTerms}",
|
||||
"startpage" => "https://www.startpage.com/osuggestions?q={searchTerms}",
|
||||
"yahoo" => "https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=sd1&command={searchTerms}",
|
||||
"purili" => "https://puri.li/api/suggest?q={searchTerms}",
|
||||
"coccoc" => "https://coccoc.com/composer/autocomplete?q={searchTerms}",
|
||||
"naver" => "https://ac.search.naver.com/nx/ac?q={searchTerms}&st=100"
|
||||
];
|
||||
|
||||
// add duplicates
|
||||
$this->scrapers["google_api"] = &$this->scrapers["google"];
|
||||
$this->scrapers["google_cse"] = &$this->scrapers["google"];
|
||||
$this->scrapers["yahoo_japan"] = &$this->scrapers["yahoo"];
|
||||
|
||||
/*
|
||||
Sanitize input
|
||||
*/
|
||||
if(!isset($_GET["s"])){
|
||||
|
||||
$this->do404("Missing search(s) parameter");
|
||||
// Missing search(s) parameter
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
if(is_string($_GET["s"]) === false){
|
||||
|
||||
$this->do404("Invalid search(s) parameter");
|
||||
// Invalid search(s) parameter
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
if(strlen($_GET["s"]) > 500){
|
||||
|
||||
$this->do404("Search(s) exceeds the 500 char length");
|
||||
// Search(s) exceeds the 500 char length
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -72,9 +86,8 @@ class autocomplete{
|
||||
// return results
|
||||
switch($scraper){
|
||||
|
||||
case "google":
|
||||
case "yt":
|
||||
// handle google cause they want to be a special snowflake :(
|
||||
// handle youtube cause it wants to be a special snowflake :(
|
||||
$js = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||
|
||||
preg_match(
|
||||
@@ -141,12 +154,12 @@ class autocomplete{
|
||||
break;
|
||||
|
||||
case "marginalia":
|
||||
case "purili":
|
||||
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||
|
||||
$json = json_decode($json, true);
|
||||
if($json === null){
|
||||
|
||||
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
@@ -159,10 +172,98 @@ class autocomplete{
|
||||
);
|
||||
break;
|
||||
|
||||
case "yahoo":
|
||||
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||
|
||||
$json = json_decode($json, true);
|
||||
if(
|
||||
$json === null ||
|
||||
!isset($json["r"][0]["k"])
|
||||
){
|
||||
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
$payload = [];
|
||||
|
||||
foreach($json["r"] as $term){
|
||||
|
||||
if(isset($term["k"])){
|
||||
|
||||
$payload[] = $term["k"];
|
||||
}
|
||||
}
|
||||
|
||||
echo json_encode(
|
||||
[
|
||||
$_GET["s"],
|
||||
$payload
|
||||
],
|
||||
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||
);
|
||||
break;
|
||||
|
||||
case "naver":
|
||||
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||
|
||||
$json = json_decode($json, true);
|
||||
if(
|
||||
$json === null ||
|
||||
!isset($json["items"][0][0])
|
||||
){
|
||||
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
$payload = [];
|
||||
|
||||
foreach($json["items"][0] as $term){
|
||||
|
||||
$payload[] = $term[0];
|
||||
}
|
||||
|
||||
echo json_encode(
|
||||
[
|
||||
$_GET["s"],
|
||||
$payload
|
||||
],
|
||||
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||
);
|
||||
break;
|
||||
|
||||
case "coccoc":
|
||||
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||
|
||||
$json = json_decode($json, true);
|
||||
if(
|
||||
$json === null ||
|
||||
!isset($json["suggestions"])
|
||||
){
|
||||
|
||||
$this->doempty();
|
||||
}
|
||||
|
||||
echo json_encode(
|
||||
[
|
||||
$_GET["s"],
|
||||
$json["suggestions"]
|
||||
],
|
||||
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
// if it respects the openSearch protocol
|
||||
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
|
||||
|
||||
if(
|
||||
!isset($json[1]) ||
|
||||
!is_array($json[1])
|
||||
){
|
||||
|
||||
$json[1] = [];
|
||||
}
|
||||
|
||||
echo json_encode(
|
||||
[
|
||||
$_GET["s"],
|
||||
@@ -185,10 +286,10 @@ class autocomplete{
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: application/json, text/javascript, */*; q=0.01",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
@@ -214,19 +315,10 @@ class autocomplete{
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
do404("Curl error: " . $error->getMessage());
|
||||
doempty();
|
||||
}
|
||||
}
|
||||
|
||||
private function do404($error){
|
||||
|
||||
echo json_encode(
|
||||
["error" => $error],
|
||||
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||
);
|
||||
die();
|
||||
}
|
||||
|
||||
private function doempty(){
|
||||
|
||||
echo json_encode(
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
|
||||
+3
-2
@@ -154,7 +154,7 @@ class config{
|
||||
|
||||
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
|
||||
// Changing this might break things.
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0";
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:160.0) Gecko/20100101 Firefox/160.0";
|
||||
|
||||
// User agent to use with 4get-friendly APIs
|
||||
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
|
||||
@@ -225,6 +225,7 @@ class config{
|
||||
// use the API (Eg: "public"). API has less filters.
|
||||
const MARGINALIA_API_KEY = null;
|
||||
|
||||
// Yep
|
||||
// Scrapers to use the API on
|
||||
const YEP_USE_API = false;
|
||||
const BRAVE_USE_API = false;
|
||||
}
|
||||
+155
-164
@@ -1,31 +1,16 @@
|
||||
# Sample Apache2 configuration
|
||||
This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
|
||||
|
||||
Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
|
||||
|
||||
Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
|
||||
|
||||
```xml
|
||||
<VirtualHost *:443>
|
||||
ServerName www.4get.ca
|
||||
|
||||
SSLEngine On
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
|
||||
RedirectMatch 301 ^(.*)$ https://4get.ca$1
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
<VirtualHost *:8080>
|
||||
ServerName 4get.ca
|
||||
ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
|
||||
|
||||
ServerAdmin will@lolcat.ca
|
||||
DocumentRoot /var/www/4get
|
||||
|
||||
SSLEngine On
|
||||
SSLOptions +StdEnvVars
|
||||
|
||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
|
||||
# compress
|
||||
AddOutputFilterByType DEFLATE application/json
|
||||
AddOutputFilterByType DEFLATE application/javascript
|
||||
AddOutputFilterByType DEFLATE application/x-javascript
|
||||
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
|
||||
AddOutputFilterByType DEFLATE text/plain
|
||||
AddOutputFilterByType DEFLATE text/css
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
DocumentRoot /var/www/4get
|
||||
ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
CustomLog /dev/null combined
|
||||
#CustomLog /var/log/apache2/access.log combined
|
||||
|
||||
<FilesMatch \.php$>
|
||||
SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
|
||||
</FilesMatch>
|
||||
|
||||
<Directory /var/www/4get>
|
||||
Options -MultiViews
|
||||
AllowOverride All
|
||||
Require all granted
|
||||
|
||||
# prevent 404s from hitting php-fpm
|
||||
RewriteEngine On
|
||||
RewriteCond %{REQUEST_URI} \.php$ [NC]
|
||||
RewriteCond %{REQUEST_FILENAME} !-f
|
||||
RewriteRule ^ - [R=404,L]
|
||||
|
||||
# prevent /index hits
|
||||
RewriteRule ^index$ - [R=404,L]
|
||||
|
||||
# 404 direct .php file access
|
||||
RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
|
||||
RewriteRule ^ - [R=404,L]
|
||||
|
||||
# internally redirect to non-php extension
|
||||
RewriteCond %{REQUEST_FILENAME} !-d
|
||||
RewriteCond %{REQUEST_FILENAME} !-f
|
||||
RewriteCond %{REQUEST_FILENAME}.php -f
|
||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
||||
</Directory>
|
||||
|
||||
# deny access to private resources
|
||||
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
|
||||
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
|
||||
Require all denied
|
||||
</DirectoryMatch>
|
||||
|
||||
# remove directory listing from /icons
|
||||
|
||||
<Directory /var/www/4get/icons>
|
||||
Options -Indexes
|
||||
</Directory>
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName www.lolcat.ca
|
||||
|
||||
SSLEngine On
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
|
||||
RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName lolcat.ca
|
||||
|
||||
ServerAdmin will@lolcat.ca
|
||||
DocumentRoot /var/www/lolcat
|
||||
|
||||
SSLEngine On
|
||||
SSLOptions +StdEnvVars
|
||||
|
||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
|
||||
AddOutputFilterByType DEFLATE application/json
|
||||
AddOutputFilterByType DEFLATE application/javascript
|
||||
AddOutputFilterByType DEFLATE application/x-javascript
|
||||
AddOutputFilterByType DEFLATE text/html
|
||||
AddOutputFilterByType DEFLATE text/plain
|
||||
AddOutputFilterByType DEFLATE text/css
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
|
||||
<Directory /var/www/lolcat>
|
||||
Options -MultiViews
|
||||
AllowOverride All
|
||||
Require all granted
|
||||
|
||||
RewriteEngine On
|
||||
RewriteCond %{REQUEST_FILENAME} !-d
|
||||
RewriteCond %{REQUEST_FILENAME} !-f
|
||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
||||
</Directory>
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName www.nyym.co
|
||||
|
||||
SSLEngine On
|
||||
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
|
||||
|
||||
RedirectMatch 301 ^(.*)$ https://nyym.co$1
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName nyym.co
|
||||
|
||||
ServerAdmin will@lolcat.ca
|
||||
DocumentRoot /var/www/nyym
|
||||
|
||||
SSLEngine On
|
||||
SSLOptions +StdEnvVars
|
||||
|
||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
|
||||
AddOutputFilterByType DEFLATE application/json
|
||||
AddOutputFilterByType DEFLATE application/javascript
|
||||
AddOutputFilterByType DEFLATE application/x-javascript
|
||||
AddOutputFilterByType DEFLATE text/html
|
||||
AddOutputFilterByType DEFLATE text/plain
|
||||
AddOutputFilterByType DEFLATE text/css
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
|
||||
|
||||
<Directory /var/www/nyym>
|
||||
Options -MultiViews
|
||||
AllowOverride All
|
||||
Require all granted
|
||||
|
||||
RewriteEngine On
|
||||
RewriteCond %{REQUEST_FILENAME} !-d
|
||||
RewriteCond %{REQUEST_FILENAME} !-f
|
||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
||||
</Directory>
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName git.lolcat.ca
|
||||
|
||||
SSLEngine On
|
||||
SSLOptions +StdEnvVars
|
||||
|
||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
|
||||
AddOutputFilterByType DEFLATE application/json
|
||||
AddOutputFilterByType DEFLATE application/javascript
|
||||
AddOutputFilterByType DEFLATE application/x-javascript
|
||||
AddOutputFilterByType DEFLATE text/html
|
||||
AddOutputFilterByType DEFLATE text/plain
|
||||
AddOutputFilterByType DEFLATE text/css
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
|
||||
ProxyPreserveHost On
|
||||
ProxyRequests off
|
||||
AllowEncodedSlashes NoDecode
|
||||
ProxyPass / http://localhost:3000/ nocanon
|
||||
</VirtualHost>
|
||||
|
||||
<VirtualHost *:443>
|
||||
ServerName live.lolcat.ca
|
||||
|
||||
ServerAdmin will@lolcat.ca
|
||||
DocumentRoot /var/www/live
|
||||
|
||||
SSLEngine On
|
||||
SSLOptions +StdEnvVars
|
||||
|
||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||
|
||||
AddOutputFilterByType DEFLATE application/json
|
||||
AddOutputFilterByType DEFLATE application/javascript
|
||||
AddOutputFilterByType DEFLATE application/x-javascript
|
||||
AddOutputFilterByType DEFLATE text/html
|
||||
AddOutputFilterByType DEFLATE text/plain
|
||||
AddOutputFilterByType DEFLATE text/css
|
||||
|
||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
||||
</VirtualHost>
|
||||
```
|
||||
|
||||
# Haproxy configuration
|
||||
|
||||
```cfg
|
||||
global
|
||||
#log /dev/log local0
|
||||
log /dev/log local0 warning
|
||||
log /dev/log local1 notice
|
||||
chroot /var/lib/haproxy
|
||||
stats socket /run/haproxy/admin.sock mode 660 level admin
|
||||
stats timeout 30s
|
||||
user haproxy
|
||||
group haproxy
|
||||
daemon
|
||||
|
||||
# Default SSL material locations
|
||||
ca-base /etc/ssl/certs
|
||||
crt-base /etc/ssl/private
|
||||
|
||||
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
|
||||
#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
|
||||
ssl-default-bind-ciphers HIGH:!aNULL
|
||||
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
|
||||
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||
|
||||
# ja4 settings
|
||||
tune.lua.bool-sample-conversion normal
|
||||
tune.ssl.capture-buffer-size 190
|
||||
lua-load /etc/haproxy/lua/ja4.lua
|
||||
defaults
|
||||
log global
|
||||
mode http
|
||||
#option httplog
|
||||
option dontlognull
|
||||
timeout connect 5000
|
||||
timeout client 50000
|
||||
timeout server 50000
|
||||
errorfile 400 /etc/haproxy/errors/400.http
|
||||
errorfile 403 /etc/haproxy/errors/403.http
|
||||
errorfile 408 /etc/haproxy/errors/408.http
|
||||
errorfile 500 /etc/haproxy/errors/500.http
|
||||
errorfile 502 /etc/haproxy/errors/502.http
|
||||
errorfile 503 /etc/haproxy/errors/503.http
|
||||
errorfile 504 /etc/haproxy/errors/504.http
|
||||
|
||||
frontend http_80_in
|
||||
# bind tcp ip4/6
|
||||
bind *:80
|
||||
bind [::]:80
|
||||
|
||||
mode http
|
||||
|
||||
# handle ip.lolcat.ca+onion addy on plain HTTP
|
||||
acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
|
||||
|
||||
# redirect to https
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
|
||||
http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
|
||||
|
||||
# redirect www.lolcat.ca to lolcat.ca
|
||||
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
|
||||
|
||||
# everything else redirects to 4get.ca
|
||||
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
|
||||
|
||||
# ... unless its an ip host, or the onion domain
|
||||
use_backend apache if is_ip_host
|
||||
|
||||
frontend https_443_in
|
||||
# bind tcp ip4/6
|
||||
bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
|
||||
bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
|
||||
|
||||
# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
|
||||
# bind udp ip4/6
|
||||
#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
|
||||
#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
|
||||
|
||||
# tell client we support http3 now
|
||||
#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
|
||||
|
||||
mode http
|
||||
|
||||
# redirect all unknown domains to 4get.ca
|
||||
acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
|
||||
|
||||
# add onion domain to 4get.ca
|
||||
acl is_4get hdr(host) -i 4get.ca
|
||||
http-request set-var(txn.is_4get) bool(true) if is_4get
|
||||
http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
|
||||
|
||||
# git.lolcat.ca -> gitea service
|
||||
acl acl_gitea hdr(host) -i git.lolcat.ca
|
||||
|
||||
# redirects
|
||||
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
|
||||
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
|
||||
|
||||
# add security headers
|
||||
http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
|
||||
http-response set-header X-Content-Type-Options "nosniff"
|
||||
http-response set-header X-Frame-Options "SAMEORIGIN"
|
||||
|
||||
# route
|
||||
use_backend gitea if acl_gitea
|
||||
default_backend apache
|
||||
|
||||
backend apache
|
||||
# pass ja4
|
||||
http-request lua.fingerprint_ja4 if { ssl_fc }
|
||||
http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
|
||||
|
||||
# delete server header
|
||||
http-response del-header Server
|
||||
|
||||
# pass remote IP
|
||||
option forwardfor
|
||||
server apache_server 127.0.0.1:8080 check
|
||||
|
||||
backend gitea
|
||||
server gitea_server 127.0.0.1:3000 check
|
||||
```
|
||||
+2
-2
@@ -12,9 +12,9 @@ class backend{
|
||||
/*
|
||||
Proxy stuff
|
||||
*/
|
||||
public function get_ip($proxy_index_raw = null){
|
||||
public function get_ip($proxy_index_raw = null, $use_alt_list = false){
|
||||
|
||||
$pool = constant("config::PROXY_" . strtoupper($this->scraper));
|
||||
$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
|
||||
if($pool === false){
|
||||
|
||||
// we don't want a proxy, fuck off!
|
||||
|
||||
+1
-2
@@ -1119,7 +1119,6 @@ class frontend{
|
||||
$filters["scraper"] = [
|
||||
"display" => "Scraper",
|
||||
"option" => [
|
||||
//"fget" => "fget",
|
||||
"ddg" => "DuckDuckGo",
|
||||
//"yahoo" => "Yahoo!",
|
||||
"brave" => "Brave",
|
||||
@@ -1207,7 +1206,7 @@ class frontend{
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
//"mojeek" => "Mojeek",
|
||||
"baidu" => "Baidu"
|
||||
//"baidu" => "Baidu"
|
||||
]
|
||||
];
|
||||
break;
|
||||
|
||||
+26
-13
@@ -3,32 +3,45 @@
|
||||
header("Content-Type: application/xml");
|
||||
include "data/config.php";
|
||||
|
||||
$domain =
|
||||
htmlspecialchars(
|
||||
(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on" ? "https" : "http") .
|
||||
'://' . $_SERVER["HTTP_HOST"]
|
||||
);
|
||||
$host = strtolower(explode(":", $_SERVER["HTTP_HOST"], 2)[0]);
|
||||
|
||||
// detect if 4get should serve the HTTPS endpoint
|
||||
if(
|
||||
preg_match(
|
||||
'/\.onion$/',
|
||||
$domain
|
||||
'/\.(onion|i2p)$/',
|
||||
strtolower($host),
|
||||
$match
|
||||
)
|
||||
){
|
||||
|
||||
$onion = true;
|
||||
$ssl = false;
|
||||
$src = $match[1];
|
||||
}elseif(
|
||||
is_string(
|
||||
filter_var(
|
||||
$host,
|
||||
FILTER_VALIDATE_IP
|
||||
)
|
||||
) ||
|
||||
$host == "localhost"
|
||||
){
|
||||
|
||||
$ssl = false;
|
||||
$src = "localhost";
|
||||
}else{
|
||||
|
||||
$onion = false;
|
||||
$ssl = true;
|
||||
}
|
||||
|
||||
$domain = ($ssl ? "https" : "http") . "://" . $_SERVER["HTTP_HOST"];
|
||||
|
||||
echo
|
||||
'<?xml version="1.0" encoding="UTF-8"?>' .
|
||||
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
|
||||
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($onion ? " (onion)" : "") . '</ShortName>' .
|
||||
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($ssl === false ? " ({$src})" : "") . '</ShortName>' .
|
||||
'<Description>' . htmlspecialchars(config::SERVER_SHORT_DESCRIPTION) . '</Description>' .
|
||||
'<InputEncoding>UTF-8</InputEncoding>' .
|
||||
'<Image width="16" height="16">' . $domain . '/favicon.ico</Image>' .
|
||||
'<Url type="text/html" method="GET" template="' . $domain . '/web?s={searchTerms}"/>';
|
||||
'<Image width="32" height="32" type="image/x-icon">' . htmlspecialchars($domain) . '/favicon.ico</Image>' .
|
||||
'<Url rel="results" type="text/html" method="GET" template="' . htmlspecialchars($domain) . '/web?s={searchTerms}"/>';
|
||||
|
||||
if(
|
||||
isset($_GET["ac"]) &&
|
||||
|
||||
+283
-1066
File diff suppressed because it is too large.
Load diff
+1048
-148
File diff suppressed because it is too large.
Load diff
+303
-51
@@ -10,8 +10,9 @@ class google{
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("google");
|
||||
|
||||
$this->proc = curl_multi_init();
|
||||
$this->handles = [];
|
||||
$this->urls_to_resolve = [];
|
||||
|
||||
$this->init_multi_curl();
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
@@ -617,23 +618,57 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
private function redirect_add_url($url){
|
||||
private function init_multi_curl(){
|
||||
|
||||
$this->proc = curl_multi_init();
|
||||
$this->handles = [];
|
||||
}
|
||||
|
||||
private function redirect_add_url($url, $replace = null){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/goto\?url=([^&]+)/',
|
||||
$url,
|
||||
$slug
|
||||
) === 0
|
||||
) === 1
|
||||
){
|
||||
|
||||
// not a google redirect
|
||||
return $url;
|
||||
// normal goto
|
||||
$this->urls_to_resolve[] = [
|
||||
"replace" => $url,
|
||||
"slug" => $slug[1],
|
||||
"type" => "goto"
|
||||
];
|
||||
}
|
||||
|
||||
elseif(
|
||||
preg_match(
|
||||
'/https?:\/\/share\.google\/([^"]+)+/',
|
||||
$url,
|
||||
$link_regex
|
||||
) === 1
|
||||
){
|
||||
|
||||
// share.google
|
||||
$this->urls_to_resolve[] = [
|
||||
"replace" => $replace,
|
||||
"slug" => $link_regex[1],
|
||||
"type" => "share"
|
||||
];
|
||||
}
|
||||
|
||||
// rest is ignored, since its not a redirect
|
||||
|
||||
// always return url
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function urlfuck_goto($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
@@ -658,7 +693,7 @@ class google{
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $proxy);
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
@@ -669,7 +704,92 @@ class google{
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function resolve_urls(&$scraped_data){
|
||||
private function urlfuck_shareservice($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
"replacement" => $url,
|
||||
"handle" => $curlproc
|
||||
];
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function urlfuck_share($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
"replacement" => $url,
|
||||
"handle" => $curlproc
|
||||
];
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function resolve_gotos(&$scraped_data){
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_goto($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
@@ -706,20 +826,163 @@ class google{
|
||||
|
||||
if(count($a) === 0){
|
||||
|
||||
throw new Exception("Failed to find link");
|
||||
// failed to find link, will serve a resolver target as fallback
|
||||
$fallback = true;
|
||||
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||
}else{
|
||||
|
||||
$fallback = false;
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
}
|
||||
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
if(
|
||||
(
|
||||
$fallback === false &&
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
) ||
|
||||
$fallback === true
|
||||
){
|
||||
|
||||
$keys[] = $this->handles[$i]["replacement"];
|
||||
$replacements[] = $link;
|
||||
}
|
||||
}
|
||||
|
||||
$this->recursive_replace($keys, $replacements, $scraped_data);
|
||||
|
||||
curl_multi_close($this->proc);
|
||||
}
|
||||
|
||||
private function resolve_share(&$scraped_data){
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_shareservice($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
$status = curl_multi_exec($this->proc, $active);
|
||||
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||
|
||||
if($active){
|
||||
curl_multi_select($this->proc);
|
||||
}
|
||||
|
||||
}while($active && $status === CURLM_OK);
|
||||
|
||||
//
|
||||
// if we reach this, we're done downloading garbage
|
||||
//
|
||||
|
||||
// get the share links
|
||||
$this->urls_to_resolve = [];
|
||||
|
||||
for($i=0; $i<count($this->handles); $i++){
|
||||
|
||||
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
'/https?:\/\/share\.google\/(?:[^"]+)/',
|
||||
$html,
|
||||
$link_regex
|
||||
) !== 1
|
||||
){
|
||||
|
||||
// failed to find link, will serve a resolver target as fallback
|
||||
$fallback = true;
|
||||
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||
}else{
|
||||
|
||||
$fallback = false;
|
||||
$link = $link_regex[0];
|
||||
}
|
||||
|
||||
if(
|
||||
(
|
||||
$fallback === false &&
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
) ||
|
||||
$fallback === true
|
||||
){
|
||||
|
||||
$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
|
||||
}
|
||||
}
|
||||
|
||||
// resolve shareable urls
|
||||
$this->init_multi_curl();
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_share($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
$status = curl_multi_exec($this->proc, $active);
|
||||
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||
|
||||
if($active){
|
||||
curl_multi_select($this->proc);
|
||||
}
|
||||
|
||||
}while($active && $status === CURLM_OK);
|
||||
|
||||
// parse downloaded garbage
|
||||
$keys = [];
|
||||
$replacements = [];
|
||||
|
||||
for($i=0; $i<count($this->handles); $i++){
|
||||
|
||||
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$a =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"a"
|
||||
);
|
||||
|
||||
if(count($a) === 0){
|
||||
|
||||
// failed to find link, will serve a resolver target as fallback
|
||||
$fallback = true;
|
||||
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||
}else{
|
||||
|
||||
$fallback = false;
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
}
|
||||
|
||||
if(
|
||||
(
|
||||
$fallback === false &&
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
) ||
|
||||
$fallback === true
|
||||
){
|
||||
|
||||
$keys[] = $this->handles[$i]["replacement"];
|
||||
@@ -785,22 +1048,6 @@ class google{
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
/*
|
||||
$session = $this->backend->session_get();
|
||||
|
||||
if(config::FPLAY_ENABLE_API === false){
|
||||
|
||||
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
|
||||
}
|
||||
|
||||
if($session === false){
|
||||
|
||||
throw new Exception("Session database is empty");
|
||||
}
|
||||
|
||||
$proxy = &$session["proxy"];
|
||||
$headers = &$session["headers"];
|
||||
*/
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
@@ -817,7 +1064,7 @@ class google{
|
||||
// country
|
||||
if($get["country"] != "any"){
|
||||
|
||||
$params["gl"] = $country;
|
||||
$params["gl"] = $get["country"];
|
||||
}
|
||||
|
||||
// nsfw
|
||||
@@ -826,15 +1073,15 @@ class google{
|
||||
// language
|
||||
if($get["lang"] != "any"){
|
||||
|
||||
$params["lr"] = "lang_" . $lang;
|
||||
$params["lr"] = "lang_" . $get["lang"];
|
||||
}
|
||||
|
||||
// generate tbs
|
||||
$tbs = [];
|
||||
|
||||
// get date
|
||||
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
|
||||
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
|
||||
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
|
||||
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
|
||||
|
||||
if(
|
||||
$get["older"] !== null ||
|
||||
@@ -1466,7 +1713,7 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -1478,15 +1725,15 @@ class google{
|
||||
// generate parameters
|
||||
if($get["npt"]){
|
||||
|
||||
[$params, $proxy] =
|
||||
[$params_json, $proxy] =
|
||||
$this->backend->get(
|
||||
$get["npt"],
|
||||
"images"
|
||||
);
|
||||
|
||||
$params = json_decode($params, true);
|
||||
$container = $params["c"];
|
||||
unset($params["c"]);
|
||||
$params_json = json_decode($params_json, true);
|
||||
$container = &$params_json["c"];
|
||||
$params = &$params_json["p"];
|
||||
}else{
|
||||
|
||||
$search = $get["s"];
|
||||
@@ -1505,6 +1752,7 @@ class google{
|
||||
$type = $get["type"];
|
||||
$format = $get["format"];
|
||||
$rights = $get["rights"];
|
||||
$container = null;
|
||||
|
||||
$params = [
|
||||
"q" => $search,
|
||||
@@ -1593,17 +1841,18 @@ class google{
|
||||
}
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
$data = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data["data"];
|
||||
$container = &$data["container"];
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
@@ -1701,13 +1950,16 @@ class google{
|
||||
$out["npt"] =
|
||||
$this->backend
|
||||
->store(
|
||||
json_encode($params),
|
||||
json_encode([
|
||||
"p" => $params,
|
||||
"c" => $container
|
||||
]),
|
||||
"image",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_gotos($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -2115,7 +2367,7 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -2474,14 +2726,14 @@ class google{
|
||||
"date" => $date,
|
||||
"thumb" => $thumb,
|
||||
"url" =>
|
||||
$this->unshiturl(
|
||||
$this->redirect_add_url(
|
||||
$a["attributes"]
|
||||
["href"]
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -567,6 +567,8 @@ class google_api{
|
||||
|
||||
foreach($json["items"] as $result){
|
||||
|
||||
$date = null;
|
||||
|
||||
//
|
||||
// probe for thumbnail
|
||||
//
|
||||
|
||||
+17
-8
@@ -725,18 +725,27 @@ class google_cse{
|
||||
// calculate rounded ratio
|
||||
if($found_size){
|
||||
|
||||
$aspect_ratio = $width / $height;
|
||||
|
||||
if($aspect_ratio >= 1.5){
|
||||
|
||||
$thumb["ratio"] = "16:9";
|
||||
}
|
||||
elseif($aspect_ratio >= 0.8){
|
||||
if(
|
||||
$width === 0 ||
|
||||
$height === 0
|
||||
){
|
||||
|
||||
$thumb["ratio"] = "1:1";
|
||||
}else{
|
||||
|
||||
$thumb["ratio"] = "9:16";
|
||||
$aspect_ratio = $width / $height;
|
||||
|
||||
if($aspect_ratio >= 1.5){
|
||||
|
||||
$thumb["ratio"] = "16:9";
|
||||
}
|
||||
elseif($aspect_ratio >= 0.8){
|
||||
|
||||
$thumb["ratio"] = "1:1";
|
||||
}else{
|
||||
|
||||
$thumb["ratio"] = "9:16";
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+170
-28
@@ -545,17 +545,20 @@ class startpage{
|
||||
switch($category["display_type"]){
|
||||
|
||||
case "web-google":
|
||||
case "web-bing":
|
||||
foreach($category["results"] as $result){
|
||||
|
||||
$sublinks = [];
|
||||
|
||||
foreach($result["siteLinks"] as $sublink){
|
||||
|
||||
$sublinks[] = [
|
||||
"title" => $sublink["title"],
|
||||
"description" => null,
|
||||
"url" => $sublink["clickUrl"]
|
||||
];
|
||||
if(isset($result["siteLinks"])){
|
||||
foreach($result["siteLinks"] as $sublink){
|
||||
|
||||
$sublinks[] = [
|
||||
"title" => $sublink["title"],
|
||||
"description" => null,
|
||||
"url" => $sublink["clickUrl"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$description =
|
||||
@@ -596,15 +599,17 @@ class startpage{
|
||||
|
||||
$out["web"][] = [
|
||||
"title" =>
|
||||
$this->titledots(
|
||||
html_entity_decode(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$result["title"]
|
||||
$this->remove_penguins(
|
||||
$this->titledots(
|
||||
html_entity_decode(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$result["title"]
|
||||
)
|
||||
)
|
||||
)
|
||||
),
|
||||
"description" => $description,
|
||||
"description" => $this->remove_penguins($description),
|
||||
"url" => $result["clickUrl"],
|
||||
"date" => $date,
|
||||
"type" => "web",
|
||||
@@ -619,8 +624,43 @@ class startpage{
|
||||
break;
|
||||
|
||||
case "images-qi-top":
|
||||
case "images-bing-top":
|
||||
foreach($category["results"] as $result){
|
||||
|
||||
if(isset($result["mdThumbnailWidth"])){
|
||||
|
||||
$thumb_width = $result["mdThumbnailWidth"];
|
||||
}elseif(isset($result["thumbnailWidth"])){
|
||||
|
||||
$thumb_width = $result["thumbnailWidth"];
|
||||
}else{
|
||||
|
||||
$thumb_width = null;
|
||||
}
|
||||
|
||||
if(isset($result["mdThumbnailHeight"])){
|
||||
|
||||
$thumb_height = $result["mdThumbnailHeight"];
|
||||
}elseif(isset($result["thumbnailHeight"])){
|
||||
|
||||
$thumb_height = $result["thumbnailHeight"];
|
||||
}else{
|
||||
|
||||
$thumb_height = null;
|
||||
}
|
||||
|
||||
if(isset($result["mdThumbnailUrl"])){
|
||||
|
||||
$thumb_source = $result["mdThumbnailUrl"];
|
||||
}elseif(isset($result["thumbnailUrl"])){
|
||||
|
||||
$thumb_source = $result["thumbnailUrl"];
|
||||
}else{
|
||||
|
||||
// should not happen
|
||||
continue;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" =>
|
||||
$this->titledots(
|
||||
@@ -638,19 +678,17 @@ class startpage{
|
||||
"height" => (int)$result["height"]
|
||||
],
|
||||
[
|
||||
"url" => $this->unshitimage($result["mdThumbnailUrl"]),
|
||||
"width" => (int)$result["mdThumbnailWidth"],
|
||||
"height" => (int)$result["mdThumbnailHeight"]
|
||||
"url" => $this->unshitimage($thumb_source),
|
||||
"width" => (int)$thumb_width,
|
||||
"height" => (int)$thumb_height
|
||||
]
|
||||
],
|
||||
"url" =>
|
||||
$result["altClickUrl"]
|
||||
"url" => $result["altClickUrl"]
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "spellsuggest-google":
|
||||
|
||||
if(isset($category["results"][0]["query"])){
|
||||
|
||||
$out["spelling"] =
|
||||
@@ -662,6 +700,60 @@ class startpage{
|
||||
}
|
||||
break;
|
||||
|
||||
case "spelladjust-bing":
|
||||
if(isset($category["results"][0]["alterationDisplayQuery"])){
|
||||
|
||||
$out["spelling"] =
|
||||
[
|
||||
"type" => "including",
|
||||
"using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]),
|
||||
"correction" => $category["results"][0]["originalQuery"]
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "aylf-bing-top":
|
||||
if(isset($category["results"][0]["query"])){
|
||||
|
||||
foreach($category["results"] as $ddm){
|
||||
|
||||
$out["related"][] = $ddm["query"];
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "video-bing-top":
|
||||
if(isset($category["results"][0]["clickUrl"])){
|
||||
|
||||
foreach($category["results"] as $video){
|
||||
|
||||
if(isset($video["thumbnailUrl"])){
|
||||
|
||||
$thumb = [
|
||||
"ratio" => "16:9",
|
||||
"url" => $this->unshitimage($video["thumbnailUrl"])
|
||||
];
|
||||
}else{
|
||||
|
||||
$thumb = [
|
||||
"ratio" => null,
|
||||
"url" => null
|
||||
];
|
||||
}
|
||||
|
||||
$out["video"][] = [
|
||||
"title" => $video["title"],
|
||||
"description" => $this->remove_penguins($video["description"]),
|
||||
"date" => isset($video["date"]) ? strtotime($video["date"]) : null,
|
||||
"duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null,
|
||||
"views" => $video["viewCount"],
|
||||
"thumb" => $thumb,
|
||||
"url" => $video["clickUrl"]
|
||||
];
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "dictionary-qi":
|
||||
foreach($category["results"] as $result){
|
||||
|
||||
@@ -732,6 +824,36 @@ class startpage{
|
||||
$out["answer"][] = $answer;
|
||||
}
|
||||
break;
|
||||
|
||||
case "news-bing-top":
|
||||
if(isset($category["results"][0]["clickUrl"])){
|
||||
|
||||
foreach($category["results"] as $news){
|
||||
|
||||
if(isset($news["thumbnailUrl"])){
|
||||
|
||||
$thumb = [
|
||||
"ratio" => "16:9",
|
||||
"url" => $news["thumbnailUrl"]
|
||||
];
|
||||
}else{
|
||||
|
||||
$thumb = [
|
||||
"ratio" => null,
|
||||
"url" => null
|
||||
];
|
||||
}
|
||||
|
||||
$out["news"][] = [
|
||||
"title" => $this->remove_penguins($news["title"]),
|
||||
"description" => $news["description"],
|
||||
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
|
||||
"thumb" => $thumb,
|
||||
"url" => $news["clickUrl"],
|
||||
];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1246,14 +1368,10 @@ class startpage{
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
|
||||
|
||||
//print_r($json);
|
||||
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
|
||||
if($category["display_type"] != "news-bing"){
|
||||
|
||||
// unsupported category
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach($category["results"] as $news){
|
||||
|
||||
if(
|
||||
@@ -1276,10 +1394,10 @@ class startpage{
|
||||
$out["news"][] = [
|
||||
"title" => $this->titledots($this->remove_penguins($news["title"])),
|
||||
"author" => $news["source"],
|
||||
"description" => $this->titledots($this->remove_penguins($news["description"])),
|
||||
"date" => (int)substr((string)$news["date"], 0, -3),
|
||||
"description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null,
|
||||
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
|
||||
"thumb" => $thumb,
|
||||
"url" => $news["clickUrl"]
|
||||
"url" => $this->remove_tracking_link($news["clickUrl"])
|
||||
];
|
||||
}
|
||||
}
|
||||
@@ -1327,6 +1445,30 @@ class startpage{
|
||||
return null;
|
||||
}
|
||||
|
||||
private function remove_tracking_link($url){
|
||||
|
||||
// https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog
|
||||
$query = parse_url($url);
|
||||
|
||||
if(
|
||||
!isset($query["host"]) ||
|
||||
$query["host"] != "api.airfind.com" ||
|
||||
!isset($query["query"])
|
||||
){
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
parse_str($query["query"], $res);
|
||||
|
||||
if(isset($res["redirectURL"])){
|
||||
|
||||
return $res["redirectURL"];
|
||||
}
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function unshitimage($url){
|
||||
|
||||
$query = parse_url($url, PHP_URL_QUERY);
|
||||
|
||||
+31
-70
@@ -205,13 +205,6 @@ class yep{
|
||||
"zh-tw" => "Chinese (Traditional)",
|
||||
"zu" => "Zulu"
|
||||
]
|
||||
],
|
||||
"nsfw" => [
|
||||
"display" => "NSFW",
|
||||
"option" => [
|
||||
"yes" => "Yes",
|
||||
"no" => "No"
|
||||
]
|
||||
]
|
||||
];
|
||||
}
|
||||
@@ -317,20 +310,23 @@ class yep{
|
||||
|
||||
// parse filters
|
||||
$filters = [
|
||||
"limit" => 100, // wwwwwwwwwwwwwww
|
||||
"query" => $search,
|
||||
"q" => $search,
|
||||
"limit" => 10 // no fun allowed
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
|
||||
if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
|
||||
if($get["lang"] != "any"){
|
||||
|
||||
$filters["language"] = $get["lang"];
|
||||
$filters["lang"] = $get["lang"];
|
||||
}
|
||||
|
||||
try{
|
||||
|
||||
// https://api.yep.com/search?limit=20&query=asmr
|
||||
// https://yep.com/web/v1/search?q=asmr&limit=10
|
||||
$json =
|
||||
$this->get(
|
||||
$this->backend->get_ip(),
|
||||
"https://api.yep.com/search",
|
||||
"https://yep.com/web/v1/search",
|
||||
$filters
|
||||
);
|
||||
|
||||
@@ -343,71 +339,37 @@ class yep{
|
||||
|
||||
$json = json_decode($json, true);
|
||||
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
if(isset($json[1]["correction"])){
|
||||
if(isset($json["code"])){
|
||||
|
||||
$out["spelling"] = [
|
||||
"type" => "not_many",
|
||||
"using" => $search,
|
||||
"correction" => $json[1]["correction"][1]
|
||||
];
|
||||
if(isset($json["reason"])){
|
||||
|
||||
throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
|
||||
}
|
||||
|
||||
throw new Exception("Yep returned an unknown error");
|
||||
}
|
||||
|
||||
if(isset($json[1]["results"])){
|
||||
foreach($json[1]["results"] as $item){
|
||||
if(isset($json["results"])){
|
||||
foreach($json["results"] as $item){
|
||||
|
||||
switch(strtolower($item["type"])){
|
||||
|
||||
case "organic":
|
||||
$sublinks = [];
|
||||
|
||||
if(isset($item["sitelinks"]["full"])){
|
||||
|
||||
foreach($item["sitelinks"]["full"] as $link){
|
||||
|
||||
$sublinks[] = [
|
||||
"title" => $link["title"],
|
||||
"date" => null,
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$link["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"url" => $link["url"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$out["web"][] = [
|
||||
"title" => $item["title"],
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$item["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"url" => $item["url"],
|
||||
"date" => null,
|
||||
"type" => "web",
|
||||
"thumb" => [
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"sublink" => $sublinks,
|
||||
"table" => []
|
||||
];
|
||||
break;
|
||||
}
|
||||
$out["web"][] = [
|
||||
"title" => $item["title"],
|
||||
"description" => str_replace("\n", " ", $item["description"]),
|
||||
"url" => $item["url"],
|
||||
"date" => null,
|
||||
"type" => "web",
|
||||
"thumb" => [
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"sublink" => [],
|
||||
"table" => []
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -445,7 +407,6 @@ class yep{
|
||||
"limit" => 99 // for some reason they dont show anything sometimes at 100 results
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
|
||||
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
|
||||
|
||||
// add api key
|
||||
|
||||
+10
-1
@@ -678,6 +678,15 @@ class yt{
|
||||
$video["longBylineText"]
|
||||
),
|
||||
"url" =>
|
||||
isset(
|
||||
$video["ownerText"]
|
||||
["runs"]
|
||||
[0]
|
||||
["navigationEndpoint"]
|
||||
["commandMetadata"]
|
||||
["webCommandMetadata"]
|
||||
["url"]
|
||||
) ?
|
||||
"https://www.youtube.com" .
|
||||
$video["ownerText"]
|
||||
["runs"]
|
||||
@@ -685,7 +694,7 @@ class yt{
|
||||
["navigationEndpoint"]
|
||||
["commandMetadata"]
|
||||
["webCommandMetadata"]
|
||||
["url"],
|
||||
["url"] : null,
|
||||
"avatar" =>
|
||||
$video["channelThumbnailSupportedRenderers"]
|
||||
["channelThumbnailWithLinkRenderer"]
|
||||
|
||||
+28
-4
@@ -67,6 +67,10 @@ $settings = [
|
||||
"value" => "brave",
|
||||
"text" => "Brave"
|
||||
],
|
||||
[
|
||||
"value" => "bing",
|
||||
"text" => "Bing"
|
||||
],
|
||||
[
|
||||
"value" => "ddg",
|
||||
"text" => "DuckDuckGo"
|
||||
@@ -79,17 +83,25 @@ $settings = [
|
||||
"value" => "google",
|
||||
"text" => "Google"
|
||||
],
|
||||
[
|
||||
"value" => "yahoo",
|
||||
"text" => "Yahoo"
|
||||
],
|
||||
[
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "kagi",
|
||||
"text" => "Kagi"
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
"value" => "ecosia",
|
||||
"text" => "Ecosia"
|
||||
],
|
||||
[
|
||||
"value" => "kagi",
|
||||
"text" => "Kagi"
|
||||
],
|
||||
[
|
||||
"value" => "marginalia",
|
||||
@@ -102,6 +114,18 @@ $settings = [
|
||||
[
|
||||
"value" => "sc",
|
||||
"text" => "SoundCloud"
|
||||
],
|
||||
[
|
||||
"value" => "coccoc",
|
||||
"text" => "Coccoc"
|
||||
],
|
||||
[
|
||||
"value" => "naver",
|
||||
"text" => "Naver"
|
||||
],
|
||||
[
|
||||
"value" => "purili",
|
||||
"text" => "Purili"
|
||||
]
|
||||
]
|
||||
],
|
||||
|
||||
Reference in new issue
Block a user