Compare commits
25
Commits
7eb4ac14c8
...
master
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4c7c4ea072 | ||
|
|
761a1e91cd | ||
|
|
c5311bc399 | ||
|
|
4536cef88f | ||
|
|
0c3844d578 | ||
|
|
30cd3c8492 | ||
|
|
aa818ac925 | ||
|
|
5967462b6e | ||
|
|
b014a3199d | ||
|
|
24130effe6 | ||
|
|
5d8c9a5798 | ||
|
|
6bbb37d86e | ||
|
|
c0324d7d61 | ||
|
|
2218fd6f04 | ||
|
|
be91ec1806 | ||
|
|
57a72e5b5e | ||
|
|
9f0437d949 | ||
|
|
d91834f86f | ||
|
|
42d0252e2f | ||
|
|
a27a8ed01a | ||
|
|
9db95c79d2 | ||
|
|
25d4f991c2 | ||
|
|
e1d0ab121f | ||
|
|
33ce0a062b | ||
|
|
3b3c759ce5 |
No files matched your search
+1
-1
@@ -31,7 +31,7 @@ echo json_encode(
|
|||||||
"real_requests" => $real_requests,
|
"real_requests" => $real_requests,
|
||||||
"bot_requests" => $bot_requests,
|
"bot_requests" => $bot_requests,
|
||||||
"api_enabled" => config::API_ENABLED,
|
"api_enabled" => config::API_ENABLED,
|
||||||
"alt_addresses" => config::ALT_ADDRESSES,
|
"alt_addresses" => config::ADDRESSES,
|
||||||
"version" => config::VERSION
|
"version" => config::VERSION
|
||||||
],
|
],
|
||||||
"instances" => config::INSTANCES
|
"instances" => config::INSTANCES
|
||||||
|
|||||||
+116
-24
@@ -7,37 +7,51 @@ class autocomplete{
|
|||||||
|
|
||||||
public function __construct(){
|
public function __construct(){
|
||||||
|
|
||||||
header("Content-Type: application/json");
|
header("Content-Type: application/x-suggestions+json");
|
||||||
|
|
||||||
$this->scrapers = [
|
$this->scrapers = [
|
||||||
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
|
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
|
||||||
|
"bing" => "https://www.bing.com/osjson.aspx?query={searchTerms}",
|
||||||
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
|
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
|
||||||
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
|
"ecosia" => "https://ac.ecosia.org/autocomplete?q={searchTerms}&type=list",
|
||||||
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
|
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3",
|
||||||
|
"google" => "https://www.google.com/complete/search?client=chrome&q={searchTerms}",
|
||||||
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
|
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
|
||||||
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
|
"marginalia" => "https://marginalia-search.com/suggest/?partial={searchTerms}",
|
||||||
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
|
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
|
||||||
"sc" => "",
|
"sc" => "",
|
||||||
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
|
"kagi" => "https://kagisuggest.com/api/autosuggest?q={searchTerms}",
|
||||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
|
"startpage" => "https://www.startpage.com/osuggestions?q={searchTerms}",
|
||||||
|
"yahoo" => "https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=sd1&command={searchTerms}",
|
||||||
|
"purili" => "https://puri.li/api/suggest?q={searchTerms}",
|
||||||
|
"coccoc" => "https://coccoc.com/composer/autocomplete?q={searchTerms}",
|
||||||
|
"naver" => "https://ac.search.naver.com/nx/ac?q={searchTerms}&st=100"
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// add duplicates
|
||||||
|
$this->scrapers["google_api"] = &$this->scrapers["google"];
|
||||||
|
$this->scrapers["google_cse"] = &$this->scrapers["google"];
|
||||||
|
$this->scrapers["yahoo_japan"] = &$this->scrapers["yahoo"];
|
||||||
|
|
||||||
/*
|
/*
|
||||||
Sanitize input
|
Sanitize input
|
||||||
*/
|
*/
|
||||||
if(!isset($_GET["s"])){
|
if(!isset($_GET["s"])){
|
||||||
|
|
||||||
$this->do404("Missing search(s) parameter");
|
// Missing search(s) parameter
|
||||||
|
$this->doempty();
|
||||||
}
|
}
|
||||||
|
|
||||||
if(is_string($_GET["s"]) === false){
|
if(is_string($_GET["s"]) === false){
|
||||||
|
|
||||||
$this->do404("Invalid search(s) parameter");
|
// Invalid search(s) parameter
|
||||||
|
$this->doempty();
|
||||||
}
|
}
|
||||||
|
|
||||||
if(strlen($_GET["s"]) > 500){
|
if(strlen($_GET["s"]) > 500){
|
||||||
|
|
||||||
$this->do404("Search(s) exceeds the 500 char length");
|
// Search(s) exceeds the 500 char length
|
||||||
|
$this->doempty();
|
||||||
}
|
}
|
||||||
|
|
||||||
/*
|
/*
|
||||||
@@ -72,9 +86,8 @@ class autocomplete{
|
|||||||
// return results
|
// return results
|
||||||
switch($scraper){
|
switch($scraper){
|
||||||
|
|
||||||
case "google":
|
|
||||||
case "yt":
|
case "yt":
|
||||||
// handle google cause they want to be a special snowflake :(
|
// handle youtube cause it wants to be a special snowflake :(
|
||||||
$js = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
$js = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||||
|
|
||||||
preg_match(
|
preg_match(
|
||||||
@@ -141,12 +154,12 @@ class autocomplete{
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case "marginalia":
|
case "marginalia":
|
||||||
|
case "purili":
|
||||||
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||||
|
|
||||||
$json = json_decode($json, true);
|
$json = json_decode($json, true);
|
||||||
if($json === null){
|
if($json === null){
|
||||||
|
|
||||||
|
|
||||||
$this->doempty();
|
$this->doempty();
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -159,10 +172,98 @@ class autocomplete{
|
|||||||
);
|
);
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case "yahoo":
|
||||||
|
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||||
|
|
||||||
|
$json = json_decode($json, true);
|
||||||
|
if(
|
||||||
|
$json === null ||
|
||||||
|
!isset($json["r"][0]["k"])
|
||||||
|
){
|
||||||
|
|
||||||
|
$this->doempty();
|
||||||
|
}
|
||||||
|
|
||||||
|
$payload = [];
|
||||||
|
|
||||||
|
foreach($json["r"] as $term){
|
||||||
|
|
||||||
|
if(isset($term["k"])){
|
||||||
|
|
||||||
|
$payload[] = $term["k"];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
echo json_encode(
|
||||||
|
[
|
||||||
|
$_GET["s"],
|
||||||
|
$payload
|
||||||
|
],
|
||||||
|
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case "naver":
|
||||||
|
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||||
|
|
||||||
|
$json = json_decode($json, true);
|
||||||
|
if(
|
||||||
|
$json === null ||
|
||||||
|
!isset($json["items"][0][0])
|
||||||
|
){
|
||||||
|
|
||||||
|
$this->doempty();
|
||||||
|
}
|
||||||
|
|
||||||
|
$payload = [];
|
||||||
|
|
||||||
|
foreach($json["items"][0] as $term){
|
||||||
|
|
||||||
|
$payload[] = $term[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
echo json_encode(
|
||||||
|
[
|
||||||
|
$_GET["s"],
|
||||||
|
$payload
|
||||||
|
],
|
||||||
|
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
|
||||||
|
case "coccoc":
|
||||||
|
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
|
||||||
|
|
||||||
|
$json = json_decode($json, true);
|
||||||
|
if(
|
||||||
|
$json === null ||
|
||||||
|
!isset($json["suggestions"])
|
||||||
|
){
|
||||||
|
|
||||||
|
$this->doempty();
|
||||||
|
}
|
||||||
|
|
||||||
|
echo json_encode(
|
||||||
|
[
|
||||||
|
$_GET["s"],
|
||||||
|
$json["suggestions"]
|
||||||
|
],
|
||||||
|
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
||||||
|
);
|
||||||
|
break;
|
||||||
|
|
||||||
default:
|
default:
|
||||||
// if it respects the openSearch protocol
|
// if it respects the openSearch protocol
|
||||||
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
|
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
|
||||||
|
|
||||||
|
if(
|
||||||
|
!isset($json[1]) ||
|
||||||
|
!is_array($json[1])
|
||||||
|
){
|
||||||
|
|
||||||
|
$json[1] = [];
|
||||||
|
}
|
||||||
|
|
||||||
echo json_encode(
|
echo json_encode(
|
||||||
[
|
[
|
||||||
$_GET["s"],
|
$_GET["s"],
|
||||||
@@ -185,10 +286,10 @@ class autocomplete{
|
|||||||
|
|
||||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||||
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0",
|
["User-Agent: " . config::USER_AGENT,
|
||||||
"Accept: application/json, text/javascript, */*; q=0.01",
|
"Accept: application/json, text/javascript, */*; q=0.01",
|
||||||
"Accept-Language: en-US,en;q=0.5",
|
"Accept-Language: en-US,en;q=0.5",
|
||||||
"Accept-Encoding: gzip",
|
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||||
"DNT: 1",
|
"DNT: 1",
|
||||||
"Connection: keep-alive",
|
"Connection: keep-alive",
|
||||||
"Sec-Fetch-Dest: empty",
|
"Sec-Fetch-Dest: empty",
|
||||||
@@ -214,19 +315,10 @@ class autocomplete{
|
|||||||
|
|
||||||
}catch(Exception $error){
|
}catch(Exception $error){
|
||||||
|
|
||||||
do404("Curl error: " . $error->getMessage());
|
doempty();
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private function do404($error){
|
|
||||||
|
|
||||||
echo json_encode(
|
|
||||||
["error" => $error],
|
|
||||||
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
|
|
||||||
);
|
|
||||||
die();
|
|
||||||
}
|
|
||||||
|
|
||||||
private function doempty(){
|
private function doempty(){
|
||||||
|
|
||||||
echo json_encode(
|
echo json_encode(
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
|
||||||
+3
-2
@@ -154,7 +154,7 @@ class config{
|
|||||||
|
|
||||||
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
|
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
|
||||||
// Changing this might break things.
|
// Changing this might break things.
|
||||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0";
|
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:160.0) Gecko/20100101 Firefox/160.0";
|
||||||
|
|
||||||
// User agent to use with 4get-friendly APIs
|
// User agent to use with 4get-friendly APIs
|
||||||
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
|
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
|
||||||
@@ -225,6 +225,7 @@ class config{
|
|||||||
// use the API (Eg: "public"). API has less filters.
|
// use the API (Eg: "public"). API has less filters.
|
||||||
const MARGINALIA_API_KEY = null;
|
const MARGINALIA_API_KEY = null;
|
||||||
|
|
||||||
// Yep
|
// Scrapers to use the API on
|
||||||
const YEP_USE_API = false;
|
const YEP_USE_API = false;
|
||||||
|
const BRAVE_USE_API = false;
|
||||||
}
|
}
|
||||||
+154
-163
@@ -1,31 +1,16 @@
|
|||||||
# Sample Apache2 configuration
|
# Sample Apache2 configuration
|
||||||
This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
|
This is the apache2 configuration file used on the 4get.ca official instance, in hopes that it's useful to you!
|
||||||
|
|
||||||
|
Apache2 now sits behind haproxy, so it's configured to accept plain HTTP traffic on port `8080`. I highly recommend you setup haproxy, since apache2 is pretty slow at handling SSL/HTTP2. A sample haproxy config has been added at the end of this document as a reference.
|
||||||
|
|
||||||
Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
|
Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/branch/master/docs/apache2.md">go here.</a>.
|
||||||
|
|
||||||
```xml
|
```xml
|
||||||
<VirtualHost *:443>
|
<VirtualHost *:8080>
|
||||||
ServerName www.4get.ca
|
|
||||||
|
|
||||||
SSLEngine On
|
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
|
||||||
|
|
||||||
RedirectMatch 301 ^(.*)$ https://4get.ca$1
|
|
||||||
</VirtualHost>
|
|
||||||
|
|
||||||
<VirtualHost *:443>
|
|
||||||
ServerName 4get.ca
|
ServerName 4get.ca
|
||||||
|
ServerAlias 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
|
||||||
|
|
||||||
ServerAdmin will@lolcat.ca
|
# compress
|
||||||
DocumentRoot /var/www/4get
|
|
||||||
|
|
||||||
SSLEngine On
|
|
||||||
SSLOptions +StdEnvVars
|
|
||||||
|
|
||||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
|
||||||
|
|
||||||
AddOutputFilterByType DEFLATE application/json
|
AddOutputFilterByType DEFLATE application/json
|
||||||
AddOutputFilterByType DEFLATE application/javascript
|
AddOutputFilterByType DEFLATE application/javascript
|
||||||
AddOutputFilterByType DEFLATE application/x-javascript
|
AddOutputFilterByType DEFLATE application/x-javascript
|
||||||
@@ -33,167 +18,173 @@ Looking for the apache2 guide? <a href="https://git.lolcat.ca/lolcat/4get/src/br
|
|||||||
AddOutputFilterByType DEFLATE text/plain
|
AddOutputFilterByType DEFLATE text/plain
|
||||||
AddOutputFilterByType DEFLATE text/css
|
AddOutputFilterByType DEFLATE text/css
|
||||||
|
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
DocumentRoot /var/www/4get
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
ErrorLog ${APACHE_LOG_DIR}/error.log
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
CustomLog /dev/null combined
|
||||||
|
#CustomLog /var/log/apache2/access.log combined
|
||||||
|
|
||||||
|
<FilesMatch \.php$>
|
||||||
|
SetHandler "proxy:unix:/run/php/php-fpm.sock|fcgi://localhost/"
|
||||||
|
</FilesMatch>
|
||||||
|
|
||||||
<Directory /var/www/4get>
|
<Directory /var/www/4get>
|
||||||
Options -MultiViews
|
Options -MultiViews
|
||||||
AllowOverride All
|
AllowOverride All
|
||||||
Require all granted
|
Require all granted
|
||||||
|
|
||||||
|
# prevent 404s from hitting php-fpm
|
||||||
RewriteEngine On
|
RewriteEngine On
|
||||||
|
RewriteCond %{REQUEST_URI} \.php$ [NC]
|
||||||
|
RewriteCond %{REQUEST_FILENAME} !-f
|
||||||
|
RewriteRule ^ - [R=404,L]
|
||||||
|
|
||||||
|
# prevent /index hits
|
||||||
|
RewriteRule ^index$ - [R=404,L]
|
||||||
|
|
||||||
|
# 404 direct .php file access
|
||||||
|
RewriteCond %{THE_REQUEST} "\s/+[^?\s]*\.php(?:/|[?\s])" [NC]
|
||||||
|
RewriteRule ^ - [R=404,L]
|
||||||
|
|
||||||
|
# internally redirect to non-php extension
|
||||||
RewriteCond %{REQUEST_FILENAME} !-d
|
RewriteCond %{REQUEST_FILENAME} !-d
|
||||||
RewriteCond %{REQUEST_FILENAME} !-f
|
RewriteCond %{REQUEST_FILENAME} !-f
|
||||||
|
RewriteCond %{REQUEST_FILENAME}.php -f
|
||||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
||||||
</Directory>
|
</Directory>
|
||||||
|
|
||||||
# deny access to private resources
|
# deny access to private resources
|
||||||
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
|
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
|
||||||
Require all denied
|
Require all denied
|
||||||
</DirectoryMatch>
|
</DirectoryMatch>
|
||||||
|
|
||||||
# remove directory listing from /icons
|
|
||||||
<Directory /var/www/4get/icons>
|
<Directory /var/www/4get/icons>
|
||||||
Options -Indexes
|
Options -Indexes
|
||||||
</Directory>
|
</Directory>
|
||||||
</VirtualHost>
|
</VirtualHost>
|
||||||
|
```
|
||||||
<VirtualHost *:443>
|
|
||||||
ServerName www.lolcat.ca
|
# Haproxy configuration
|
||||||
|
|
||||||
SSLEngine On
|
```cfg
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
global
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
#log /dev/log local0
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
log /dev/log local0 warning
|
||||||
|
log /dev/log local1 notice
|
||||||
RedirectMatch 301 ^(.*)$ https://lolcat.ca$1
|
chroot /var/lib/haproxy
|
||||||
</VirtualHost>
|
stats socket /run/haproxy/admin.sock mode 660 level admin
|
||||||
|
stats timeout 30s
|
||||||
<VirtualHost *:443>
|
user haproxy
|
||||||
ServerName lolcat.ca
|
group haproxy
|
||||||
|
daemon
|
||||||
ServerAdmin will@lolcat.ca
|
|
||||||
DocumentRoot /var/www/lolcat
|
# Default SSL material locations
|
||||||
|
ca-base /etc/ssl/certs
|
||||||
SSLEngine On
|
crt-base /etc/ssl/private
|
||||||
SSLOptions +StdEnvVars
|
|
||||||
|
# See: https://ssl-config.mozilla.org/#server=haproxy&server-version=2.0.3&config=intermediate
|
||||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
#ssl-default-bind-ciphers ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384
|
||||||
|
ssl-default-bind-ciphers HIGH:!aNULL
|
||||||
AddOutputFilterByType DEFLATE application/json
|
ssl-default-bind-ciphersuites TLS_AES_128_GCM_SHA256:TLS_AES_256_GCM_SHA384:TLS_CHACHA20_POLY1305_SHA256
|
||||||
AddOutputFilterByType DEFLATE application/javascript
|
ssl-default-bind-options ssl-min-ver TLSv1.2 no-tls-tickets
|
||||||
AddOutputFilterByType DEFLATE application/x-javascript
|
|
||||||
AddOutputFilterByType DEFLATE text/html
|
# ja4 settings
|
||||||
AddOutputFilterByType DEFLATE text/plain
|
tune.lua.bool-sample-conversion normal
|
||||||
AddOutputFilterByType DEFLATE text/css
|
tune.ssl.capture-buffer-size 190
|
||||||
|
lua-load /etc/haproxy/lua/ja4.lua
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
defaults
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
log global
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
mode http
|
||||||
|
#option httplog
|
||||||
<Directory /var/www/lolcat>
|
option dontlognull
|
||||||
Options -MultiViews
|
timeout connect 5000
|
||||||
AllowOverride All
|
timeout client 50000
|
||||||
Require all granted
|
timeout server 50000
|
||||||
|
errorfile 400 /etc/haproxy/errors/400.http
|
||||||
RewriteEngine On
|
errorfile 403 /etc/haproxy/errors/403.http
|
||||||
RewriteCond %{REQUEST_FILENAME} !-d
|
errorfile 408 /etc/haproxy/errors/408.http
|
||||||
RewriteCond %{REQUEST_FILENAME} !-f
|
errorfile 500 /etc/haproxy/errors/500.http
|
||||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
errorfile 502 /etc/haproxy/errors/502.http
|
||||||
</Directory>
|
errorfile 503 /etc/haproxy/errors/503.http
|
||||||
</VirtualHost>
|
errorfile 504 /etc/haproxy/errors/504.http
|
||||||
|
|
||||||
<VirtualHost *:443>
|
frontend http_80_in
|
||||||
ServerName www.nyym.co
|
# bind tcp ip4/6
|
||||||
|
bind *:80
|
||||||
SSLEngine On
|
bind [::]:80
|
||||||
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
|
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
|
mode http
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
|
|
||||||
|
# handle ip.lolcat.ca+onion addy on plain HTTP
|
||||||
RedirectMatch 301 ^(.*)$ https://nyym.co$1
|
acl is_ip_host hdr(host) -i ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca 4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion
|
||||||
</VirtualHost>
|
|
||||||
|
# redirect to https
|
||||||
<VirtualHost *:443>
|
http-request redirect scheme https code 301 if { hdr(host) -i lolcat.ca }
|
||||||
ServerName nyym.co
|
http-request redirect scheme https code 301 if { hdr(host) -i git.lolcat.ca }
|
||||||
|
http-request redirect scheme https code 301 if { hdr(host) -i lick.lolcat.ca }
|
||||||
ServerAdmin will@lolcat.ca
|
http-request redirect scheme https code 301 if { hdr(host) -i cf-tls.lolcat.ca }
|
||||||
DocumentRoot /var/www/nyym
|
http-request redirect scheme https code 301 if { hdr(host) -i cf-js.lolcat.ca }
|
||||||
|
http-request redirect scheme https code 301 if { hdr(host) -i cf-captcha.lolcat.ca }
|
||||||
SSLEngine On
|
|
||||||
SSLOptions +StdEnvVars
|
# redirect www.lolcat.ca to lolcat.ca
|
||||||
|
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
|
||||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
|
||||||
|
# everything else redirects to 4get.ca
|
||||||
AddOutputFilterByType DEFLATE application/json
|
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless is_ip_host
|
||||||
AddOutputFilterByType DEFLATE application/javascript
|
|
||||||
AddOutputFilterByType DEFLATE application/x-javascript
|
# ... unless its an ip host, or the onion domain
|
||||||
AddOutputFilterByType DEFLATE text/html
|
use_backend apache if is_ip_host
|
||||||
AddOutputFilterByType DEFLATE text/plain
|
|
||||||
AddOutputFilterByType DEFLATE text/css
|
frontend https_443_in
|
||||||
|
# bind tcp ip4/6
|
||||||
SSLCertificateFile /etc/letsencrypt/live/nyym.co/fullchain.pem
|
bind *:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/nyym.co/privkey.pem
|
bind [::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/nyym.co/chain.pem
|
|
||||||
|
# QUIC is disabled. See https://github.com/haproxy/haproxy/issues/3529
|
||||||
<Directory /var/www/nyym>
|
# bind udp ip4/6
|
||||||
Options -MultiViews
|
#bind quic4@*:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
|
||||||
AllowOverride All
|
#bind quic6@[::]:443 ssl crt /etc/apache2/ssl/4get-haproxy.pem crt /etc/apache2/ssl/lolcat-haproxy.pem alpn h3
|
||||||
Require all granted
|
|
||||||
|
# tell client we support http3 now
|
||||||
RewriteEngine On
|
#http-response set-header Alt-Svc "h3=\":443\"; ma=86400"
|
||||||
RewriteCond %{REQUEST_FILENAME} !-d
|
|
||||||
RewriteCond %{REQUEST_FILENAME} !-f
|
mode http
|
||||||
RewriteRule ^([^\.]+)$ $1.php [NC,L]
|
|
||||||
</Directory>
|
# redirect all unknown domains to 4get.ca
|
||||||
</VirtualHost>
|
acl valid_https_hosts hdr(host) -i 4get.ca lolcat.ca ip.lolcat.ca ip4.lolcat.ca ip6.lolcat.ca git.lolcat.ca lick.lolcat.ca cf-tls.lolcat.ca cf-js.lolcat.ca cf-captcha.lolcat.ca www.lolcat.ca
|
||||||
|
|
||||||
<VirtualHost *:443>
|
# add onion domain to 4get.ca
|
||||||
ServerName git.lolcat.ca
|
acl is_4get hdr(host) -i 4get.ca
|
||||||
|
http-request set-var(txn.is_4get) bool(true) if is_4get
|
||||||
SSLEngine On
|
http-response set-header Onion-Location http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion%[capture.req.uri] if { var(txn.is_4get) -m bool }
|
||||||
SSLOptions +StdEnvVars
|
|
||||||
|
# git.lolcat.ca -> gitea service
|
||||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
acl acl_gitea hdr(host) -i git.lolcat.ca
|
||||||
|
|
||||||
AddOutputFilterByType DEFLATE application/json
|
# redirects
|
||||||
AddOutputFilterByType DEFLATE application/javascript
|
http-request redirect location https://4get.ca%[capture.req.uri] code 301 unless valid_https_hosts
|
||||||
AddOutputFilterByType DEFLATE application/x-javascript
|
http-request redirect location https://lolcat.ca%[capture.req.uri] code 301 if { hdr(host) -i www.lolcat.ca }
|
||||||
AddOutputFilterByType DEFLATE text/html
|
|
||||||
AddOutputFilterByType DEFLATE text/plain
|
# add security headers
|
||||||
AddOutputFilterByType DEFLATE text/css
|
http-response set-header Strict-Transport-Security "max-age=31536000; includeSubDomains; preload"
|
||||||
|
http-response set-header X-Content-Type-Options "nosniff"
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
http-response set-header X-Frame-Options "SAMEORIGIN"
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
# route
|
||||||
|
use_backend gitea if acl_gitea
|
||||||
ProxyPreserveHost On
|
default_backend apache
|
||||||
ProxyRequests off
|
|
||||||
AllowEncodedSlashes NoDecode
|
backend apache
|
||||||
ProxyPass / http://localhost:3000/ nocanon
|
# pass ja4
|
||||||
</VirtualHost>
|
http-request lua.fingerprint_ja4 if { ssl_fc }
|
||||||
|
http-request set-header X-Ja4 %[var(txn.fingerprint_ja4)] if { ssl_fc }
|
||||||
<VirtualHost *:443>
|
|
||||||
ServerName live.lolcat.ca
|
# delete server header
|
||||||
|
http-response del-header Server
|
||||||
ServerAdmin will@lolcat.ca
|
|
||||||
DocumentRoot /var/www/live
|
# pass remote IP
|
||||||
|
option forwardfor
|
||||||
SSLEngine On
|
server apache_server 127.0.0.1:8080 check
|
||||||
SSLOptions +StdEnvVars
|
|
||||||
|
backend gitea
|
||||||
#ErrorLog ${APACHE_LOG_DIR}/error.log
|
server gitea_server 127.0.0.1:3000 check
|
||||||
|
|
||||||
AddOutputFilterByType DEFLATE application/json
|
|
||||||
AddOutputFilterByType DEFLATE application/javascript
|
|
||||||
AddOutputFilterByType DEFLATE application/x-javascript
|
|
||||||
AddOutputFilterByType DEFLATE text/html
|
|
||||||
AddOutputFilterByType DEFLATE text/plain
|
|
||||||
AddOutputFilterByType DEFLATE text/css
|
|
||||||
|
|
||||||
SSLCertificateFile /etc/letsencrypt/live/4get.ca/fullchain.pem
|
|
||||||
SSLCertificateKeyFile /etc/letsencrypt/live/4get.ca/privkey.pem
|
|
||||||
SSLCertificateChainFile /etc/letsencrypt/live/4get.ca/chain.pem
|
|
||||||
</VirtualHost>
|
|
||||||
```
|
```
|
||||||
+2
-2
@@ -12,9 +12,9 @@ class backend{
|
|||||||
/*
|
/*
|
||||||
Proxy stuff
|
Proxy stuff
|
||||||
*/
|
*/
|
||||||
public function get_ip($proxy_index_raw = null){
|
public function get_ip($proxy_index_raw = null, $use_alt_list = false){
|
||||||
|
|
||||||
$pool = constant("config::PROXY_" . strtoupper($this->scraper));
|
$pool = constant("config::PROXY_" . strtoupper($this->scraper) . ($use_alt_list ? "_ALT" : ""));
|
||||||
if($pool === false){
|
if($pool === false){
|
||||||
|
|
||||||
// we don't want a proxy, fuck off!
|
// we don't want a proxy, fuck off!
|
||||||
|
|||||||
+1
-2
@@ -1119,7 +1119,6 @@ class frontend{
|
|||||||
$filters["scraper"] = [
|
$filters["scraper"] = [
|
||||||
"display" => "Scraper",
|
"display" => "Scraper",
|
||||||
"option" => [
|
"option" => [
|
||||||
//"fget" => "fget",
|
|
||||||
"ddg" => "DuckDuckGo",
|
"ddg" => "DuckDuckGo",
|
||||||
//"yahoo" => "Yahoo!",
|
//"yahoo" => "Yahoo!",
|
||||||
"brave" => "Brave",
|
"brave" => "Brave",
|
||||||
@@ -1207,7 +1206,7 @@ class frontend{
|
|||||||
"yahoo_japan" => "Yahoo! JAPAN",
|
"yahoo_japan" => "Yahoo! JAPAN",
|
||||||
"startpage" => "Startpage",
|
"startpage" => "Startpage",
|
||||||
//"mojeek" => "Mojeek",
|
//"mojeek" => "Mojeek",
|
||||||
"baidu" => "Baidu"
|
//"baidu" => "Baidu"
|
||||||
]
|
]
|
||||||
];
|
];
|
||||||
break;
|
break;
|
||||||
|
|||||||
+26
-13
@@ -3,32 +3,45 @@
|
|||||||
header("Content-Type: application/xml");
|
header("Content-Type: application/xml");
|
||||||
include "data/config.php";
|
include "data/config.php";
|
||||||
|
|
||||||
$domain =
|
$host = strtolower(explode(":", $_SERVER["HTTP_HOST"], 2)[0]);
|
||||||
htmlspecialchars(
|
|
||||||
(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on" ? "https" : "http") .
|
|
||||||
'://' . $_SERVER["HTTP_HOST"]
|
|
||||||
);
|
|
||||||
|
|
||||||
|
// detect if 4get should serve the HTTPS endpoint
|
||||||
if(
|
if(
|
||||||
preg_match(
|
preg_match(
|
||||||
'/\.onion$/',
|
'/\.(onion|i2p)$/',
|
||||||
$domain
|
strtolower($host),
|
||||||
|
$match
|
||||||
)
|
)
|
||||||
){
|
){
|
||||||
|
|
||||||
$onion = true;
|
$ssl = false;
|
||||||
|
$src = $match[1];
|
||||||
|
}elseif(
|
||||||
|
is_string(
|
||||||
|
filter_var(
|
||||||
|
$host,
|
||||||
|
FILTER_VALIDATE_IP
|
||||||
|
)
|
||||||
|
) ||
|
||||||
|
$host == "localhost"
|
||||||
|
){
|
||||||
|
|
||||||
|
$ssl = false;
|
||||||
|
$src = "localhost";
|
||||||
}else{
|
}else{
|
||||||
|
|
||||||
$onion = false;
|
$ssl = true;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$domain = ($ssl ? "https" : "http") . "://" . $_SERVER["HTTP_HOST"];
|
||||||
|
|
||||||
echo
|
echo
|
||||||
'<?xml version="1.0" encoding="UTF-8"?>' .
|
|
||||||
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
|
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
|
||||||
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($onion ? " (onion)" : "") . '</ShortName>' .
|
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($ssl === false ? " ({$src})" : "") . '</ShortName>' .
|
||||||
|
'<Description>' . htmlspecialchars(config::SERVER_SHORT_DESCRIPTION) . '</Description>' .
|
||||||
'<InputEncoding>UTF-8</InputEncoding>' .
|
'<InputEncoding>UTF-8</InputEncoding>' .
|
||||||
'<Image width="16" height="16">' . $domain . '/favicon.ico</Image>' .
|
'<Image width="32" height="32" type="image/x-icon">' . htmlspecialchars($domain) . '/favicon.ico</Image>' .
|
||||||
'<Url type="text/html" method="GET" template="' . $domain . '/web?s={searchTerms}"/>';
|
'<Url rel="results" type="text/html" method="GET" template="' . htmlspecialchars($domain) . '/web?s={searchTerms}"/>';
|
||||||
|
|
||||||
if(
|
if(
|
||||||
isset($_GET["ac"]) &&
|
isset($_GET["ac"]) &&
|
||||||
|
|||||||
+223
-1006
File diff suppressed because it is too large.
Load diff
+1047
-147
File diff suppressed because it is too large.
Load diff
+295
-43
@@ -10,8 +10,9 @@ class google{
|
|||||||
include "lib/backend.php";
|
include "lib/backend.php";
|
||||||
$this->backend = new backend("google");
|
$this->backend = new backend("google");
|
||||||
|
|
||||||
$this->proc = curl_multi_init();
|
$this->urls_to_resolve = [];
|
||||||
$this->handles = [];
|
|
||||||
|
$this->init_multi_curl();
|
||||||
}
|
}
|
||||||
|
|
||||||
public function getfilters($page){
|
public function getfilters($page){
|
||||||
@@ -617,23 +618,57 @@ class google{
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
private function redirect_add_url($url){
|
private function init_multi_curl(){
|
||||||
|
|
||||||
|
$this->proc = curl_multi_init();
|
||||||
|
$this->handles = [];
|
||||||
|
}
|
||||||
|
|
||||||
|
private function redirect_add_url($url, $replace = null){
|
||||||
|
|
||||||
if(
|
if(
|
||||||
preg_match(
|
preg_match(
|
||||||
'/^\/goto\?url=([^&]+)/',
|
'/^\/goto\?url=([^&]+)/',
|
||||||
$url,
|
$url,
|
||||||
$slug
|
$slug
|
||||||
) === 0
|
) === 1
|
||||||
){
|
){
|
||||||
|
|
||||||
// not a google redirect
|
// normal goto
|
||||||
|
$this->urls_to_resolve[] = [
|
||||||
|
"replace" => $url,
|
||||||
|
"slug" => $slug[1],
|
||||||
|
"type" => "goto"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
elseif(
|
||||||
|
preg_match(
|
||||||
|
'/https?:\/\/share\.google\/([^"]+)+/',
|
||||||
|
$url,
|
||||||
|
$link_regex
|
||||||
|
) === 1
|
||||||
|
){
|
||||||
|
|
||||||
|
// share.google
|
||||||
|
$this->urls_to_resolve[] = [
|
||||||
|
"replace" => $replace,
|
||||||
|
"slug" => $link_regex[1],
|
||||||
|
"type" => "share"
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
// rest is ignored, since its not a redirect
|
||||||
|
|
||||||
|
// always return url
|
||||||
return $url;
|
return $url;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function urlfuck_goto($url, $slug){
|
||||||
|
|
||||||
$curlproc = curl_init();
|
$curlproc = curl_init();
|
||||||
|
|
||||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
|
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
|
||||||
|
|
||||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||||
@@ -658,7 +693,7 @@ class google{
|
|||||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||||
|
|
||||||
//$this->backend->assign_proxy($curlproc, $proxy);
|
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||||
|
|
||||||
curl_multi_add_handle($this->proc, $curlproc);
|
curl_multi_add_handle($this->proc, $curlproc);
|
||||||
$this->handles[] = [
|
$this->handles[] = [
|
||||||
@@ -669,7 +704,92 @@ class google{
|
|||||||
return $url;
|
return $url;
|
||||||
}
|
}
|
||||||
|
|
||||||
private function resolve_urls(&$scraped_data){
|
private function urlfuck_shareservice($url, $slug){
|
||||||
|
|
||||||
|
$curlproc = curl_init();
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||||
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||||
|
["User-Agent: " . config::USER_AGENT,
|
||||||
|
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language: en-US,en;q=0.5",
|
||||||
|
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||||
|
"DNT: 1",
|
||||||
|
"Sec-GPC: 1",
|
||||||
|
"Connection: keep-alive",
|
||||||
|
"Upgrade-Insecure-Requests: 1",
|
||||||
|
"Sec-Fetch-Dest: document",
|
||||||
|
"Sec-Fetch-Mode: navigate",
|
||||||
|
"Sec-Fetch-Site: none",
|
||||||
|
"Sec-Fetch-User: ?1",
|
||||||
|
"Priority: u=0, i"]
|
||||||
|
);
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||||
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||||
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||||
|
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||||
|
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||||
|
|
||||||
|
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||||
|
|
||||||
|
curl_multi_add_handle($this->proc, $curlproc);
|
||||||
|
$this->handles[] = [
|
||||||
|
"replacement" => $url,
|
||||||
|
"handle" => $curlproc
|
||||||
|
];
|
||||||
|
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function urlfuck_share($url, $slug){
|
||||||
|
|
||||||
|
$curlproc = curl_init();
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||||
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||||
|
["User-Agent: " . config::USER_AGENT,
|
||||||
|
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||||
|
"Accept-Language: en-US,en;q=0.5",
|
||||||
|
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||||
|
"DNT: 1",
|
||||||
|
"Sec-GPC: 1",
|
||||||
|
"Connection: keep-alive",
|
||||||
|
"Upgrade-Insecure-Requests: 1",
|
||||||
|
"Sec-Fetch-Dest: document",
|
||||||
|
"Sec-Fetch-Mode: navigate",
|
||||||
|
"Sec-Fetch-Site: none",
|
||||||
|
"Sec-Fetch-User: ?1",
|
||||||
|
"Priority: u=0, i"]
|
||||||
|
);
|
||||||
|
|
||||||
|
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||||
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||||
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||||
|
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||||
|
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||||
|
|
||||||
|
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||||
|
|
||||||
|
curl_multi_add_handle($this->proc, $curlproc);
|
||||||
|
$this->handles[] = [
|
||||||
|
"replacement" => $url,
|
||||||
|
"handle" => $curlproc
|
||||||
|
];
|
||||||
|
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function resolve_gotos(&$scraped_data){
|
||||||
|
|
||||||
|
foreach($this->urls_to_resolve as $item){
|
||||||
|
|
||||||
|
$this->urlfuck_goto($item["replace"], $item["slug"]);
|
||||||
|
}
|
||||||
|
|
||||||
do{
|
do{
|
||||||
do{
|
do{
|
||||||
@@ -706,20 +826,163 @@ class google{
|
|||||||
|
|
||||||
if(count($a) === 0){
|
if(count($a) === 0){
|
||||||
|
|
||||||
throw new Exception("Failed to find link");
|
// failed to find link, will serve a resolver target as fallback
|
||||||
}
|
$fallback = true;
|
||||||
|
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||||
|
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$fallback = false;
|
||||||
$link =
|
$link =
|
||||||
$this->fuckhtml
|
$this->fuckhtml
|
||||||
->getTextContent(
|
->getTextContent(
|
||||||
$a[0]["attributes"]["href"]
|
$a[0]["attributes"]["href"]
|
||||||
);
|
);
|
||||||
|
}
|
||||||
|
|
||||||
if(
|
if(
|
||||||
|
(
|
||||||
|
$fallback === false &&
|
||||||
preg_match(
|
preg_match(
|
||||||
'/^(https?:|)\/\//',
|
'/^(https?:|)\/\//',
|
||||||
$link
|
$link
|
||||||
)
|
)
|
||||||
|
) ||
|
||||||
|
$fallback === true
|
||||||
|
){
|
||||||
|
|
||||||
|
$keys[] = $this->handles[$i]["replacement"];
|
||||||
|
$replacements[] = $link;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->recursive_replace($keys, $replacements, $scraped_data);
|
||||||
|
|
||||||
|
curl_multi_close($this->proc);
|
||||||
|
}
|
||||||
|
|
||||||
|
private function resolve_share(&$scraped_data){
|
||||||
|
|
||||||
|
foreach($this->urls_to_resolve as $item){
|
||||||
|
|
||||||
|
$this->urlfuck_shareservice($item["replace"], $item["slug"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
do{
|
||||||
|
do{
|
||||||
|
$status = curl_multi_exec($this->proc, $active);
|
||||||
|
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||||
|
|
||||||
|
if($active){
|
||||||
|
curl_multi_select($this->proc);
|
||||||
|
}
|
||||||
|
|
||||||
|
}while($active && $status === CURLM_OK);
|
||||||
|
|
||||||
|
//
|
||||||
|
// if we reach this, we're done downloading garbage
|
||||||
|
//
|
||||||
|
|
||||||
|
// get the share links
|
||||||
|
$this->urls_to_resolve = [];
|
||||||
|
|
||||||
|
for($i=0; $i<count($this->handles); $i++){
|
||||||
|
|
||||||
|
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||||
|
|
||||||
|
if(
|
||||||
|
preg_match(
|
||||||
|
'/https?:\/\/share\.google\/(?:[^"]+)/',
|
||||||
|
$html,
|
||||||
|
$link_regex
|
||||||
|
) !== 1
|
||||||
|
){
|
||||||
|
|
||||||
|
// failed to find link, will serve a resolver target as fallback
|
||||||
|
$fallback = true;
|
||||||
|
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||||
|
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$fallback = false;
|
||||||
|
$link = $link_regex[0];
|
||||||
|
}
|
||||||
|
|
||||||
|
if(
|
||||||
|
(
|
||||||
|
$fallback === false &&
|
||||||
|
preg_match(
|
||||||
|
'/^(https?:|)\/\//',
|
||||||
|
$link
|
||||||
|
)
|
||||||
|
) ||
|
||||||
|
$fallback === true
|
||||||
|
){
|
||||||
|
|
||||||
|
$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolve shareable urls
|
||||||
|
$this->init_multi_curl();
|
||||||
|
|
||||||
|
foreach($this->urls_to_resolve as $item){
|
||||||
|
|
||||||
|
$this->urlfuck_share($item["replace"], $item["slug"]);
|
||||||
|
}
|
||||||
|
|
||||||
|
do{
|
||||||
|
do{
|
||||||
|
$status = curl_multi_exec($this->proc, $active);
|
||||||
|
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||||
|
|
||||||
|
if($active){
|
||||||
|
curl_multi_select($this->proc);
|
||||||
|
}
|
||||||
|
|
||||||
|
}while($active && $status === CURLM_OK);
|
||||||
|
|
||||||
|
// parse downloaded garbage
|
||||||
|
$keys = [];
|
||||||
|
$replacements = [];
|
||||||
|
|
||||||
|
for($i=0; $i<count($this->handles); $i++){
|
||||||
|
|
||||||
|
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||||
|
|
||||||
|
$this->fuckhtml->load($html);
|
||||||
|
|
||||||
|
$a =
|
||||||
|
$this->fuckhtml
|
||||||
|
->getElementsByTagName(
|
||||||
|
"a"
|
||||||
|
);
|
||||||
|
|
||||||
|
if(count($a) === 0){
|
||||||
|
|
||||||
|
// failed to find link, will serve a resolver target as fallback
|
||||||
|
$fallback = true;
|
||||||
|
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||||
|
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$fallback = false;
|
||||||
|
$link =
|
||||||
|
$this->fuckhtml
|
||||||
|
->getTextContent(
|
||||||
|
$a[0]["attributes"]["href"]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
if(
|
||||||
|
(
|
||||||
|
$fallback === false &&
|
||||||
|
preg_match(
|
||||||
|
'/^(https?:|)\/\//',
|
||||||
|
$link
|
||||||
|
)
|
||||||
|
) ||
|
||||||
|
$fallback === true
|
||||||
){
|
){
|
||||||
|
|
||||||
$keys[] = $this->handles[$i]["replacement"];
|
$keys[] = $this->handles[$i]["replacement"];
|
||||||
@@ -785,22 +1048,6 @@ class google{
|
|||||||
|
|
||||||
throw new Exception("Search term is empty!");
|
throw new Exception("Search term is empty!");
|
||||||
}
|
}
|
||||||
/*
|
|
||||||
$session = $this->backend->session_get();
|
|
||||||
|
|
||||||
if(config::FPLAY_ENABLE_API === false){
|
|
||||||
|
|
||||||
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
|
|
||||||
}
|
|
||||||
|
|
||||||
if($session === false){
|
|
||||||
|
|
||||||
throw new Exception("Session database is empty");
|
|
||||||
}
|
|
||||||
|
|
||||||
$proxy = &$session["proxy"];
|
|
||||||
$headers = &$session["headers"];
|
|
||||||
*/
|
|
||||||
|
|
||||||
$proxy = $this->backend->get_ip();
|
$proxy = $this->backend->get_ip();
|
||||||
|
|
||||||
@@ -817,7 +1064,7 @@ class google{
|
|||||||
// country
|
// country
|
||||||
if($get["country"] != "any"){
|
if($get["country"] != "any"){
|
||||||
|
|
||||||
$params["gl"] = $country;
|
$params["gl"] = $get["country"];
|
||||||
}
|
}
|
||||||
|
|
||||||
// nsfw
|
// nsfw
|
||||||
@@ -826,15 +1073,15 @@ class google{
|
|||||||
// language
|
// language
|
||||||
if($get["lang"] != "any"){
|
if($get["lang"] != "any"){
|
||||||
|
|
||||||
$params["lr"] = "lang_" . $lang;
|
$params["lr"] = "lang_" . $get["lang"];
|
||||||
}
|
}
|
||||||
|
|
||||||
// generate tbs
|
// generate tbs
|
||||||
$tbs = [];
|
$tbs = [];
|
||||||
|
|
||||||
// get date
|
// get date
|
||||||
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $older);
|
$get["older"] = $get["older"] === false ? null : date("m/d/Y", $get["older"]);
|
||||||
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer);
|
$get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $get["newer"]);
|
||||||
|
|
||||||
if(
|
if(
|
||||||
$get["older"] !== null ||
|
$get["older"] !== null ||
|
||||||
@@ -1466,7 +1713,7 @@ class google{
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->resolve_urls($out);
|
$this->resolve_share($out);
|
||||||
|
|
||||||
return $out;
|
return $out;
|
||||||
}
|
}
|
||||||
@@ -1478,15 +1725,15 @@ class google{
|
|||||||
// generate parameters
|
// generate parameters
|
||||||
if($get["npt"]){
|
if($get["npt"]){
|
||||||
|
|
||||||
[$params, $proxy] =
|
[$params_json, $proxy] =
|
||||||
$this->backend->get(
|
$this->backend->get(
|
||||||
$get["npt"],
|
$get["npt"],
|
||||||
"images"
|
"images"
|
||||||
);
|
);
|
||||||
|
|
||||||
$params = json_decode($params, true);
|
$params_json = json_decode($params_json, true);
|
||||||
$container = $params["c"];
|
$container = &$params_json["c"];
|
||||||
unset($params["c"]);
|
$params = &$params_json["p"];
|
||||||
}else{
|
}else{
|
||||||
|
|
||||||
$search = $get["s"];
|
$search = $get["s"];
|
||||||
@@ -1505,6 +1752,7 @@ class google{
|
|||||||
$type = $get["type"];
|
$type = $get["type"];
|
||||||
$format = $get["format"];
|
$format = $get["format"];
|
||||||
$rights = $get["rights"];
|
$rights = $get["rights"];
|
||||||
|
$container = null;
|
||||||
|
|
||||||
$params = [
|
$params = [
|
||||||
"q" => $search,
|
"q" => $search,
|
||||||
@@ -1593,17 +1841,18 @@ class google{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
//$html = file_get_contents("scraper/google.html");
|
|
||||||
|
|
||||||
$data = $this->get(
|
$data = $this->get(
|
||||||
$proxy,
|
$proxy,
|
||||||
"https://www.google.com/search",
|
"https://www.google.com/search",
|
||||||
$params
|
$params,
|
||||||
|
$container
|
||||||
);
|
);
|
||||||
|
|
||||||
$html = &$data["data"];
|
$html = &$data["data"];
|
||||||
$container = &$data["container"];
|
$container = &$data["container"];
|
||||||
|
|
||||||
|
//$html = file_get_contents("scraper/google.html");
|
||||||
|
|
||||||
$this->fuckhtml->load($html);
|
$this->fuckhtml->load($html);
|
||||||
|
|
||||||
// parse all <style> tags
|
// parse all <style> tags
|
||||||
@@ -1701,13 +1950,16 @@ class google{
|
|||||||
$out["npt"] =
|
$out["npt"] =
|
||||||
$this->backend
|
$this->backend
|
||||||
->store(
|
->store(
|
||||||
json_encode($params),
|
json_encode([
|
||||||
|
"p" => $params,
|
||||||
|
"c" => $container
|
||||||
|
]),
|
||||||
"image",
|
"image",
|
||||||
$proxy
|
$proxy
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->resolve_urls($out);
|
$this->resolve_gotos($out);
|
||||||
|
|
||||||
return $out;
|
return $out;
|
||||||
}
|
}
|
||||||
@@ -2115,7 +2367,7 @@ class google{
|
|||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->resolve_urls($out);
|
$this->resolve_share($out);
|
||||||
|
|
||||||
return $out;
|
return $out;
|
||||||
}
|
}
|
||||||
@@ -2474,14 +2726,14 @@ class google{
|
|||||||
"date" => $date,
|
"date" => $date,
|
||||||
"thumb" => $thumb,
|
"thumb" => $thumb,
|
||||||
"url" =>
|
"url" =>
|
||||||
$this->unshiturl(
|
$this->redirect_add_url(
|
||||||
$a["attributes"]
|
$a["attributes"]
|
||||||
["href"]
|
["href"]
|
||||||
)
|
)
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->resolve_urls($out);
|
$this->resolve_share($out);
|
||||||
|
|
||||||
return $out;
|
return $out;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -567,6 +567,8 @@ class google_api{
|
|||||||
|
|
||||||
foreach($json["items"] as $result){
|
foreach($json["items"] as $result){
|
||||||
|
|
||||||
|
$date = null;
|
||||||
|
|
||||||
//
|
//
|
||||||
// probe for thumbnail
|
// probe for thumbnail
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -725,6 +725,14 @@ class google_cse{
|
|||||||
// calculate rounded ratio
|
// calculate rounded ratio
|
||||||
if($found_size){
|
if($found_size){
|
||||||
|
|
||||||
|
if(
|
||||||
|
$width === 0 ||
|
||||||
|
$height === 0
|
||||||
|
){
|
||||||
|
|
||||||
|
$thumb["ratio"] = "1:1";
|
||||||
|
}else{
|
||||||
|
|
||||||
$aspect_ratio = $width / $height;
|
$aspect_ratio = $width / $height;
|
||||||
|
|
||||||
if($aspect_ratio >= 1.5){
|
if($aspect_ratio >= 1.5){
|
||||||
@@ -740,6 +748,7 @@ class google_cse{
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$out["web"][] = [
|
$out["web"][] = [
|
||||||
"title" => rtrim($result["titleNoFormatting"], " ."),
|
"title" => rtrim($result["titleNoFormatting"], " ."),
|
||||||
|
|||||||
+73
-22
@@ -261,9 +261,10 @@ class naver{
|
|||||||
|
|
||||||
$filters = [
|
$filters = [
|
||||||
"nso" => "",
|
"nso" => "",
|
||||||
|
"page" => 2, // page 2 is repeat of page 1
|
||||||
"query" => $search,
|
"query" => $search,
|
||||||
"sm" => "tab_pge",
|
"sm" => "tab_pge",
|
||||||
"where" => "web",
|
"ssc" => "tab.ur.all",
|
||||||
"start" => 1 // increment by number of results each time (16??)
|
"start" => 1 // increment by number of results each time (16??)
|
||||||
];
|
];
|
||||||
|
|
||||||
@@ -304,13 +305,13 @@ class naver{
|
|||||||
|
|
||||||
$this->fuckhtml->load($html);
|
$this->fuckhtml->load($html);
|
||||||
|
|
||||||
$results =
|
$jsons =
|
||||||
preg_split(
|
preg_split(
|
||||||
'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
|
'/entry\.bootstrap\(document\.getElementById\("[a-f0-9-r]+"\), ?/',
|
||||||
$html
|
$html
|
||||||
);
|
);
|
||||||
|
|
||||||
if(count($results) !== 2){
|
if(count($jsons) <= 1){
|
||||||
|
|
||||||
// this is thrown when no results are found
|
// this is thrown when no results are found
|
||||||
|
|
||||||
@@ -342,32 +343,29 @@ class naver{
|
|||||||
//throw new Exception("Failed to grep results entrypoint");
|
//throw new Exception("Failed to grep results entrypoint");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
foreach($jsons as $json){
|
||||||
|
|
||||||
$json =
|
$json =
|
||||||
json_decode(
|
json_decode(
|
||||||
$this->fuckhtml
|
$this->fuckhtml
|
||||||
->extract_json(
|
->extract_json(
|
||||||
$results[1]
|
$json
|
||||||
),
|
),
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
|
|
||||||
if(!isset($json["body"]["props"]["children"][0]["props"]["children"])){
|
if($json === null){
|
||||||
|
|
||||||
throw new Exception("Failed to access nested children");
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach($json["body"]["props"]["children"][0]["props"]["children"] as $result){
|
|
||||||
|
|
||||||
if(
|
|
||||||
!isset($result["templateId"]) ||
|
|
||||||
$result["templateId"] != "webItem"
|
|
||||||
){
|
|
||||||
|
|
||||||
// should not happen
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$result = $result["props"];
|
if(!isset($json["data"]["state"]["items"])){
|
||||||
|
|
||||||
|
//throw new Exception("Failed to access nested children");
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
foreach($json["data"]["state"]["items"] as $result){
|
||||||
|
|
||||||
// get sublinks
|
// get sublinks
|
||||||
$sublinks = [];
|
$sublinks = [];
|
||||||
@@ -429,12 +427,19 @@ class naver{
|
|||||||
// get date
|
// get date
|
||||||
$time = null;
|
$time = null;
|
||||||
|
|
||||||
if(isset($result["bodyPrefixes"][0]["text"])){
|
if(isset($result["profile"]["createdDate"])){
|
||||||
|
|
||||||
|
$time = $result["profile"]["createdDate"];
|
||||||
|
}elseif(isset($result["sourceProfile"]["createdDate"])){
|
||||||
|
|
||||||
|
$time = $result["sourceProfile"]["createdDate"];
|
||||||
|
}
|
||||||
|
|
||||||
|
if($time !== null){
|
||||||
$date =
|
$date =
|
||||||
strtotime(
|
strtotime(
|
||||||
substr(
|
substr(
|
||||||
$result["bodyPrefixes"][0]["text"],
|
$time,
|
||||||
-1
|
-1
|
||||||
)
|
)
|
||||||
);
|
);
|
||||||
@@ -442,13 +447,58 @@ class naver{
|
|||||||
if($date !== false){
|
if($date !== false){
|
||||||
|
|
||||||
$time = $date;
|
$time = $date;
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$time = null;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// get title
|
||||||
|
if(isset($result["title"])){
|
||||||
|
|
||||||
|
$title = $result["title"];
|
||||||
|
}elseif(isset($result["questionTitle"])){
|
||||||
|
|
||||||
|
$title = $result["questionTitle"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// get url
|
||||||
|
if(isset($result["titleHref"])){
|
||||||
|
|
||||||
|
$href = $result["titleHref"];
|
||||||
|
}elseif(isset($result["href"])){
|
||||||
|
|
||||||
|
$href = $result["href"];
|
||||||
|
}elseif(isset($result["answerHref"])){
|
||||||
|
|
||||||
|
$href = $result["answerHref"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
// get description
|
||||||
|
if(isset($result["bodyText"])){
|
||||||
|
|
||||||
|
$description = $result["bodyText"];
|
||||||
|
}elseif(isset($result["content"])){
|
||||||
|
|
||||||
|
$description = $result["content"];
|
||||||
|
}elseif(isset($result["questionContent"])){
|
||||||
|
|
||||||
|
$description = $result["questionContent"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$description = null;
|
||||||
|
}
|
||||||
|
|
||||||
$out["web"][] = [
|
$out["web"][] = [
|
||||||
"title" => $this->decode_html($result["title"]),
|
"title" => $this->decode_html($title),
|
||||||
"description" => $this->decode_html($result["bodyText"]),
|
"description" => $this->decode_html($description),
|
||||||
"url" => $result["href"],
|
"url" => $href,
|
||||||
"date" => $time,
|
"date" => $time,
|
||||||
"type" => "web",
|
"type" => "web",
|
||||||
"thumb" => $thumb,
|
"thumb" => $thumb,
|
||||||
@@ -456,6 +506,7 @@ class naver{
|
|||||||
"table" => $table
|
"table" => $table
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// get next page
|
// get next page
|
||||||
$npt =
|
$npt =
|
||||||
|
|||||||
+158
-16
@@ -545,10 +545,12 @@ class startpage{
|
|||||||
switch($category["display_type"]){
|
switch($category["display_type"]){
|
||||||
|
|
||||||
case "web-google":
|
case "web-google":
|
||||||
|
case "web-bing":
|
||||||
foreach($category["results"] as $result){
|
foreach($category["results"] as $result){
|
||||||
|
|
||||||
$sublinks = [];
|
$sublinks = [];
|
||||||
|
|
||||||
|
if(isset($result["siteLinks"])){
|
||||||
foreach($result["siteLinks"] as $sublink){
|
foreach($result["siteLinks"] as $sublink){
|
||||||
|
|
||||||
$sublinks[] = [
|
$sublinks[] = [
|
||||||
@@ -557,6 +559,7 @@ class startpage{
|
|||||||
"url" => $sublink["clickUrl"]
|
"url" => $sublink["clickUrl"]
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
$description =
|
$description =
|
||||||
explode(
|
explode(
|
||||||
@@ -596,6 +599,7 @@ class startpage{
|
|||||||
|
|
||||||
$out["web"][] = [
|
$out["web"][] = [
|
||||||
"title" =>
|
"title" =>
|
||||||
|
$this->remove_penguins(
|
||||||
$this->titledots(
|
$this->titledots(
|
||||||
html_entity_decode(
|
html_entity_decode(
|
||||||
$this->fuckhtml
|
$this->fuckhtml
|
||||||
@@ -603,8 +607,9 @@ class startpage{
|
|||||||
$result["title"]
|
$result["title"]
|
||||||
)
|
)
|
||||||
)
|
)
|
||||||
|
)
|
||||||
),
|
),
|
||||||
"description" => $description,
|
"description" => $this->remove_penguins($description),
|
||||||
"url" => $result["clickUrl"],
|
"url" => $result["clickUrl"],
|
||||||
"date" => $date,
|
"date" => $date,
|
||||||
"type" => "web",
|
"type" => "web",
|
||||||
@@ -619,8 +624,43 @@ class startpage{
|
|||||||
break;
|
break;
|
||||||
|
|
||||||
case "images-qi-top":
|
case "images-qi-top":
|
||||||
|
case "images-bing-top":
|
||||||
foreach($category["results"] as $result){
|
foreach($category["results"] as $result){
|
||||||
|
|
||||||
|
if(isset($result["mdThumbnailWidth"])){
|
||||||
|
|
||||||
|
$thumb_width = $result["mdThumbnailWidth"];
|
||||||
|
}elseif(isset($result["thumbnailWidth"])){
|
||||||
|
|
||||||
|
$thumb_width = $result["thumbnailWidth"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$thumb_width = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(isset($result["mdThumbnailHeight"])){
|
||||||
|
|
||||||
|
$thumb_height = $result["mdThumbnailHeight"];
|
||||||
|
}elseif(isset($result["thumbnailHeight"])){
|
||||||
|
|
||||||
|
$thumb_height = $result["thumbnailHeight"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$thumb_height = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
if(isset($result["mdThumbnailUrl"])){
|
||||||
|
|
||||||
|
$thumb_source = $result["mdThumbnailUrl"];
|
||||||
|
}elseif(isset($result["thumbnailUrl"])){
|
||||||
|
|
||||||
|
$thumb_source = $result["thumbnailUrl"];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
// should not happen
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
$out["image"][] = [
|
$out["image"][] = [
|
||||||
"title" =>
|
"title" =>
|
||||||
$this->titledots(
|
$this->titledots(
|
||||||
@@ -638,19 +678,17 @@ class startpage{
|
|||||||
"height" => (int)$result["height"]
|
"height" => (int)$result["height"]
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"url" => $this->unshitimage($result["mdThumbnailUrl"]),
|
"url" => $this->unshitimage($thumb_source),
|
||||||
"width" => (int)$result["mdThumbnailWidth"],
|
"width" => (int)$thumb_width,
|
||||||
"height" => (int)$result["mdThumbnailHeight"]
|
"height" => (int)$thumb_height
|
||||||
]
|
]
|
||||||
],
|
],
|
||||||
"url" =>
|
"url" => $result["altClickUrl"]
|
||||||
$result["altClickUrl"]
|
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
case "spellsuggest-google":
|
case "spellsuggest-google":
|
||||||
|
|
||||||
if(isset($category["results"][0]["query"])){
|
if(isset($category["results"][0]["query"])){
|
||||||
|
|
||||||
$out["spelling"] =
|
$out["spelling"] =
|
||||||
@@ -662,6 +700,60 @@ class startpage{
|
|||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case "spelladjust-bing":
|
||||||
|
if(isset($category["results"][0]["alterationDisplayQuery"])){
|
||||||
|
|
||||||
|
$out["spelling"] =
|
||||||
|
[
|
||||||
|
"type" => "including",
|
||||||
|
"using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]),
|
||||||
|
"correction" => $category["results"][0]["originalQuery"]
|
||||||
|
];
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case "aylf-bing-top":
|
||||||
|
if(isset($category["results"][0]["query"])){
|
||||||
|
|
||||||
|
foreach($category["results"] as $ddm){
|
||||||
|
|
||||||
|
$out["related"][] = $ddm["query"];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
|
case "video-bing-top":
|
||||||
|
if(isset($category["results"][0]["clickUrl"])){
|
||||||
|
|
||||||
|
foreach($category["results"] as $video){
|
||||||
|
|
||||||
|
if(isset($video["thumbnailUrl"])){
|
||||||
|
|
||||||
|
$thumb = [
|
||||||
|
"ratio" => "16:9",
|
||||||
|
"url" => $this->unshitimage($video["thumbnailUrl"])
|
||||||
|
];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$thumb = [
|
||||||
|
"ratio" => null,
|
||||||
|
"url" => null
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
$out["video"][] = [
|
||||||
|
"title" => $video["title"],
|
||||||
|
"description" => $this->remove_penguins($video["description"]),
|
||||||
|
"date" => isset($video["date"]) ? strtotime($video["date"]) : null,
|
||||||
|
"duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null,
|
||||||
|
"views" => $video["viewCount"],
|
||||||
|
"thumb" => $thumb,
|
||||||
|
"url" => $video["clickUrl"]
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
|
|
||||||
case "dictionary-qi":
|
case "dictionary-qi":
|
||||||
foreach($category["results"] as $result){
|
foreach($category["results"] as $result){
|
||||||
|
|
||||||
@@ -732,6 +824,36 @@ class startpage{
|
|||||||
$out["answer"][] = $answer;
|
$out["answer"][] = $answer;
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
|
||||||
|
case "news-bing-top":
|
||||||
|
if(isset($category["results"][0]["clickUrl"])){
|
||||||
|
|
||||||
|
foreach($category["results"] as $news){
|
||||||
|
|
||||||
|
if(isset($news["thumbnailUrl"])){
|
||||||
|
|
||||||
|
$thumb = [
|
||||||
|
"ratio" => "16:9",
|
||||||
|
"url" => $news["thumbnailUrl"]
|
||||||
|
];
|
||||||
|
}else{
|
||||||
|
|
||||||
|
$thumb = [
|
||||||
|
"ratio" => null,
|
||||||
|
"url" => null
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
|
$out["news"][] = [
|
||||||
|
"title" => $this->remove_penguins($news["title"]),
|
||||||
|
"description" => $news["description"],
|
||||||
|
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
|
||||||
|
"thumb" => $thumb,
|
||||||
|
"url" => $news["clickUrl"],
|
||||||
|
];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1246,14 +1368,10 @@ class startpage{
|
|||||||
// get npt
|
// get npt
|
||||||
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
|
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
|
||||||
|
|
||||||
|
//print_r($json);
|
||||||
|
|
||||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||||
|
|
||||||
if($category["display_type"] != "news-bing"){
|
|
||||||
|
|
||||||
// unsupported category
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
foreach($category["results"] as $news){
|
foreach($category["results"] as $news){
|
||||||
|
|
||||||
if(
|
if(
|
||||||
@@ -1276,10 +1394,10 @@ class startpage{
|
|||||||
$out["news"][] = [
|
$out["news"][] = [
|
||||||
"title" => $this->titledots($this->remove_penguins($news["title"])),
|
"title" => $this->titledots($this->remove_penguins($news["title"])),
|
||||||
"author" => $news["source"],
|
"author" => $news["source"],
|
||||||
"description" => $this->titledots($this->remove_penguins($news["description"])),
|
"description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null,
|
||||||
"date" => (int)substr((string)$news["date"], 0, -3),
|
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
|
||||||
"thumb" => $thumb,
|
"thumb" => $thumb,
|
||||||
"url" => $news["clickUrl"]
|
"url" => $this->remove_tracking_link($news["clickUrl"])
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1327,6 +1445,30 @@ class startpage{
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private function remove_tracking_link($url){
|
||||||
|
|
||||||
|
// https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog
|
||||||
|
$query = parse_url($url);
|
||||||
|
|
||||||
|
if(
|
||||||
|
!isset($query["host"]) ||
|
||||||
|
$query["host"] != "api.airfind.com" ||
|
||||||
|
!isset($query["query"])
|
||||||
|
){
|
||||||
|
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
|
||||||
|
parse_str($query["query"], $res);
|
||||||
|
|
||||||
|
if(isset($res["redirectURL"])){
|
||||||
|
|
||||||
|
return $res["redirectURL"];
|
||||||
|
}
|
||||||
|
|
||||||
|
return $url;
|
||||||
|
}
|
||||||
|
|
||||||
private function unshitimage($url){
|
private function unshitimage($url){
|
||||||
|
|
||||||
$query = parse_url($url, PHP_URL_QUERY);
|
$query = parse_url($url, PHP_URL_QUERY);
|
||||||
|
|||||||
+19
-58
@@ -205,13 +205,6 @@ class yep{
|
|||||||
"zh-tw" => "Chinese (Traditional)",
|
"zh-tw" => "Chinese (Traditional)",
|
||||||
"zu" => "Zulu"
|
"zu" => "Zulu"
|
||||||
]
|
]
|
||||||
],
|
|
||||||
"nsfw" => [
|
|
||||||
"display" => "NSFW",
|
|
||||||
"option" => [
|
|
||||||
"yes" => "Yes",
|
|
||||||
"no" => "No"
|
|
||||||
]
|
|
||||||
]
|
]
|
||||||
];
|
];
|
||||||
}
|
}
|
||||||
@@ -317,20 +310,23 @@ class yep{
|
|||||||
|
|
||||||
// parse filters
|
// parse filters
|
||||||
$filters = [
|
$filters = [
|
||||||
"limit" => 100, // wwwwwwwwwwwwwww
|
"q" => $search,
|
||||||
"query" => $search,
|
"limit" => 10 // no fun allowed
|
||||||
];
|
];
|
||||||
|
|
||||||
if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; }
|
if($get["lang"] != "any"){
|
||||||
if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
|
|
||||||
|
$filters["language"] = $get["lang"];
|
||||||
|
$filters["lang"] = $get["lang"];
|
||||||
|
}
|
||||||
|
|
||||||
try{
|
try{
|
||||||
|
|
||||||
// https://api.yep.com/search?limit=20&query=asmr
|
// https://yep.com/web/v1/search?q=asmr&limit=10
|
||||||
$json =
|
$json =
|
||||||
$this->get(
|
$this->get(
|
||||||
$this->backend->get_ip(),
|
$this->backend->get_ip(),
|
||||||
"https://api.yep.com/search",
|
"https://yep.com/web/v1/search",
|
||||||
$filters
|
$filters
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -343,59 +339,27 @@ class yep{
|
|||||||
|
|
||||||
$json = json_decode($json, true);
|
$json = json_decode($json, true);
|
||||||
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
|
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
|
||||||
|
|
||||||
if($json === null){
|
if($json === null){
|
||||||
|
|
||||||
throw new Exception("Failed to decode JSON");
|
throw new Exception("Failed to decode JSON");
|
||||||
}
|
}
|
||||||
|
|
||||||
if(isset($json[1]["correction"])){
|
if(isset($json["code"])){
|
||||||
|
|
||||||
$out["spelling"] = [
|
if(isset($json["reason"])){
|
||||||
"type" => "not_many",
|
|
||||||
"using" => $search,
|
throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
|
||||||
"correction" => $json[1]["correction"][1]
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if(isset($json[1]["results"])){
|
throw new Exception("Yep returned an unknown error");
|
||||||
foreach($json[1]["results"] as $item){
|
|
||||||
|
|
||||||
switch(strtolower($item["type"])){
|
|
||||||
|
|
||||||
case "organic":
|
|
||||||
$sublinks = [];
|
|
||||||
|
|
||||||
if(isset($item["sitelinks"]["full"])){
|
|
||||||
|
|
||||||
foreach($item["sitelinks"]["full"] as $link){
|
|
||||||
|
|
||||||
$sublinks[] = [
|
|
||||||
"title" => $link["title"],
|
|
||||||
"date" => null,
|
|
||||||
"description" =>
|
|
||||||
$this->titledots(
|
|
||||||
strip_tags(
|
|
||||||
html_entity_decode(
|
|
||||||
$link["snippet"]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
),
|
|
||||||
"url" => $link["url"]
|
|
||||||
];
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if(isset($json["results"])){
|
||||||
|
foreach($json["results"] as $item){
|
||||||
|
|
||||||
$out["web"][] = [
|
$out["web"][] = [
|
||||||
"title" => $item["title"],
|
"title" => $item["title"],
|
||||||
"description" =>
|
"description" => str_replace("\n", " ", $item["description"]),
|
||||||
$this->titledots(
|
|
||||||
strip_tags(
|
|
||||||
html_entity_decode(
|
|
||||||
$item["snippet"]
|
|
||||||
)
|
|
||||||
)
|
|
||||||
),
|
|
||||||
"url" => $item["url"],
|
"url" => $item["url"],
|
||||||
"date" => null,
|
"date" => null,
|
||||||
"type" => "web",
|
"type" => "web",
|
||||||
@@ -403,11 +367,9 @@ class yep{
|
|||||||
"url" => null,
|
"url" => null,
|
||||||
"ratio" => null
|
"ratio" => null
|
||||||
],
|
],
|
||||||
"sublink" => $sublinks,
|
"sublink" => [],
|
||||||
"table" => []
|
"table" => []
|
||||||
];
|
];
|
||||||
break;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -445,7 +407,6 @@ class yep{
|
|||||||
"limit" => 99 // for some reason they dont show anything sometimes at 100 results
|
"limit" => 99 // for some reason they dont show anything sometimes at 100 results
|
||||||
];
|
];
|
||||||
|
|
||||||
if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
|
|
||||||
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
|
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
|
||||||
|
|
||||||
// add api key
|
// add api key
|
||||||
|
|||||||
+10
-1
@@ -678,6 +678,15 @@ class yt{
|
|||||||
$video["longBylineText"]
|
$video["longBylineText"]
|
||||||
),
|
),
|
||||||
"url" =>
|
"url" =>
|
||||||
|
isset(
|
||||||
|
$video["ownerText"]
|
||||||
|
["runs"]
|
||||||
|
[0]
|
||||||
|
["navigationEndpoint"]
|
||||||
|
["commandMetadata"]
|
||||||
|
["webCommandMetadata"]
|
||||||
|
["url"]
|
||||||
|
) ?
|
||||||
"https://www.youtube.com" .
|
"https://www.youtube.com" .
|
||||||
$video["ownerText"]
|
$video["ownerText"]
|
||||||
["runs"]
|
["runs"]
|
||||||
@@ -685,7 +694,7 @@ class yt{
|
|||||||
["navigationEndpoint"]
|
["navigationEndpoint"]
|
||||||
["commandMetadata"]
|
["commandMetadata"]
|
||||||
["webCommandMetadata"]
|
["webCommandMetadata"]
|
||||||
["url"],
|
["url"] : null,
|
||||||
"avatar" =>
|
"avatar" =>
|
||||||
$video["channelThumbnailSupportedRenderers"]
|
$video["channelThumbnailSupportedRenderers"]
|
||||||
["channelThumbnailWithLinkRenderer"]
|
["channelThumbnailWithLinkRenderer"]
|
||||||
|
|||||||
+28
-4
@@ -67,6 +67,10 @@ $settings = [
|
|||||||
"value" => "brave",
|
"value" => "brave",
|
||||||
"text" => "Brave"
|
"text" => "Brave"
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
"value" => "bing",
|
||||||
|
"text" => "Bing"
|
||||||
|
],
|
||||||
[
|
[
|
||||||
"value" => "ddg",
|
"value" => "ddg",
|
||||||
"text" => "DuckDuckGo"
|
"text" => "DuckDuckGo"
|
||||||
@@ -79,17 +83,25 @@ $settings = [
|
|||||||
"value" => "google",
|
"value" => "google",
|
||||||
"text" => "Google"
|
"text" => "Google"
|
||||||
],
|
],
|
||||||
|
[
|
||||||
|
"value" => "yahoo",
|
||||||
|
"text" => "Yahoo"
|
||||||
|
],
|
||||||
[
|
[
|
||||||
"value" => "startpage",
|
"value" => "startpage",
|
||||||
"text" => "Startpage"
|
"text" => "Startpage"
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"value" => "kagi",
|
"value" => "qwant",
|
||||||
"text" => "Kagi"
|
"text" => "Qwant"
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"value" => "yep",
|
"value" => "ecosia",
|
||||||
"text" => "Yep"
|
"text" => "Ecosia"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"value" => "kagi",
|
||||||
|
"text" => "Kagi"
|
||||||
],
|
],
|
||||||
[
|
[
|
||||||
"value" => "marginalia",
|
"value" => "marginalia",
|
||||||
@@ -102,6 +114,18 @@ $settings = [
|
|||||||
[
|
[
|
||||||
"value" => "sc",
|
"value" => "sc",
|
||||||
"text" => "SoundCloud"
|
"text" => "SoundCloud"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"value" => "coccoc",
|
||||||
|
"text" => "Coccoc"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"value" => "naver",
|
||||||
|
"text" => "Naver"
|
||||||
|
],
|
||||||
|
[
|
||||||
|
"value" => "purili",
|
||||||
|
"text" => "Purili"
|
||||||
]
|
]
|
||||||
]
|
]
|
||||||
],
|
],
|
||||||
|
|||||||
@@ -349,8 +349,8 @@ if(count($results["answer"]) !== 0){
|
|||||||
|
|
||||||
$right["answer"] .=
|
$right["answer"] .=
|
||||||
'<tr>' .
|
'<tr>' .
|
||||||
'<td>' . $info . '</td>' .
|
'<td>' . htmlspecialchars($info) . '</td>' .
|
||||||
'<td>' . $value . '</td>' .
|
'<td>' . htmlspecialchars($value) . '</td>' .
|
||||||
'</tr>';
|
'</tr>';
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Reference in new issue
Block a user