Compare commits

...
12 Commits
10 changed files with 1458 additions and 1139 deletions

No files matched your search

+115 -25
View File
@@ -7,37 +7,51 @@ class autocomplete{
public function __construct(){ public function __construct(){
header("Content-Type: application/json"); header("Content-Type: application/x-suggestions+json");
$this->scrapers = [ $this->scrapers = [
"brave" => "https://search.brave.com/api/suggest?q={searchTerms}", "brave" => "https://search.brave.com/api/suggest?q={searchTerms}",
"bing" => "https://www.bing.com/osjson.aspx?query={searchTerms}",
"ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list", "ddg" => "https://duckduckgo.com/ac/?q={searchTerms}&type=list",
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015", "ecosia" => "https://ac.ecosia.org/autocomplete?q={searchTerms}&type=list",
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}", "yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3",
"google" => "https://www.google.com/complete/search?client=chrome&q={searchTerms}",
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch", "qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}", "marginalia" => "https://marginalia-search.com/suggest/?partial={searchTerms}",
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}", "yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
"sc" => "", "sc" => "",
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english", "kagi" => "https://kagisuggest.com/api/autosuggest?q={searchTerms}",
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}" "startpage" => "https://www.startpage.com/osuggestions?q={searchTerms}",
"yahoo" => "https://search.yahoo.com/sugg/gossip/gossip-us-ura/?output=sd1&command={searchTerms}",
"purili" => "https://puri.li/api/suggest?q={searchTerms}",
"coccoc" => "https://coccoc.com/composer/autocomplete?q={searchTerms}",
"naver" => "https://ac.search.naver.com/nx/ac?q={searchTerms}&st=100"
]; ];
// add duplicates
$this->scrapers["google_api"] = &$this->scrapers["google"];
$this->scrapers["google_cse"] = &$this->scrapers["google"];
$this->scrapers["yahoo_japan"] = &$this->scrapers["yahoo"];
/* /*
Sanitize input Sanitize input
*/ */
if(!isset($_GET["s"])){ if(!isset($_GET["s"])){
$this->do404("Missing search(s) parameter"); // Missing search(s) parameter
$this->doempty();
} }
if(is_string($_GET["s"]) === false){ if(is_string($_GET["s"]) === false){
$this->do404("Invalid search(s) parameter"); // Invalid search(s) parameter
$this->doempty();
} }
if(strlen($_GET["s"]) > 500){ if(strlen($_GET["s"]) > 500){
$this->do404("Search(s) exceeds the 500 char length"); // Search(s) exceeds the 500 char length
$this->doempty();
} }
/* /*
@@ -72,9 +86,8 @@ class autocomplete{
// return results // return results
switch($scraper){ switch($scraper){
case "google":
case "yt": case "yt":
// handle google cause they want to be a special snowflake :( // handle youtube cause it wants to be a special snowflake :(
$js = $this->get($this->scrapers[$scraper], $_GET["s"]); $js = $this->get($this->scrapers[$scraper], $_GET["s"]);
preg_match( preg_match(
@@ -141,12 +154,12 @@ class autocomplete{
break; break;
case "marginalia": case "marginalia":
case "purili":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]); $json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true); $json = json_decode($json, true);
if($json === null){ if($json === null){
$this->doempty(); $this->doempty();
} }
@@ -159,11 +172,97 @@ class autocomplete{
); );
break; break;
case "yahoo":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["r"][0]["k"])
){
$this->doempty();
}
$payload = [];
foreach($json["r"] as $term){
if(isset($term["k"])){
$payload[] = $term["k"];
}
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "naver":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["items"][0][0])
){
$this->doempty();
}
$payload = [];
foreach($json["items"][0] as $term){
$payload[] = $term[0];
}
echo json_encode(
[
$_GET["s"],
$payload
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
case "coccoc":
$json = $this->get($this->scrapers[$scraper], $_GET["s"]);
$json = json_decode($json, true);
if(
$json === null ||
!isset($json["suggestions"])
){
$this->doempty();
}
echo json_encode(
[
$_GET["s"],
$json["suggestions"]
],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
break;
default: default:
// if it respects the openSearch protocol // if it respects the openSearch protocol
$json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true); $json = json_decode($this->get($this->scrapers[$scraper], $_GET["s"]), true);
if(!isset($json[1])){ $json[1] = []; } if(
!isset($json[1]) ||
!is_array($json[1])
){
$json[1] = [];
}
echo json_encode( echo json_encode(
[ [
@@ -187,10 +286,10 @@ class autocomplete{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER, curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/116.0", ["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/javascript, */*; q=0.01", "Accept: application/json, text/javascript, */*; q=0.01",
"Accept-Language: en-US,en;q=0.5", "Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip", "Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1", "DNT: 1",
"Connection: keep-alive", "Connection: keep-alive",
"Sec-Fetch-Dest: empty", "Sec-Fetch-Dest: empty",
@@ -216,19 +315,10 @@ class autocomplete{
}catch(Exception $error){ }catch(Exception $error){
do404("Curl error: " . $error->getMessage()); doempty();
} }
} }
private function do404($error){
echo json_encode(
["error" => $error],
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES | JSON_INVALID_UTF8_IGNORE
);
die();
}
private function doempty(){ private function doempty(){
echo json_encode( echo json_encode(
+1
View File
@@ -0,0 +1 @@
+3 -3
View File
@@ -154,7 +154,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages // Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things. // Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:157.0) Gecko/20100101 Firefox/157.0"; const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:160.0) Gecko/20100101 Firefox/160.0";
// User agent to use with 4get-friendly APIs // User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)"; const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
@@ -168,7 +168,6 @@ class config{
const PROXY_YAHOO_JAPAN = false; const PROXY_YAHOO_JAPAN = false;
const PROXY_BRAVE = false; const PROXY_BRAVE = false;
const PROXY_GOOGLE = false; const PROXY_GOOGLE = false;
const PROXY_GOOGLE_ALT = false; // alt proxies for url resolution
const PROXY_GOOGLE_API = false; const PROXY_GOOGLE_API = false;
const PROXY_GOOGLE_CSE = false; const PROXY_GOOGLE_CSE = false;
const PROXY_STARTPAGE = false; const PROXY_STARTPAGE = false;
@@ -226,6 +225,7 @@ class config{
// use the API (Eg: "public"). API has less filters. // use the API (Eg: "public"). API has less filters.
const MARGINALIA_API_KEY = null; const MARGINALIA_API_KEY = null;
// Yep // Scrapers to use the API on
const YEP_USE_API = false; const YEP_USE_API = false;
const BRAVE_USE_API = false;
} }
+1 -2
View File
@@ -1119,7 +1119,6 @@ class frontend{
$filters["scraper"] = [ $filters["scraper"] = [
"display" => "Scraper", "display" => "Scraper",
"option" => [ "option" => [
//"fget" => "fget",
"ddg" => "DuckDuckGo", "ddg" => "DuckDuckGo",
//"yahoo" => "Yahoo!", //"yahoo" => "Yahoo!",
"brave" => "Brave", "brave" => "Brave",
@@ -1207,7 +1206,7 @@ class frontend{
"yahoo_japan" => "Yahoo! JAPAN", "yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage", "startpage" => "Startpage",
//"mojeek" => "Mojeek", //"mojeek" => "Mojeek",
"baidu" => "Baidu" //"baidu" => "Baidu"
] ]
]; ];
break; break;
+26 -13
View File
@@ -3,32 +3,45 @@
header("Content-Type: application/xml"); header("Content-Type: application/xml");
include "data/config.php"; include "data/config.php";
$domain = $host = strtolower(explode(":", $_SERVER["HTTP_HOST"], 2)[0]);
htmlspecialchars(
(isset($_SERVER['HTTPS']) && $_SERVER['HTTPS'] == "on" ? "https" : "http") .
'://' . $_SERVER["HTTP_HOST"]
);
// detect if 4get should serve the HTTPS endpoint
if( if(
preg_match( preg_match(
'/\.onion$/', '/\.(onion|i2p)$/',
$domain strtolower($host),
$match
) )
){ ){
$onion = true; $ssl = false;
$src = $match[1];
}elseif(
is_string(
filter_var(
$host,
FILTER_VALIDATE_IP
)
) ||
$host == "localhost"
){
$ssl = false;
$src = "localhost";
}else{ }else{
$onion = false; $ssl = true;
} }
$domain = ($ssl ? "https" : "http") . "://" . $_SERVER["HTTP_HOST"];
echo echo
'<?xml version="1.0" encoding="UTF-8"?>' .
'<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' . '<OpenSearchDescription xmlns="http://a9.com/-/spec/opensearch/1.1/">' .
'<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($onion ? " (onion)" : "") . '</ShortName>' . '<ShortName>' . htmlspecialchars(config::SERVER_NAME) . ($ssl === false ? " ({$src})" : "") . '</ShortName>' .
'<Description>' . htmlspecialchars(config::SERVER_SHORT_DESCRIPTION) . '</Description>' .
'<InputEncoding>UTF-8</InputEncoding>' . '<InputEncoding>UTF-8</InputEncoding>' .
'<Image width="16" height="16">' . $domain . '/favicon.ico</Image>' . '<Image width="32" height="32" type="image/x-icon">' . htmlspecialchars($domain) . '/favicon.ico</Image>' .
'<Url type="text/html" method="GET" template="' . $domain . '/web?s={searchTerms}"/>'; '<Url rel="results" type="text/html" method="GET" template="' . htmlspecialchars($domain) . '/web?s={searchTerms}"/>';
if( if(
isset($_GET["ac"]) && isset($_GET["ac"]) &&
+223 -1006
View File
File diff suppressed because it is too large. Load diff
+884 -12
View File
@@ -167,17 +167,60 @@ class brave{
$headers = [ $headers = [
"User-Agent: " . config::USER_AGENT, "User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5", "Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip", "Accept-Encoding: gzip, deflate, br, zstd",
"Cookie: safesearch={$nsfw}; country={$country}; useLocation=0; summarizer=0", "Referer: https://search.brave.com/",
"DNT: 1", "DNT: 1",
"Sec-GPC: 1",
"Cookie: safesearch={$nsfw}; country={$country}; summarizer=0",
"Connection: keep-alive", "Connection: keep-alive",
"Upgrade-Insecure-Requests: 1", "Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document", "Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate", "Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none", "Sec-Fetch-Site: same-origin",
"Sec-Fetch-User: ?1" "Sec-Fetch-User: ?1",
"Priority: u=0, i",
"TE: trailers"
];
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
private function api_get($proxy, $key, $url, $get = []){
$headers = [
"User-Agent: " . config::USER_AGENT,
"Accept: application/json",
"Accept-Encoding: gzip",
"X-Subscription-Token: " . $key
]; ];
$curlproc = curl_init(); $curlproc = curl_init();
@@ -253,6 +296,11 @@ class brave{
public function web($get){ public function web($get){
if(config::BRAVE_USE_API){
return $this->api_web($get);
}
if($get["npt"]){ if($get["npt"]){
// get next page data // get next page data
@@ -278,7 +326,7 @@ class brave{
throw new Exception("Search term is empty!"); throw new Exception("Search term is empty!");
} }
if(strlen($search) > 2048){ if(strlen($search) > 600){
throw new Exception("Search term is too long!"); throw new Exception("Search term is too long!");
} }
@@ -1046,10 +1094,10 @@ class brave{
"description" => $this->titledots($news["description"]), "description" => $this->titledots($news["description"]),
"date" => isset($news["age"]) ? strtotime($news["age"]) : null, "date" => isset($news["age"]) ? strtotime($news["age"]) : null,
"thumb" => "thumb" =>
isset($video["thumbnail"]["src"]) ? isset($news["thumbnail"]["src"]) ?
[ [
"ratio" => "16:9", "ratio" => "16:9",
"url" => $this->unshiturl($video["thumbnail"]["src"]) "url" => $this->unshiturl($news["thumbnail"]["src"])
] : ] :
[ [
"ratio" => null, "ratio" => null,
@@ -1122,13 +1170,18 @@ class brave{
public function image($get){ public function image($get){
if(config::BRAVE_USE_API){
return $this->api_image($get);
}
$search = $get["s"]; $search = $get["s"];
if(strlen($search) === 0){ if(strlen($search) === 0){
throw new Exception("Search term is empty!"); throw new Exception("Search term is empty!");
} }
if(strlen($search) > 2048){ if(strlen($search) > 400){
throw new Exception("Search term is too long!"); throw new Exception("Search term is too long!");
} }
@@ -1211,6 +1264,11 @@ class brave{
public function video($get){ public function video($get){
if(config::BRAVE_USE_API){
return $this->api_video($get);
}
if($get["npt"]){ if($get["npt"]){
[$npt, $proxy] = $this->backend->get($get["npt"], "videos"); [$npt, $proxy] = $this->backend->get($get["npt"], "videos");
@@ -1249,7 +1307,7 @@ class brave{
throw new Exception("Search term is empty!"); throw new Exception("Search term is empty!");
} }
if(strlen($search) > 2048){ if(strlen($search) > 400){
throw new Exception("Search term is too long!"); throw new Exception("Search term is too long!");
} }
@@ -1377,6 +1435,11 @@ class brave{
public function news($get){ public function news($get){
if(config::BRAVE_USE_API){
return $this->api_news($get);
}
if($get["npt"]){ if($get["npt"]){
[$req, $proxy] = $this->backend->get($get["npt"], "news"); [$req, $proxy] = $this->backend->get($get["npt"], "news");
@@ -1415,7 +1478,7 @@ class brave{
throw new Exception("Search term is empty!"); throw new Exception("Search term is empty!");
} }
if(strlen($search) > 2048){ if(strlen($search) > 400){
throw new Exception("Search term is too long!"); throw new Exception("Search term is too long!");
} }
@@ -1516,6 +1579,815 @@ class brave{
return $out; return $out;
} }
private function api_web($get){
$keydata = $this->backend->get_key();
$proxy = $this->backend->get_ip($keydata["increment"]);
if($get["npt"]){
[$filters, $n] = $this->backend->get($get["npt"], "web");
$filters = json_decode($filters, true);
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
if(strlen($get["s"]) > 600){
throw new Exception("Search term is too long!");
}
// push filters
$nsfw = $get["nsfw"];
$country = $get["country"];
$older = $get["older"];
$newer = $get["newer"];
$spellcheck = $get["spellcheck"];
$filters = [
"q" => $get["s"],
"count" => 20
];
// freshness filter
if($newer !== false){
$newer = date("Y-m-d", $newer);
if($older === false){
$older = date("Y-m-d", time());
}
}
if(
is_string($older) === false &&
$older !== false
){
$older = date("Y-m-d", $older);
if($newer === false){
$newer = "1970-01-02";
}
}
if($older !== false){
$filters["freshness"] = "{$newer}to{$older}";
}
// spellcheck
if($spellcheck == "no"){
$filters["spellcheck"] = "false";
}
// country
if($country != "all"){
$filters["country"] = strtoupper($country);
}
switch($nsfw){
case "yes": $filters["safesearch"] = "off"; break;
case "maybe": $filters["safesearch"] = "moderate"; break;
case "no": $filters["safesearch"] = "strict"; break;
}
}
//$json = file_get_contents("scraper/brave4.json");
try{
$json =
$this->api_get(
$proxy,
$keydata["key"],
"https://api.search.brave.com/res/v1/web/search",
$filters
);
}catch(Exception $error){
throw new Exception("Could not fetch search page");
}
$out = [
"status" => "ok",
"spelling" => [
"type" => "no_correction",
"using" => null,
"correction" => null
],
"npt" => null,
"answer" => [],
"web" => [],
"image" => [],
"video" => [],
"news" => [],
"related" => []
];
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json["error"])){
if(isset($json["error"]["detail"])){
throw new Exception("Brave returned an error: {$json["error"]["detail"]}");
}
throw new Exception("Brave returned an unknown error");
}
if(!isset($json["web"]["results"])){
throw new Exception("Brave did not return a web results object");
}
// parse web results
foreach($json["web"]["results"] as $result){
$sublinks = [];
if(isset($result["cluster"])){
foreach($result["cluster"] as $sublink){
$sublinks[] = [
"title" => $sublink["title"],
"description" =>
$this->titledots(
$this->fuckhtml
->getTextContent(
$sublink["description"]
)
),
"url" => $sublink["url"],
"date" => null
];
}
}
if(isset($result["deep_results"]["buttons"])){
foreach($result["deep_results"]["buttons"] as $sublink){
$sublinks[] = [
"title" => $sublink["title"],
"description" => null,
"url" => $sublink["url"],
"date" => null
];
}
}
$out["web"][] = [
"title" =>
$this->titledots(
$result["title"]
),
"description" =>
$this->titledots(
$this->fuckhtml
->getTextContent(
$result["description"]
)
),
"url" => $result["url"],
"date" => isset($result["age"]) ? strtotime($result["age"]) : null,
"type" => $result["is_live"] ? "Live" : "web",
"thumb" =>
isset($result["thumbnail"]["original"]) &&
$result["thumbnail"]["logo"] === false ?
[
"ratio" => "16:9",
"url" => $result["thumbnail"]["original"]
] :
[
"ratio" => null,
"url" => null
],
"sublink" => $sublinks,
"table" => []
];
}
// parse videos
if(isset($json["videos"]["results"])){
foreach($json["videos"]["results"] as $video){
$out["video"][] = [
"title" => $this->titledots($video["title"]),
"description" => $this->titledots($video["description"]),
"date" => isset($video["age"]) ? strtotime($video["age"]) : null,
"duration" => isset($video["video"]["duration"]) ? $this->hms2int($video["video"]["duration"]) : null,
"views" => isset($video["video"]["views"]) ? (int)$video["video"]["views"] : null,
"thumb" =>
isset($video["thumbnail"]["original"]) ?
[
"ratio" => "16:9",
"url" => $video["thumbnail"]["original"]
] :
[
"ratio" => null,
"url" => null
],
"url" => $video["url"]
];
}
}
// parse news
if(isset($json["news"]["results"])){
foreach($json["news"]["results"] as $nw){
$out["news"][] = [
"title" => $this->titledots($nw["title"]),
"description" =>
$this->titledots(
$this->fuckhtml
->getTextContent(
$nw["description"]
)
),
"date" => isset($nw["page_age"]) ? strtotime($nw["page_age"]) : null,
"thumb" =>
isset($nw["thumbnail"]["original"]) ?
[
"ratio" => "16:9",
"url" => $nw["thumbnail"]["original"]
] :
[
"ratio" => null,
"url" => null
],
"url" => $nw["url"]
];
}
}
// parse infobox
if(isset($json["infobox"]["results"])){
foreach($json["infobox"]["results"] as $info){
$table = [];
$sublinks = [];
$description = [];
$thumb =
isset($info["images"][0]["src"]) ?
$info["images"][0]["original"] : null;
// dedup wikipedia entries
if($info["url"] !== null){
$query1 = parse_url($info["url"]);
if(str_contains($query1["host"], "wikipedia.org")){
foreach($out["answer"] as $check){
$query2 = parse_url($check["url"]);
if(
str_contains($query2["host"], "wikipedia.org") &&
isset($query1["path"]) &&
isset($query2["path"]) &&
$query1["path"] == $query2["path"]
){ continue 2; }
}
}
}
if(isset($info["description"])){
$description[] = [
"type" => "quote",
"value" => $info["description"]
];
}
if(isset($info["long_desc"])){
$description[] = [
"type" => "text",
"value" => $this->titledots($info["long_desc"])
];
}
// get table elements
if(isset($info["attributes"])){
foreach($info["attributes"] as $attribute){
if(
isset($attribute[0]) &&
isset($attribute[1])
){
$table[
$this->fuckhtml
->getTextContent(
$attribute[0]
)
] =
$this->fuckhtml
->getTextContent(
$attribute[1]
);
}
}
}
if(isset($info["profiles"])){
foreach($info["profiles"] as $sublink){
$sublinks[$sublink["name"]] = $sublink["url"];
}
}
if(isset($info["website_url"])){
$sublinks["Website"] = $info["website_url"];
}
$out["answer"][] = [
"title" => $info["title"],
"description" => $description,
"url" => isset($info["url"]) ? $info["url"] : null,
"thumb" => $thumb,
"table" => $table,
"sublink" => $sublinks
];
}
}
// parse related searches
if(isset($json["query"]["related_queries"])){
foreach($json["query"]["related_queries"] as $related){
$out["related"][] = $related;
}
}
// parse correction
if(isset($json["query"]["altered"])){
$out["spelling"] = [
"type" => "including",
"using" => $json["query"]["altered"],
"correction" => $json["query"]["original"]
];
}
// parse npt
if($json["query"]["more_results_available"]){
if(!isset($filters["offset"])){
$filters["offset"] = 1;
}else{
$filters["offset"]++;
}
$out["npt"] =
$this->backend->store(
json_encode($filters),
"web",
$proxy
);
}
return $out;
}
private function api_image($get){
$keydata = $this->backend->get_key();
$proxy = $this->backend->get_ip($keydata["increment"]);
if($get["npt"]){
// no pagination support
//[$filters, $n] = $this->backend->get($get["npt"], "web");
//$filters = json_decode($filters, true);
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
if(strlen($get["s"]) > 400){
throw new Exception("Search term is too long!");
}
// push filters
$nsfw = $get["nsfw"];
$country = $get["country"];
$spellcheck = $get["spellcheck"];
$filters = [
"q" => $get["s"],
"count" => 200
];
// spellcheck
if($spellcheck == "no"){
$filters["spellcheck"] = "false";
}
// country
if($country != "all"){
$filters["country"] = strtoupper($country);
}
switch($nsfw){
case "yes": $filters["safesearch"] = "off"; break;
case "maybe": $filters["safesearch"] = "moderate"; break;
case "no": $filters["safesearch"] = "strict"; break;
}
}
//$json = file_get_contents("scraper/brave-img.json");
try{
$json =
$this->api_get(
$proxy,
$keydata["key"],
"https://api.search.brave.com/res/v1/images/search",
$filters
);
}catch(Exception $error){
throw new Exception("Could not fetch search page");
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json["error"])){
if(isset($json["error"]["detail"])){
throw new Exception("Brave returned an error: {$json["error"]["detail"]}");
}
throw new Exception("Brave returned an unknown error");
}
if(!isset($json["results"])){
throw new Exception("Brave did not return a results object");
}
// parse web results
foreach($json["results"] as $result){
$out["image"][] = [
"title" => $result["title"],
"source" => [
[
"url" => $result["properties"]["url"],
"width" => (int)$result["properties"]["width"],
"height" => (int)$result["properties"]["height"]
],
[
"url" => $result["thumbnail"]["src"],
"width" => (int)$result["thumbnail"]["width"],
"height" => (int)$result["thumbnail"]["height"]
]
],
"url" => $result["url"]
];
}
return $out;
}
private function api_video($get){
$keydata = $this->backend->get_key();
$proxy = $this->backend->get_ip($keydata["increment"]);
if($get["npt"]){
[$filters, $n] = $this->backend->get($get["npt"], "videos");
$filters = json_decode($filters, true);
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
if(strlen($get["s"]) > 600){
throw new Exception("Search term is too long!");
}
// push filters
$nsfw = $get["nsfw"];
$country = $get["country"];
$spellcheck = $get["spellcheck"];
$filters = [
"q" => $get["s"],
"count" => 100
];
// spellcheck
if($spellcheck == "no"){
$filters["spellcheck"] = "false";
}
// country
if($country != "all"){
$filters["country"] = strtoupper($country);
}
switch($nsfw){
case "yes": $filters["safesearch"] = "off"; break;
case "maybe": $filters["safesearch"] = "moderate"; break;
case "no": $filters["safesearch"] = "strict"; break;
}
}
//$json = file_get_contents("scraper/brave-video.json");
try{
$json =
$this->api_get(
$proxy,
$keydata["key"],
"https://api.search.brave.com/res/v1/videos/search",
$filters
);
}catch(Exception $error){
throw new Exception("Could not fetch search page");
}
$out = [
"status" => "ok",
"npt" => null,
"video" => [],
"author" => [],
"livestream" => [],
"playlist" => [],
"reel" => []
];
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json["error"])){
if(isset($json["error"]["detail"])){
throw new Exception("Brave returned an error: {$json["error"]["detail"]}");
}
throw new Exception("Brave returned an unknown error");
}
if(!isset($json["results"])){
throw new Exception("Brave did not return a results object");
}
// parse web results
foreach($json["results"] as $result){
$out["video"][] = [
"title" => $result["title"],
"description" => isset($result["description"]) ? $this->titledots($result["description"]) : null,
"author" => [
"name" => isset($result["author"]["name"]) ? $result["author"]["name"] : null,
"url" => isset($result["author"]["url"]) ? $result["author"]["url"] : null,
"avatar" => null
],
"date" => isset($result["age"]) ? strtotime($result["age"]) : null,
"duration" => isset($result["video"]["duration"]) ? $this->hms2int($result["video"]["duration"]) : null,
"views" => null,
"thumb" =>
isset($result["thumbnail"]["original"]) ?
[
"ratio" => "16:9",
"url" => $result["thumbnail"]["original"]
] :
[
"ratio" => null,
"url" => null
],
"url" => $result["url"]
];
}
if(count($out["video"]) >= 95){
if(!isset($filters["offset"])){
$filters["offset"] = 1;
}else{
$filters["offset"]++;
}
$out["npt"] =
$this->backend->store(
json_encode($filters),
"videos",
$proxy
);
}
return $out;
}
private function api_news($get){
$keydata = $this->backend->get_key();
$proxy = $this->backend->get_ip($keydata["increment"]);
if($get["npt"]){
[$filters, $n] = $this->backend->get($get["npt"], "news");
$filters = json_decode($filters, true);
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
if(strlen($get["s"]) > 400){
throw new Exception("Search term is too long!");
}
// push filters
$nsfw = $get["nsfw"];
$country = $get["country"];
$spellcheck = $get["spellcheck"];
$filters = [
"q" => $get["s"],
"count" => 50
];
// spellcheck
if($spellcheck == "no"){
$filters["spellcheck"] = "false";
}
// country
if($country != "all"){
$filters["country"] = strtoupper($country);
}
switch($nsfw){
case "yes": $filters["safesearch"] = "off"; break;
case "maybe": $filters["safesearch"] = "moderate"; break;
case "no": $filters["safesearch"] = "strict"; break;
}
}
//$json = file_get_contents("scraper/brave-news.json");
try{
$json =
$this->api_get(
$proxy,
$keydata["key"],
"https://api.search.brave.com/res/v1/news/search",
$filters
);
}catch(Exception $error){
throw new Exception("Could not fetch search page");
}
$out = [
"status" => "ok",
"npt" => null,
"news" => []
];
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json["error"])){
if(isset($json["error"]["detail"])){
throw new Exception("Brave returned an error: {$json["error"]["detail"]}");
}
throw new Exception("Brave returned an unknown error");
}
if(!isset($json["results"])){
throw new Exception("Brave did not return a results object");
}
// parse web results
foreach($json["results"] as $news){
$out["news"][] = [
"title" => $news["title"],
"author" =>
isset($news["profile"]["name"]) ?
$news["profile"]["name"] : null,
"description" => $news["description"],
"date" => isset($news["age"]) ? strtotime($news["age"]) : null,
"thumb" =>
isset($news["thumbnail"]["src"]) ?
[
"ratio" => "16:9",
"url" => $news["thumbnail"]["src"]
] :
[
"ratio" => null,
"url" => null,
],
"url" => $news["url"]
];
}
if(count($out["news"]) >= 50){
if(!isset($filters["offset"])){
$filters["offset"] = 1;
}else{
$filters["offset"]++;
}
$out["npt"] =
$this->backend->store(
json_encode($filters),
"news",
$proxy
);
}
return $out;
}
private function stackoverflow_parse($html){ private function stackoverflow_parse($html){
$i = 0; $i = 0;
@@ -1954,7 +2826,7 @@ class brave{
->getTextContent($title[0]) ->getTextContent($title[0])
); );
if($title == "human verification"){ if($title == "brave search"){
throw new Exception("Brave returned a Captcha"); throw new Exception("Brave returned a Captcha");
} }
+158 -16
View File
@@ -545,10 +545,12 @@ class startpage{
switch($category["display_type"]){ switch($category["display_type"]){
case "web-google": case "web-google":
case "web-bing":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
$sublinks = []; $sublinks = [];
if(isset($result["siteLinks"])){
foreach($result["siteLinks"] as $sublink){ foreach($result["siteLinks"] as $sublink){
$sublinks[] = [ $sublinks[] = [
@@ -557,6 +559,7 @@ class startpage{
"url" => $sublink["clickUrl"] "url" => $sublink["clickUrl"]
]; ];
} }
}
$description = $description =
explode( explode(
@@ -596,6 +599,7 @@ class startpage{
$out["web"][] = [ $out["web"][] = [
"title" => "title" =>
$this->remove_penguins(
$this->titledots( $this->titledots(
html_entity_decode( html_entity_decode(
$this->fuckhtml $this->fuckhtml
@@ -603,8 +607,9 @@ class startpage{
$result["title"] $result["title"]
) )
) )
)
), ),
"description" => $description, "description" => $this->remove_penguins($description),
"url" => $result["clickUrl"], "url" => $result["clickUrl"],
"date" => $date, "date" => $date,
"type" => "web", "type" => "web",
@@ -619,8 +624,43 @@ class startpage{
break; break;
case "images-qi-top": case "images-qi-top":
case "images-bing-top":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
if(isset($result["mdThumbnailWidth"])){
$thumb_width = $result["mdThumbnailWidth"];
}elseif(isset($result["thumbnailWidth"])){
$thumb_width = $result["thumbnailWidth"];
}else{
$thumb_width = null;
}
if(isset($result["mdThumbnailHeight"])){
$thumb_height = $result["mdThumbnailHeight"];
}elseif(isset($result["thumbnailHeight"])){
$thumb_height = $result["thumbnailHeight"];
}else{
$thumb_height = null;
}
if(isset($result["mdThumbnailUrl"])){
$thumb_source = $result["mdThumbnailUrl"];
}elseif(isset($result["thumbnailUrl"])){
$thumb_source = $result["thumbnailUrl"];
}else{
// should not happen
continue;
}
$out["image"][] = [ $out["image"][] = [
"title" => "title" =>
$this->titledots( $this->titledots(
@@ -638,19 +678,17 @@ class startpage{
"height" => (int)$result["height"] "height" => (int)$result["height"]
], ],
[ [
"url" => $this->unshitimage($result["mdThumbnailUrl"]), "url" => $this->unshitimage($thumb_source),
"width" => (int)$result["mdThumbnailWidth"], "width" => (int)$thumb_width,
"height" => (int)$result["mdThumbnailHeight"] "height" => (int)$thumb_height
] ]
], ],
"url" => "url" => $result["altClickUrl"]
$result["altClickUrl"]
]; ];
} }
break; break;
case "spellsuggest-google": case "spellsuggest-google":
if(isset($category["results"][0]["query"])){ if(isset($category["results"][0]["query"])){
$out["spelling"] = $out["spelling"] =
@@ -662,6 +700,60 @@ class startpage{
} }
break; break;
case "spelladjust-bing":
if(isset($category["results"][0]["alterationDisplayQuery"])){
$out["spelling"] =
[
"type" => "including",
"using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]),
"correction" => $category["results"][0]["originalQuery"]
];
}
break;
case "aylf-bing-top":
if(isset($category["results"][0]["query"])){
foreach($category["results"] as $ddm){
$out["related"][] = $ddm["query"];
}
}
break;
case "video-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $video){
if(isset($video["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $this->unshitimage($video["thumbnailUrl"])
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["video"][] = [
"title" => $video["title"],
"description" => $this->remove_penguins($video["description"]),
"date" => isset($video["date"]) ? strtotime($video["date"]) : null,
"duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null,
"views" => $video["viewCount"],
"thumb" => $thumb,
"url" => $video["clickUrl"]
];
}
}
break;
case "dictionary-qi": case "dictionary-qi":
foreach($category["results"] as $result){ foreach($category["results"] as $result){
@@ -732,6 +824,36 @@ class startpage{
$out["answer"][] = $answer; $out["answer"][] = $answer;
} }
break; break;
case "news-bing-top":
if(isset($category["results"][0]["clickUrl"])){
foreach($category["results"] as $news){
if(isset($news["thumbnailUrl"])){
$thumb = [
"ratio" => "16:9",
"url" => $news["thumbnailUrl"]
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["news"][] = [
"title" => $this->remove_penguins($news["title"]),
"description" => $news["description"],
"date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb,
"url" => $news["clickUrl"],
];
}
}
break;
} }
} }
@@ -1246,14 +1368,10 @@ class startpage{
// get npt // get npt
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies); $out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
//print_r($json);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){ foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
if($category["display_type"] != "news-bing"){
// unsupported category
continue;
}
foreach($category["results"] as $news){ foreach($category["results"] as $news){
if( if(
@@ -1276,10 +1394,10 @@ class startpage{
$out["news"][] = [ $out["news"][] = [
"title" => $this->titledots($this->remove_penguins($news["title"])), "title" => $this->titledots($this->remove_penguins($news["title"])),
"author" => $news["source"], "author" => $news["source"],
"description" => $this->titledots($this->remove_penguins($news["description"])), "description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null,
"date" => (int)substr((string)$news["date"], 0, -3), "date" => isset($news["date"]) ? strtotime($news["date"]) : null,
"thumb" => $thumb, "thumb" => $thumb,
"url" => $news["clickUrl"] "url" => $this->remove_tracking_link($news["clickUrl"])
]; ];
} }
} }
@@ -1327,6 +1445,30 @@ class startpage{
return null; return null;
} }
private function remove_tracking_link($url){
// https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog
$query = parse_url($url);
if(
!isset($query["host"]) ||
$query["host"] != "api.airfind.com" ||
!isset($query["query"])
){
return $url;
}
parse_str($query["query"], $res);
if(isset($res["redirectURL"])){
return $res["redirectURL"];
}
return $url;
}
private function unshitimage($url){ private function unshitimage($url){
$query = parse_url($url, PHP_URL_QUERY); $query = parse_url($url, PHP_URL_QUERY);
+19 -58
View File
@@ -205,13 +205,6 @@ class yep{
"zh-tw" => "Chinese (Traditional)", "zh-tw" => "Chinese (Traditional)",
"zu" => "Zulu" "zu" => "Zulu"
] ]
],
"nsfw" => [
"display" => "NSFW",
"option" => [
"yes" => "Yes",
"no" => "No"
]
] ]
]; ];
} }
@@ -317,20 +310,23 @@ class yep{
// parse filters // parse filters
$filters = [ $filters = [
"limit" => 100, // wwwwwwwwwwwwwww "q" => $search,
"query" => $search, "limit" => 10 // no fun allowed
]; ];
if($get["nsfw"] == "no"){ $filters["safeSearch"] = "moderate"; } if($get["lang"] != "any"){
if($get["lang"] != "any"){ $filters["hl"] = $get["lang"]; }
$filters["language"] = $get["lang"];
$filters["lang"] = $get["lang"];
}
try{ try{
// https://api.yep.com/search?limit=20&query=asmr // https://yep.com/web/v1/search?q=asmr&limit=10
$json = $json =
$this->get( $this->get(
$this->backend->get_ip(), $this->backend->get_ip(),
"https://api.yep.com/search", "https://yep.com/web/v1/search",
$filters $filters
); );
@@ -343,59 +339,27 @@ class yep{
$json = json_decode($json, true); $json = json_decode($json, true);
//$json = json_decode(file_get_contents("scraper/yep.json"), true); //$json = json_decode(file_get_contents("scraper/yep.json"), true);
if($json === null){ if($json === null){
throw new Exception("Failed to decode JSON"); throw new Exception("Failed to decode JSON");
} }
if(isset($json[1]["correction"])){ if(isset($json["code"])){
$out["spelling"] = [ if(isset($json["reason"])){
"type" => "not_many",
"using" => $search, throw new Exception("Yep returned an API error: [{$json["code"]}] {$json["reason"]}");
"correction" => $json[1]["correction"][1]
];
} }
if(isset($json[1]["results"])){ throw new Exception("Yep returned an unknown error");
foreach($json[1]["results"] as $item){
switch(strtolower($item["type"])){
case "organic":
$sublinks = [];
if(isset($item["sitelinks"]["full"])){
foreach($item["sitelinks"]["full"] as $link){
$sublinks[] = [
"title" => $link["title"],
"date" => null,
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$link["snippet"]
)
)
),
"url" => $link["url"]
];
}
} }
if(isset($json["results"])){
foreach($json["results"] as $item){
$out["web"][] = [ $out["web"][] = [
"title" => $item["title"], "title" => $item["title"],
"description" => "description" => str_replace("\n", " ", $item["description"]),
$this->titledots(
strip_tags(
html_entity_decode(
$item["snippet"]
)
)
),
"url" => $item["url"], "url" => $item["url"],
"date" => null, "date" => null,
"type" => "web", "type" => "web",
@@ -403,11 +367,9 @@ class yep{
"url" => null, "url" => null,
"ratio" => null "ratio" => null
], ],
"sublink" => $sublinks, "sublink" => [],
"table" => [] "table" => []
]; ];
break;
}
} }
} }
@@ -445,7 +407,6 @@ class yep{
"limit" => 99 // for some reason they dont show anything sometimes at 100 results "limit" => 99 // for some reason they dont show anything sometimes at 100 results
]; ];
if($get["nsfw"] == "no"){ $filters["safe_search"] = true; }
if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; } if($get["lang"] != "any"){ $filters["language"] = [ $get["lang"] ]; }
// add api key // add api key
+28 -4
View File
@@ -67,6 +67,10 @@ $settings = [
"value" => "brave", "value" => "brave",
"text" => "Brave" "text" => "Brave"
], ],
[
"value" => "bing",
"text" => "Bing"
],
[ [
"value" => "ddg", "value" => "ddg",
"text" => "DuckDuckGo" "text" => "DuckDuckGo"
@@ -79,17 +83,25 @@ $settings = [
"value" => "google", "value" => "google",
"text" => "Google" "text" => "Google"
], ],
[
"value" => "yahoo",
"text" => "Yahoo"
],
[ [
"value" => "startpage", "value" => "startpage",
"text" => "Startpage" "text" => "Startpage"
], ],
[ [
"value" => "kagi", "value" => "qwant",
"text" => "Kagi" "text" => "Qwant"
], ],
[ [
"value" => "yep", "value" => "ecosia",
"text" => "Yep" "text" => "Ecosia"
],
[
"value" => "kagi",
"text" => "Kagi"
], ],
[ [
"value" => "marginalia", "value" => "marginalia",
@@ -102,6 +114,18 @@ $settings = [
[ [
"value" => "sc", "value" => "sc",
"text" => "SoundCloud" "text" => "SoundCloud"
],
[
"value" => "coccoc",
"text" => "Coccoc"
],
[
"value" => "naver",
"text" => "Naver"
],
[
"value" => "purili",
"text" => "Purili"
] ]
] ]
], ],