lolcat
/
4get
6
13
Fork 4
Code Issues 13 Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: lolcat:master
Branches Tags
lolcat:master
bread:master
..
compare: bread:master
Branches Tags
bread:master
lolcat:master

No commits in common. "master" and "master" have entirely different histories.
master ... master

9 changed files with 42 additions and 1259 deletions
Show Stats Expand all files Collapse all files

6
data/config.php
View File

@ -119,7 +119,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:129.0) Gecko/20100101 Firefox/129.0";
// Proxy pool assignments for each scraper
// false = Use server's raw IP
@ -129,7 +129,6 @@ class config{
const PROXY_BRAVE = false;
const PROXY_FB = false; // facebook
const PROXY_GOOGLE = false;
const PROXY_GOOGLE_CSE = false;
const PROXY_STARTPAGE = false;
const PROXY_QWANT = false;
const PROXY_GHOSTERY = false;
@ -158,9 +157,6 @@ class config{
// Scraper-specific parameters
//
// GOOGLE CSE
const GOOGLE_CX_ENDPOINT = "d4e68b99b876541f0";
// MARGINALIA
// Use "null" to default out to HTML scraping OR specify a string to
// use the API (Eg: "public"). API has less filters.

2
lib/frontend.php
View File

@ -939,7 +939,6 @@ class frontend{
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"google_cse" => "Google CSE",
"startpage" => "Startpage",
"qwant" => "Qwant",
"ghostery" => "Ghostery",
@ -964,7 +963,6 @@ class frontend{
"yandex" => "Yandex",
"brave" => "Brave",
"google" => "Google",
"google_cse" => "Google CSE",
"startpage" => "Startpage",
"qwant" => "Qwant",
"yep" => "Yep",

28
lib/fuckhtml.php
View File

@ -381,8 +381,6 @@ class fuckhtml{
$json_out = null;
$last_char = null;
$keyword_check = null;
for($i=0; $i<strlen($json); $i++){
switch($json[$i]){
@ -398,7 +396,6 @@ class fuckhtml{
$bracket = false;
$is_close_bracket = true;
}else{
if($bracket === false){
@ -432,31 +429,6 @@ class fuckhtml{
$is_close_bracket === false
){
// do keyword check
$keyword_check .= $json[$i];
if(in_array($json[$i], [":", "{"])){
$keyword_check = substr($keyword_check, 0, -1);
if(
preg_match(
'/function|array|return/i',
$keyword_check
)
){
$json_out =
preg_replace(
'/[{"]*' . preg_quote($keyword_check, "/") . '$/',
"",
$json_out
);
}
$keyword_check = null;
}
// here we know we're not iterating over a quoted string
switch($json[$i]){

31
scraper/brave.php
View File

@ -293,8 +293,8 @@ class brave{
/*
$handle = fopen("scraper/brave.html", "r");
$html = fread($handle, filesize("scraper/brave.html"));
fclose($handle);*/
fclose($handle);
*/
try{
$html =
@ -410,20 +410,10 @@ class brave{
throw new Exception("Could not grep JavaScript object");
}
$data =
rtrim(
preg_replace(
'/\(Array\(0\)\)\).*$/',
"",
$grep[1]
),
" ]"
) . "]";
$data =
$this->fuckhtml
->parseJsObject(
$data
$grep[1]
);
unset($grep);
@ -673,10 +663,7 @@ class brave{
$table["Address"] = $result["location"]["postal_address"]["displayAddress"];
}
if(
isset($result["location"]["rating"]) &&
$result["location"]["rating"] != "void 0"
){
if(isset($result["location"]["rating"])){
$table["Rating"] =
$result["location"]["rating"]["ratingValue"] . "/" .
@ -684,19 +671,13 @@ class brave{
number_format($result["location"]["rating"]["reviewCount"]) . " votes)";
}
if(
isset($result["location"]["contact"]["telephone"]) &&
$result["location"]["contact"]["telephone"] != "void 0"
){
if(isset($result["location"]["contact"]["telephone"])){
$table["Phone number"] =
$result["location"]["contact"]["telephone"];
}
if(
isset($result["location"]["price_range"]) &&
$result["location"]["price_range"] != "void 0"
){
if(isset($result["location"]["price_range"])){
$table["Price"] =
$result["location"]["price_range"];

37
scraper/ddg.php
View File

@ -28,9 +28,6 @@ class ddg{
curl_setopt($curlproc, CURLOPT_URL, $url);
// http2 bypass
curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
switch($reqtype){
case self::req_web:
$headers =
@ -39,33 +36,27 @@ class ddg{
"Accept-Encoding: gzip",
"Accept-Language: en-US,en;q=0.5",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: same-origin",
"Sec-Fetch-User: ?1",
"Priority: u=0, i",
"TE: trailers"];
"Sec-Fetch-Site: cross-site",
"Upgrade-Insecure-Requests: 1"];
break;
case self::req_xhr:
$headers =
["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/javascript, */*; q=0.01",
"Accept: */*",
"Accept-Encoding: gzip",
"Accept-Language: en-US,en;q=0.5",
"Connection: keep-alive",
"Referer: https://duckduckgo.com/",
"X-Requested-With: XMLHttpRequest",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"TE: trailers"];
"Sec-Fetch-Dest: script",
"Sec-Fetch-Mode: no-cors",
"Sec-Fetch-Site: same-site"];
break;
}
@ -1898,12 +1889,12 @@ class ddg{
[$npt, $proxy] = $this->backend->get($get["npt"], "images");
try{
$json = $this->get(
$json = json_decode($this->get(
$proxy,
"https://duckduckgo.com/i.js?" . $npt,
[],
ddg::req_xhr
);
), true);
}catch(Exception $err){
@ -1929,7 +1920,6 @@ class ddg{
$filter = [];
$get_filters = [
"hps" => "1",
"q" => $search,
"iax" => "images",
"ia" => "images"
@ -2004,12 +1994,12 @@ class ddg{
}
try{
$json = $this->get(
$json = json_decode($this->get(
$proxy,
"https://duckduckgo.com/i.js",
$js_params,
ddg::req_xhr
);
), true);
}catch(Exception $err){
@ -2017,13 +2007,6 @@ class ddg{
}
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$out = [
"status" => "ok",
"npt" => null,

1054
scraper/google_cse.php
View File

File diff suppressed because it is too large Load Diff

123
scraper/marginalia.php
View File

@ -220,7 +220,6 @@ class marginalia{
"related" => []
];
// API scraper
if(config::MARGINALIA_API_KEY !== null){
try{
@ -264,57 +263,34 @@ class marginalia{
return $out;
}
// HTML parser
$proxy = $this->backend->get_ip();
// no more cloudflare!! Parse html by default
$params = [
"query" => $search
];
if($get["npt"]){
foreach(["adtech", "recent", "intitle"] as $v){
[$params, $proxy] =
$this->backend->get(
$get["npt"],
"web"
);
if($get[$v] == "yes"){
try{
$html =
$this->get(
$proxy,
"https://search.marginalia.nu/search?" . $params
);
}catch(Exception $error){
switch($v){
throw new Exception("Failed to get HTML");
}
}else{
$params = [
"query" => $search
];
foreach(["adtech", "recent", "intitle"] as $v){
if($get[$v] == "yes"){
switch($v){
case "adtech": $params["adtech"] = "reduce"; break;
case "recent": $params["recent"] = "recent"; break;
case "adtech": $params["searchTitle"] = "title"; break;
}
case "adtech": $params["adtech"] = "reduce"; break;
case "recent": $params["recent"] = "recent"; break;
case "adtech": $params["searchTitle"] = "title"; break;
}
}
}
try{
$html =
$this->get(
$proxy,
"https://search.marginalia.nu/search",
$params
);
}catch(Exception $error){
try{
$html =
$this->get(
$this->backend->get_ip(),
"https://search.marginalia.nu/search",
$params
);
}catch(Exception $error){
throw new Exception("Failed to get HTML");
}
throw new Exception("Failed to get HTML");
}
$this->fuckhtml->load($html);
@ -411,65 +387,6 @@ class marginalia{
];
}
// get next page
$this->fuckhtml->load($html);
$pagination =
$this->fuckhtml
->getElementsByAttributeValue(
"aria-label",
"pagination",
"nav"
);
if(count($pagination) === 0){
// no pagination
return $out;
}
$this->fuckhtml->load($pagination[0]);
$pages =
$this->fuckhtml
->getElementsByClassName(
"page-link",
"a"
);
$found_current_page = false;
foreach($pages as $page){
if(
stripos(
$page["attributes"]["class"],
"active"
) !== false
){
$found_current_page = true;
continue;
}
if($found_current_page){
// we found current page index, and we iterated over
// the next page <a>
$out["npt"] =
$this->backend->store(
parse_url(
$page["attributes"]["href"],
PHP_URL_QUERY
),
"web",
$proxy
);
break;
}
}
return $out;
}
}

8
scraper/mojeek.php
View File

@ -701,11 +701,9 @@ class mojeek{
if(count($thumb) === 2){
$answer["thumb"] =
urldecode(
$this->fuckhtml
->getTextContent(
$thumb[1]
)
$this->fuckhtml
->getTextContent(
$thumb[1]
);
}
}

8
settings.php
View File

@ -133,10 +133,6 @@ $settings = [
"value" => "google",
"text" => "Google"
],
[
"value" => "google_cse",
"text" => "Google CSE"
],
[
"value" => "startpage",
"text" => "Startpage"
@ -207,10 +203,6 @@ $settings = [
"value" => "google",
"text" => "Google"
],
[
"value" => "google_cse",
"text" => "Google CSE"
],
[
"value" => "startpage",
"text" => "Startpage"