forked from lolcat/4get
1
0
Fork 0

added brave image+video support

This commit is contained in:
lolcat 2023-08-08 03:09:47 -04:00
parent 7c771c82c8
commit 4559857380
63 changed files with 786 additions and 307 deletions

View File

@ -81,7 +81,7 @@ $left =
<div class="code-inline">Escape</div> to exit the image viewer. <div class="code-inline">Escape</div> to exit the image viewer.
<a href="#instances"><h2 id="instances">Instances</h2></a> <a href="#instances"><h2 id="instances">Instances</h2></a>
4get is open source, anyone can create their own 4get instance! If you wish to add your website to this list, please <a href="https://lolcat.ca/contact">contact me</a>. 4get is open source, anyone can create their own 4get instance! If you wish to add your website to this list, please <a href="https://lolcat.ca">contact me</a>.
<table> <table>
<tr> <tr>
@ -102,7 +102,7 @@ $left =
<b>Message to all DMCA enforcers:</b> I don\'t host any of the content. Everything you see here is <u>proxied</u> trough my shitbox with no moderation. Please reach out to the people hosting the infringing content instead.<br><br> <b>Message to all DMCA enforcers:</b> I don\'t host any of the content. Everything you see here is <u>proxied</u> trough my shitbox with no moderation. Please reach out to the people hosting the infringing content instead.<br><br>
<a href="https://lolcat.ca/contact" rel="dofollow" class="link">Click here to contact me!</a><br><br> <a href="https://lolcat.ca" rel="dofollow" class="link">Click here to contact me!</a><br><br>
<a href="https://validator.w3.org/nu/?doc=https%3A%2F%2F4get.ca" title="W3 Valid!"> <a href="https://validator.w3.org/nu/?doc=https%3A%2F%2F4get.ca" title="W3 Valid!">
<img src="/static/icon/w3html.png" alt="Valid W3C HTML 4.01" width="88" height="31"> <img src="/static/icon/w3html.png" alt="Valid W3C HTML 4.01" width="88" height="31">

View File

@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
try{ try{
echo json_encode( echo json_encode(
$scraper->image($get) $scraper->image($get),
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
); );
}catch(Exception $e){ }catch(Exception $e){

View File

@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
try{ try{
echo json_encode( echo json_encode(
$scraper->news($get) $scraper->news($get),
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
); );
}catch(Exception $e){ }catch(Exception $e){

View File

@ -16,7 +16,8 @@ $get = $frontend->parsegetfilters($_GET, $filters);
try{ try{
echo json_encode( echo json_encode(
$scraper->video($get) $scraper->video($get),
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
); );
}catch(Exception $e){ }catch(Exception $e){

View File

@ -21,7 +21,8 @@ if(!isset($_GET["extendedsearch"])){
try{ try{
echo json_encode( echo json_encode(
$scraper->web($get) $scraper->web($get),
JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES
); );
}catch(Exception $e){ }catch(Exception $e){

Binary file not shown.

Before

Width:  |  Height:  |  Size: 84 KiB

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 42 KiB

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 9.4 KiB

After

Width:  |  Height:  |  Size: 9.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.5 KiB

After

Width:  |  Height:  |  Size: 6.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.8 KiB

After

Width:  |  Height:  |  Size: 6.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 6.1 KiB

After

Width:  |  Height:  |  Size: 6.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.9 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.6 KiB

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 17 KiB

After

Width:  |  Height:  |  Size: 5.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 5.1 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 11 KiB

After

Width:  |  Height:  |  Size: 11 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 70 KiB

After

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 33 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 51 KiB

After

Width:  |  Height:  |  Size: 13 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 22 KiB

After

Width:  |  Height:  |  Size: 6.5 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 46 KiB

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 68 KiB

After

Width:  |  Height:  |  Size: 20 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 62 KiB

After

Width:  |  Height:  |  Size: 59 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 66 KiB

After

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 59 KiB

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 34 KiB

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

After

Width:  |  Height:  |  Size: 753 B

View File

@ -62,8 +62,13 @@ foreach($results["image"] as $image){
'<div class="image-wrapper" title="' . htmlspecialchars($image["title"]) .'" data-json="' . htmlspecialchars(json_encode($image["source"])) . '">' . '<div class="image-wrapper" title="' . htmlspecialchars($image["title"]) .'" data-json="' . htmlspecialchars(json_encode($image["source"])) . '">' .
'<div class="image">' . '<div class="image">' .
'<a href="' . htmlspecialchars($image["source"][0]["url"]) . '" rel="noreferrer nofollow" class="thumb">' . '<a href="' . htmlspecialchars($image["source"][0]["url"]) . '" rel="noreferrer nofollow" class="thumb">' .
'<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "thumb") . '" alt="thumbnail">' . '<img src="' . $frontend->htmlimage($image["source"][count($image["source"]) - 1]["url"], "thumb") . '" alt="thumbnail">';
'<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>' .
if($image["source"][0]["width"] !== null){
$payload["images"] .= '<div class="duration">' . $image["source"][0]["width"] . 'x' . $image["source"][0]["height"] . '</div>';
}
$payload["images"] .=
'</a>' . '</a>' .
'<a href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow">' . '<a href="' . htmlspecialchars($image["url"]) . '" rel="noreferrer nofollow">' .
'<div class="title">' . htmlspecialchars(parse_url($image["url"], PHP_URL_HOST)) . '</div>' . '<div class="title">' . htmlspecialchars(parse_url($image["url"], PHP_URL_HOST)) . '</div>' .

View File

@ -892,6 +892,7 @@ class frontend{
"option" => [ "option" => [
"ddg" => "DuckDuckGo", "ddg" => "DuckDuckGo",
"yandex" => "Yandex", "yandex" => "Yandex",
"brave" => "Brave"//,
//"google" => "Google" //"google" => "Google"
] ]
]; ];
@ -903,6 +904,7 @@ class frontend{
"option" => [ "option" => [
"yt" => "YouTube", "yt" => "YouTube",
"ddg" => "DuckDuckGo", "ddg" => "DuckDuckGo",
"brave" => "Brave"//,
//"google" => "Google" //"google" => "Google"
] ]
]; ];
@ -1285,7 +1287,7 @@ class frontend{
return htmlspecialchars($image); return htmlspecialchars($image);
} }
return "/proxy?i=" . urlencode($image) . "&s=" . $format; return "/proxy.php?i=" . urlencode($image) . "&s=" . $format;
} }
public function htmlnextpage($gets, $npt, $page){ public function htmlnextpage($gets, $npt, $page){

View File

@ -356,6 +356,91 @@ class fuckhtml{
return $out; return $out;
} }
public function parseJsObject(string $json){
$bracket = false;
$is_close_bracket = false;
$escape = false;
$json_out = null;
$last_char = null;
for($i=0; $i<strlen($json); $i++){
switch($json[$i]){
case "\"":
case "'":
if($escape === true){
break;
}
if($json[$i] == $bracket){
$bracket = false;
$is_close_bracket = true;
}else{
if($bracket === false){
$bracket = $json[$i];
}
}
break;
default:
$is_close_bracket = false;
break;
}
$escape = $json[$i] == "\\" ? true : false;
if(
$bracket === false &&
$is_close_bracket === false
){
// here we know we're not iterating over a quoted string
switch($json[$i]){
case "[":
case "{":
// dont execute whats in "default"
$json_out .= $json[$i];
break;
case "]":
case "}":
case ",":
case ":":
if(!in_array($last_char, ["[", "{", "}", "]", "\""])){
$json_out .= "\"";
}
$json_out .= $json[$i];
break;
default:
if(in_array($last_char, ["{", "[", ",", ":"])){
$json_out .= "\"";
}
$json_out .= $json[$i];
break;
}
}else{
$json_out .= $json[$i];
}
$last_char = $json[$i];
}
return json_decode($json_out, true);
}
} }
?> ?>

View File

@ -57,6 +57,7 @@ try{
} }
$image->readImageBlob($payload["body"]); $image->readImageBlob($payload["body"]);
$image_width = $image->getImageWidth(); $image_width = $image->getImageWidth();
$image_height = $image->getImageHeight(); $image_height = $image->getImageHeight();
@ -102,16 +103,16 @@ try{
$image_width = $image_height * $ratio; $image_width = $image_height * $ratio;
} }
$image->setImageBackgroundColor(new ImagickPixel("#504945")); $image->setImageBackgroundColor("#504945");
$image->mergeImageLayers(Imagick::LAYERMETHOD_FLATTEN); $image->setImageAlphaChannel(Imagick::ALPHACHANNEL_REMOVE);
$image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1);
$image->stripImage(); $image->stripImage();
$image->setFormat("jpeg"); $image->setFormat("jpeg");
$image->setImageCompressionQuality(90); $image->setImageCompressionQuality(90);
$image->setImageCompression(Imagick::COMPRESSION_JPEG2000); $image->setImageCompression(Imagick::COMPRESSION_JPEG2000);
$image->resizeImage($image_width, $image_height, Imagick::FILTER_LANCZOS, 1);
$proxy->getfilenameheader($payload["headers"], $_GET["i"]); $proxy->getfilenameheader($payload["headers"], $_GET["i"]);
header("Content-Type: image/jpeg"); header("Content-Type: image/jpeg");

View File

@ -86,6 +86,8 @@ class brave{
]; ];
break; break;
case "images":
case "videos":
case "news": case "news":
return [ return [
"country" => [ "country" => [
@ -143,7 +145,7 @@ class brave{
} }
} }
private function get($url, $get = [], $nsfw, $country/*, $is_post = false, $additional_cookies = null*/){ private function get($url, $get = [], $nsfw, $country){
switch($nsfw){ switch($nsfw){
@ -152,13 +154,6 @@ class brave{
case "no": $nsfw = "strict"; break; case "no": $nsfw = "strict"; break;
} }
//$cookie = "safesearch={$nsfw}; country={$country}; useLocation=0";
/*
if($additional_cookies !== null){
$cookie = $additional_cookies . "; " . $cookie;
}*/
$headers = [ $headers = [
"User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0", "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:107.0) Gecko/20100101 Firefox/110.0",
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
@ -171,8 +166,7 @@ class brave{
"Sec-Fetch-Dest: document", "Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate", "Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none", "Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1"//, "Sec-Fetch-User: ?1"
//"Content-Type: application/json"
]; ];
if($country == "any"){ if($country == "any"){
@ -182,22 +176,10 @@ class brave{
$curlproc = curl_init(); $curlproc = curl_init();
/*if($is_post){ if($get !== []){
$get = http_build_query($get);
curl_setopt($curlproc, CURLOPT_POST, true); $url .= "?" . $get;
curl_setopt( }
$curlproc,
CURLOPT_POSTFIELDS,
json_encode($get)
);
}else{
*/
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
//}
curl_setopt($curlproc, CURLOPT_URL, $url); curl_setopt($curlproc, CURLOPT_URL, $url);
@ -1950,18 +1932,24 @@ class brave{
return $out; return $out;
} }
/* public function image($get){
public function bypasscaptcha($html, $nsfw, $country){
// @TODO figure out why I still cant go trough $search = $get["s"];
// the captcha wall even after breaking it $country = $get["country"];
$nsfw = $get["nsfw"];
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
try{ try{
$html = $html =
$this->get( $this->get(
"https://search.brave.com/goggles", "https://search.brave.com/images",
[ [
"q" => "site:dailymotion.com my bloody valentine" "q" => $search
], ],
$nsfw, $nsfw,
$country $country
@ -1969,177 +1957,192 @@ class brave{
}catch(Exception $error){ }catch(Exception $error){
throw new Exception("Could not fetch html"); throw new Exception("Could not fetch search page");
} }
/*
$handle = fopen("scraper/brave-image.html", "r");
$html = fread($handle, filesize("scraper/brave-image.html"));
fclose($handle);*/
// Bypass brave search captcha
// this captcha only appears on the goggles page
preg_match( preg_match(
'/this\.img\.src = &#34;(.*)&#34;/', '/const data = (\[{.*}\]);/',
$html, $html,
$image $json
); );
$image = if(!isset($json[1])){
base64_decode(
explode( throw new Exception("Failed to get data object");
"data:image/png;base64,", }
$image[1]
)[1] $json =
$this->fuckhtml
->parseJsObject(
$json[1]
); );
$im = new Imagick();
$im->readImageBlob($image);
$im->blurImage(20, 20);
$im->posterizeImage(2, imagick::IMGTYPE_COLORSEPARATION);
// if we encounter a white line thats longer than 45px
// we found the circle position
$iterator = $im->getPixelRegionIterator(0, 77, 310, 1);
$found = null;
foreach( foreach(
$iterator as $row $json[1]
["data"]
["body"]
["response"]
["results"]
as $result
){ ){
$whitecount = 0; $out["image"][] = [
$count = 0; "title" => $result["title"],
"source" => [
foreach($row as $pixel){ [
"url" => $result["properties"]["url"],
if($pixel->getColor()["r"] === 255){ "width" => null,
"height" => null
$whitecount++; ],
$pixel->setColor("rgba(255,0,0,0)"); [
"url" => $result["thumbnail"]["src"],
if($whitecount === 45){ "width" => null,
"height" => null
$found = $count - 45;
break 2;
}
}else{
$whitecount = 0;
}
$count++;
$iterator->syncIterator();
}
}
$found = $found + 10;
//header("Content-Type: image/png");
//echo $im;
//die();
if($found === null){
throw new Exception("Could not bypass captcha");
}
preg_match(
'/data="{&#34;captcha_id&#34;:&#34;([0-9A-z-]+)&#34;}"/',
$html,
$key
);
$key = $key[1];
// we bypassed captcha, send POST data
$order =
$this->get(
"https://search.brave.com/api/captcha?brave=0&captcha_id={$key}",
[
"solution" => (string)$found
],
$nsfw,
$country,
true
);
$order = json_decode($order, true)["orderId"];
$orderpayload =
$this->get(
"https://search.brave.com/api/rewards/v1/orders/{$order}",
[],
$nsfw,
$country
);
$orderpayload = json_decode($orderpayload, true);
$creds =
$this->get(
"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
[
"itemId" => $orderpayload["items"][0]["id"],
"blindedCreds" => [
"fuYAVcB/m7BU66vf3wkNGxJCSaRhshB9o+8km3F1h2c=",
"uswvcWJuPK/1qFlVdzBP3eQd0+V1EQgfAtnEoMIK+Uk=",
"fJWKGLBxl3Gyn4n9FjTLq1PjupfABT7Ni8MeB+iGzUs=",
"Aq9enJ/VZP9GxQIza3n65ZK7xQhY4VwDxv53BCb/Txg=",
"FMJA9eSLHq71K+Pcwgm4gIQOmdR/6KMy5cMgXhpd5Ro=",
"2NVhIAbvI317SP9/xXbVe/U57eWgvHyqVbHL/5+Gdmw=",
"6mpjsjSCmYEzK2xlbL8DI2P4LuhWUOxjTLvsTAL9l24=",
"kAn4wuHvIlKWhfuFfPTSfD4tZ5le9t7/61YbdEc/L3k=",
"BjjUyG16aTfd1c0h4oBzgQQOekrH1f+a5CmcXqMPTR4=",
"SBNgpCt4/V44yaQTfh+D027Yv1GJFHkjUEpPw6rAwRI=",
"XDENAtdQ7PyYx+Qx1wQGQtDWgg8WpIMgWGmd4RDOVWE=",
"tF7rB4sqamsiUk3K7fojdQSI0Q6iip72yKyhnvg/bC0=",
"VsAqflirAd/u4VsLdfRS2UvnH24ZNkFh6YN3DctLjzQ=",
"MntLbXkoI0LdcisCbNazmooiHXJyX91L1KERDAu1JRU=",
"TH6Zs8JBvFDbTDWgKbfGE4M5/cSwCtHD8ms5Y/U8zHQ=",
"jsZg0Z+qDPHymrbhdnesodhLNJ26QdunyMko1aVe4So=",
"rpKsyj6/vdnuMgLI2BApeijtGq9g5USRDL0w6X2bnlQ=",
"vCzliGT8A9vcLXj2sFf2kavOuYw69d70NpfgA22B4lI=",
"7OWoxSCtYXWcaBSifF7AXNBif/sjcuO0IelzXG/3PFk=",
"iiXtByNlT6nDMN9De5B58Jl8J0p6LCjnZ9aS3w2FEQU=",
"zDhd7gsJ4h4JkDeGK0Y0mfFd8IBdkLhMOANzwO+4Dig=",
"qANZ+AikwFReEA61JF009d/c3IHM/aSfIYwljckhJWE=",
"nNC30pDLxtXvUr+WDwfDSrAInNBpfSZkPsV2JlpheWI=",
"kGXE1pkt25P71kdJzmKIg4+yMR1VA5wNmbpBb/FhJQ8=",
"aLqPsY1Qiz2UCa2Jx3YNNt8r4JINMphks/43EiyZfXU=",
"bHGYZoQARZEM5LdFF6B74PkRqNd9EKxzuTvGYxjq+hk=",
"JOsYQjfE/9Y1u29hR+GvEkNyxUI8blgLhX1iJI/aGRQ=",
"yKjHjH5j600TJD/3WPsA1N3OmItDLifdjlysq4H6NV0=",
"9lTnUbsPp7BJ7XVN5/T4yGfzD9DJdqWB7xk72s19MAA=",
"5KHG8iY45em7zDhO/HlI0ydcZ0Ubn+XSyjifMmy7qXM="
] ]
], ],
$nsfw, "url" => $result["url"]
$country, ];
true }
return $out;
}
public function video($get){
$search = $get["s"];
$country = $get["country"];
$nsfw = $get["nsfw"];
$out = [
"status" => "ok",
"npt" => null,
"video" => [],
"author" => [],
"livestream" => [],
"playlist" => [],
"reel" => []
];
try{
$html =
$this->get(
"https://search.brave.com/videos",
[
"q" => $search
],
$nsfw,
$country
);
}catch(Exception $error){
throw new Exception("Could not fetch search page");
}
/*
$handle = fopen("scraper/brave-video.html", "r");
$html = fread($handle, filesize("scraper/brave-video.html"));
fclose($handle);*/
preg_match(
'/const data = (\[{.*}\]);/',
$html,
$json
);
if(!isset($json[1])){
throw new Exception("Failed to get data object");
}
$json =
$this->fuckhtml
->parseJsObject(
$json[1]
); );
var_dump($creds); foreach(
$json
[1]
["data"]
["body"]
["response"]
["results"]
as $result
){
if($result["video"]["author"] != "null"){
$author = [
"name" => $result["video"]["author"]["name"] == "null" ? null : $result["video"]["author"]["name"],
"url" => $result["video"]["author"]["url"] == "null" ? null : $result["video"]["author"]["url"],
"avatar" => $result["video"]["author"]["img"] == "null" ? null : $result["video"]["author"]["img"]
];
}else{
$author = [
"name" => null,
"url" => null,
"avatar" => null
];
}
if($result["thumbnail"] != "null"){
$thumb = [
"url" => $result["thumbnail"]["original"],
"ratio" => "16:9"
];
}else{
$thumb = [
"url" => null,
"ratio" => null
];
}
$out["video"][] = [
"title" => $result["title"],
"description" => $result["description"] == "null" ? null : $this->titledots($result["description"]),
"author" => $author,
"date" => $result["age"] == "null" ? null : strtotime($result["age"]),
"duration" => $result["video"]["duration"] == "null" ? null : $this->hms2int($result["video"]["duration"]),
"views" => $result["video"]["views"] == "null" ? null : (int)$result["video"]["views"],
"thumb" => $thumb,
"url" => $result["url"]
];
}
return $out;
}
private function hms2int($time){
sleep(2); $parts = explode(":", $time, 3);
$test = $time = 0;
$this->get(
"https://search.brave.com/api/rewards/v1/orders/{$order}/credentials",
[],
$nsfw,
$country
);
var_dump($test); if(count($parts) === 3){
// hours
$time = $time + ((int)$parts[0] * 3600);
array_shift($parts);
}
$html = if(count($parts) === 2){
$this->get(
"https://search.brave.com/goggles", // minutes
[ $time = $time + ((int)$parts[0] * 60);
"q" => "site:dailymotion.com my bloody valentine" array_shift($parts);
], }
$nsfw,
$country,
false,
"__Secure-sku#brave-search-captcha=eyJ0eXBlIjoic2luZ2xlLXVzZSIsInZlcnNpb24iOjEsInNrdSI6ImJyYXZlLXNlYXJjaC1jYXB0Y2hhIiwicHJlc2VudGF0aW9uIjoiZXlKcGMzTjFaWElpT2lKaWNtRjJaUzVqYjIwL2MydDFQV0p5WVhabExYTmxZWEpqYUMxallYQjBZMmhoSWl3aWMybG5ibUYwZFhKbElqb2lNRzl0VDBneWQxZ3dTazkzU0VFMVJ6QTJaR1V5WjFOQ1dDdGhSM3B2Y2xsTVQwVTJZVVJtTUc5a1IweG1Wa3RhZEd0cU4xbHdia3BPT0VOVGNGbE5lVWR2YmpGRlNTOUhhMlZYU1RWNGQxTjJPWGxJTTNjOVBTSXNJblFpT2lKWlJWWldaVzR5TTJwQ01tSnZkakJ2U1hGNGJtSndUMGxEUW5Kd1drRjBRbWQxVnpoRlNURTNVREY2UVRaQlpUTXJSVGRFYm5NeVFqUmhka0pGYTFWM2FGY3JWRVZJVjNWcE9TdFllRU1yYlVSTVkyMTBRVDA5SW4wPSJ9"
);
var_dump($html); // seconds
}*/ $time = $time + (int)$parts[0];
return $time;
}
private function appendtext($payload, &$text, &$index){ private function appendtext($payload, &$text, &$index){

View File

@ -808,6 +808,7 @@ class google{
->getElementsByTagName("style"); ->getElementsByTagName("style");
$this->computedstyle = []; $this->computedstyle = [];
$this->ask = [];
foreach($styles as $style){ foreach($styles as $style){
@ -860,6 +861,22 @@ class google{
$image_grep[1][0] $image_grep[1][0]
); );
} }
// even more javascript crap
// "People also ask" node is loaded trough javascript
preg_match_all(
'/window\.jsl\.dh\(\'([^\']+)\',\'(.+)\'\);/',
$script["innerHTML"],
$ask_grep
);
for($i=0; $i<count($ask_grep[0]); $i++){
$this->ask[trim($ask_grep[1][$i])] =
stripcslashes(
$ask_grep[2][$i]
);
}
} }
// get nodes // get nodes
@ -926,22 +943,22 @@ class google{
"div" "div"
); );
$carousel_title =
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"font-size" => "16px",
"line-height" => "20px",
"font-weight" => "400"
],
self::is_class
),
"div"
);
if(count($carousel) !== 0){ if(count($carousel) !== 0){
$carousel_title =
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"font-size" => "16px",
"line-height" => "20px",
"font-weight" => "400"
],
self::is_class
),
"div"
);
$sublink = []; // twitter carousel sublinks $sublink = []; // twitter carousel sublinks
foreach($carousel as $item){ foreach($carousel as $item){
@ -1212,6 +1229,136 @@ class google{
continue; continue;
} }
$people_title =
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"font-weight" => "bold",
"font-size" => "16px",
"color" => "#000",
"margin" => "0",
"padding" => "12px 16px 0 16px"
],
self::is_class
),
"div"
);
if(
count($people_title) !== 0 &&
strtolower(
$this->fuckhtml
->getTextContent(
$people_title[0]
)
) == "people also ask"
){
/*
Parse "people also ask" node
*/
$div =
$this->fuckhtml
->getElementsByTagName("div");
// add suggestions
$suggestions =
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"display" => "inline-block",
"padding-right" => "26px"
],
self::is_class
),
$div
);
foreach($suggestions as $suggestion){
$out["related"][] =
$this->fuckhtml
->getTextContent($suggestion);
}
// parse websites
foreach($div as $d){
if(
isset($d["attributes"]["id"]) &&
strpos(
$d["attributes"]["id"],
"accdef_"
) !== false
){
$this->fuckhtml->load(
$this->ask[
$d["attributes"]["id"]
]
);
$description =
$this->titledots(
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"white-space" => "pre-line",
"word-wrap" => "break-word"
],
self::is_class
),
"div"
)[0]
)
);
$a =
$this->fuckhtml
->getElementsByTagName("a")
[0];
$this->fuckhtml->load($a);
$out["web"][] = [
"title" =>
$this->titledots(
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByTagName("span")[0]
)
),
"description" => $description,
"url" =>
$this->decodeurl(
$this->fuckhtml
->getTextContent(
$a
["attributes"]
["href"]
)
),
"date" => null,
"type" => "web",
"thumb" => [
"url" => null,
"ratio" => null
],
"sublink" => [],
"table" => []
];
}
}
continue;
}
if(count($title) !== 0){ if(count($title) !== 0){
/* /*
@ -1231,6 +1378,19 @@ class google{
"url" => $this->getimage($thumb[0]["attributes"]["id"]), "url" => $this->getimage($thumb[0]["attributes"]["id"]),
"ratio" => "1:1" "ratio" => "1:1"
]; ];
if(parse_url($thumb["url"], PHP_URL_HOST) == "i.ytimg.com"){
$thumb = [
"url" =>
str_replace(
"default.jpg",
"maxresdefault.jpg",
$thumb["url"]
),
"ratio" => "16:9"
];
}
}else{ }else{
$thumb = [ $thumb = [
@ -1287,18 +1447,33 @@ class google{
$cat = explode(":", $cat, 2); $cat = explode(":", $cat, 2);
$table[ $name =
$this->fuckhtml $this->fuckhtml
->getTextContent( ->getTextContent(
$cat[0] $cat[0]
)
] =
$this->titledots(
$this->fuckhtml
->getTextContent(
$cat[1]
)
); );
if(strtolower($name) != "posted"){
$table[$name] =
$this->titledots(
$this->fuckhtml
->getTextContent(
$cat[1]
)
);
}else{
$date =
strtotime(
$this->titledots(
$this->fuckhtml
->getTextContent(
$cat[1]
)
)
);
}
} }
continue; continue;
} }
@ -1307,6 +1482,7 @@ class google{
$this->fuckhtml $this->fuckhtml
->getElementsByTagName("span"); ->getElementsByTagName("span");
$encounter_rating = false;
foreach($spans as $span){ foreach($spans as $span){
// replace element with nothing // replace element with nothing
@ -1319,10 +1495,53 @@ class google{
); );
} }
if($encounter_rating !== false){
switch($encounter_rating){
case 3:
$table["Votes"] =
number_format(
str_replace(
[
"(",
")",
","
],
"",
$this->fuckhtml
->getTextContent(
$span["innerHTML"]
)
)
);
break;
case 6:
$table["Price"] =
$this->fuckhtml
->getTextContent(
$span["innerHTML"]
);
break;
case 8:
$table["Support"] =
$this->fuckhtml
->getTextContent(
$span["innerHTML"]
);
break;
}
$encounter_rating++;
}
// get rating // get rating
if(isset($span["attributes"]["aria-hidden"])){ if(isset($span["attributes"]["aria-hidden"])){
$table["Rating"] = $span["innerHTML"]; $table["Rating"] = $span["innerHTML"];
$encounter_rating = 0;
continue; continue;
} }
} }
@ -1565,16 +1784,7 @@ class google{
} }
/* /*
Detect if its a wikipedia thing Parse instant answers with parts
*/
$h3 =
$this->fuckhtml
->getElementsByTagName("h3");
/*
Fallback to parsing the word definitions
*/ */
$parts = $parts =
$this->fuckhtml $this->fuckhtml
@ -1588,15 +1798,8 @@ class google{
"div" "div"
); );
if(count($parts) === 0){ if(count($parts) !== 0){
continue;
}
$head = $parts[0];
if(count($h3) !== 0){
$table = [ $table = [
"title" => null, "title" => null,
"description" => [], "description" => [],
@ -1606,30 +1809,130 @@ class google{
"sublink" => [] "sublink" => []
]; ];
$h3 = $h3[0]; // get thumb
$thumb =
$table["title"] =
$this->fuckhtml $this->fuckhtml
->getTextContent( ->getElementsByClassName(
$h3 $this->findstyles(
[
"float" => "right",
"padding-left" => "16px"
],
self::is_class
),
"div"
);
if(count($thumb) !== 0){
$this->fuckhtml->load($thumb[0]);
$img =
$this->fuckhtml
->getElementsByTagName("img");
if(count($img) !== 0){
$table["thumb"] =
$this->getimage(
$img[0]["attributes"]["id"]
);
}
$this->fuckhtml->load($container);
}
$h =
$this->fuckhtml
->getElementsByTagName("h3");
if(count($h) === 0){
$h =
$this->fuckhtml
->getElementsByTagName("h2");
}
if(count($h) !== 0){
// set title + subtext for when a word definition
// appears
$h = $h[0];
$table["title"] =
$this->fuckhtml
->getTextContent(
$h
);
$parts[0]["innerHTML"] =
str_replace(
$h["outerHTML"],
"",
$parts[0]["innerHTML"]
);
$table["description"][] =
[
"type" => "quote",
"value" =>
$this->fuckhtml
->getTextContent(
$parts[0]
)
];
}else{
// parse it as a wikipedia header
}
// get table elements
$tables =
$this->fuckhtml
->getElementsByClassName(
$this->findstyles(
[
"display" => "table",
"width" => "100%",
"padding-right" => "16px",
"-webkit-box-sizing" => "border-box"
],
self::is_class
),
"div"
); );
$head["innerHTML"] = foreach($tables as $tbl){
str_replace(
$h3["outerHTML"], $this->fuckhtml->load($tbl);
"",
$head["innerHTML"] $images =
); $this->fuckhtml
->getElementsByTagName("img");
$table["description"][] =
[ if(count($images) !== 0){
"type" => "quote",
"value" => $image = $this->getimage($images[0]["attributes"]["id"]);
$text =
$this->fuckhtml $this->fuckhtml
->getTextContent( ->getTextContent(
$head $tbl
) );
];
$table["description"][] = [
"type" => "link",
"value" => $text,
"url" => "?s=" . urlencode($text) . "&scraper=google"
];
$table["description"][] = [
"type" => "image",
"url" => $image
];
}
}
$audio = $audio =
$this->fuckhtml $this->fuckhtml
@ -1828,9 +2131,9 @@ class google{
} }
} }
} }
$out["answer"][] = $table;
} }
$out["answer"][] = $table;
} }
if($dmca_table){ if($dmca_table){
@ -2136,20 +2439,65 @@ class google{
$match $match
); );
if(count($match) !== 0){ if(count($match) === 0){
if(!empty($match[1])){ return null;
return urldecode($match[1]);
}
if(!empty($match[2])){
return urldecode($match[2]);
}
} }
return null; $url = empty($match[1]) ? urldecode($match[2]) : urldecode($match[1]);
$domain = parse_url($url, PHP_URL_HOST);
if(
preg_match(
'/wikipedia.org$/',
$domain
)
){
// rewrite wikipedia mobile URLs to desktop
$url =
$this->replacedomain(
$url,
preg_replace(
'/([a-z0-9]+)(\.m\.)/',
'$1.',
$domain
)
);
}
if(
preg_match(
'/imdb\.com$|youtube\.[^.]+$/',
$domain
)
){
// rewrite imdb and youtube mobile URLs too
$url =
$this->replacedomain(
$url,
preg_replace(
'/^m\./',
"",
$domain
)
);
}
return $url;
}
private function replacedomain($url, $domain){
return
preg_replace(
'/(https?:\/\/)([^\/]+)/',
'$1' . $domain,
$url
);
} }
private function titledots($title){ private function titledots($title){

View File

@ -909,6 +909,23 @@ class mojeek{
$a = $a[0]; $a = $a[0];
$date =
explode(
" - ",
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByTagName(
"span"
)[0]
)
);
$date =
strtotime(
$date[count($date) - 1]
);
$out["news"][] = [ $out["news"][] = [
"title" => "title" =>
html_entity_decode( html_entity_decode(
@ -918,20 +935,7 @@ class mojeek{
) )
), ),
"description" => null, "description" => null,
"date" => "date" => $date,
strtotime(
explode(
" - ",
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByTagName(
"span"
)[0]
),
2
)[1]
),
"thumb" => [ "thumb" => [
"url" => null, "url" => null,
"ratio" => null "ratio" => null

View File

@ -70,10 +70,10 @@ $settings = [
"value" => "brave", "value" => "brave",
"text" => "Brave" "text" => "Brave"
], ],
//[ /*[
// "value" => "google", "value" => "google",
// "text" => "Google" "text" => "Google"
//], ],*/
[ [
"value" => "mojeek", "value" => "mojeek",
"text" => "Mojeek" "text" => "Mojeek"
@ -99,11 +99,15 @@ $settings = [
[ [
"value" => "yandex", "value" => "yandex",
"text" => "Yandex" "text" => "Yandex"
]//, ],
//[ [
// "value" => "google", "value" => "brave",
// "text" => "Google" "text" => "Brave"
//] ]/*,
[
"value" => "google",
"text" => "Google"
]*/
] ]
], ],
[ [
@ -117,11 +121,15 @@ $settings = [
[ [
"value" => "ddg", "value" => "ddg",
"text" => "DuckDuckGo" "text" => "DuckDuckGo"
]//, ],
//[ [
// "value" => "google", "value" => "brave",
// "text" => "Google" "text" => "Brave"
//] ]/*,
[
"value" => "google",
"text" => "Google"
]*/
] ]
], ],
[ [
@ -136,10 +144,10 @@ $settings = [
"value" => "brave", "value" => "brave",
"text" => "Brave" "text" => "Brave"
], ],
//[ /*[
// "value" => "google", "value" => "google",
// "text" => "Google" "text" => "Google"
//], ],*/
[ [
"value" => "mojeek", "value" => "mojeek",
"text" => "Mojeek" "text" => "Mojeek"
@ -219,7 +227,7 @@ echo
'<head>' . '<head>' .
'<meta http-equiv="Content-Type" content="text/html;charset=utf-8">' . '<meta http-equiv="Content-Type" content="text/html;charset=utf-8">' .
'<title>Settings</title>' . '<title>Settings</title>' .
'<link rel="stylesheet" href="/static/style.css?v2">' . '<link rel="stylesheet" href="/static/style.css">' .
'<meta name="viewport" content="width=device-width,initial-scale=1">' . '<meta name="viewport" content="width=device-width,initial-scale=1">' .
'<meta name="robots" content="index,follow">' . '<meta name="robots" content="index,follow">' .
'<link rel="icon" type="image/x-icon" href="/favicon.ico">' . '<link rel="icon" type="image/x-icon" href="/favicon.ico">' .

View File

@ -360,6 +360,23 @@ if(image_class !== null){
elem.getAttribute("data-json") elem.getAttribute("data-json")
); );
var imagesize = elem.getElementsByTagName("img")[0];
if(imagesize.complete){
var imagesize_w = imagesize.naturalWidth;
var imagesize_h = imagesize.naturalHeight;
}
for(var i=0; i<collection.length; i++){
if(collection[i].width === null){
collection[i].width = imagesize_w;
collection[i].height = imagesize_h;
}
}
var title = elem.title; var title = elem.title;
} }

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 574 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.4 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.2 KiB

After

Width:  |  Height:  |  Size: 511 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.4 KiB

After

Width:  |  Height:  |  Size: 1.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.6 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.2 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.9 KiB

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.7 KiB

After

Width:  |  Height:  |  Size: 841 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.0 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.1 KiB

After

Width:  |  Height:  |  Size: 468 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 585 B

After

Width:  |  Height:  |  Size: 354 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 602 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

After

Width:  |  Height:  |  Size: 689 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.8 KiB

After

Width:  |  Height:  |  Size: 629 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 698 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1000 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 1.8 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 1.3 KiB

After

Width:  |  Height:  |  Size: 1.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.3 KiB

After

Width:  |  Height:  |  Size: 1.3 KiB

View File

@ -14,7 +14,9 @@
<div id="center"> <div id="center">
<form method="GET" autocomplete="off" action="web"> <form method="GET" autocomplete="off" action="web">
<div class="logo"> <div class="logo">
<img src="{%banner%}" alt="4get"> <a href="/">
<img src="{%banner%}" alt="4get">
</a>
</div> </div>
<div class="searchbox"> <div class="searchbox">
<input type="submit" value="Search" tabindex="-1"> <input type="submit" value="Search" tabindex="-1">
@ -28,9 +30,9 @@
<div class="subtext"> <div class="subtext">
Clearnet: <a href="https://4get.ca">4get.ca</a><br> Clearnet: <a href="https://4get.ca">4get.ca</a><br>
Tor: <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion</a><br> Tor: <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion</a><br>
Report a problem: <a href="https://lolcat.ca/contact">lolcat.ca/contact</a> Report a problem: <a href="https://lolcat.ca">lolcat.ca</a>
</div> </div>
</div> </div>
<script src="/static/client.js?v2"></script> <script src="/static/client.js?v3"></script>
</body> </body>
</html> </html>

View File

@ -2,6 +2,6 @@
{%images%} {%images%}
</div> </div>
{%nextpage%} {%nextpage%}
<script src="/static/client.js?v2"></script> <script src="/static/client.js?v3"></script>
</body> </body>
</html> </html>

View File

@ -11,6 +11,6 @@
{%left%} {%left%}
</div> </div>
</div> </div>
<script src="/static/client.js?v2"></script> <script src="/static/client.js?v3"></script>
</body> </body>
</html> </html>