lolcat/4get
4
2
Fork 0
Code Issues 24 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix scrape failure when using quotes on brave

Browse Source
This commit is contained in:
lolcat
2026-10-18 22:28:02 -04:00
parent 77d636a44b
commit f1a64fc539
1 changed files with 744 additions and 745 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scraper/brave.php
+39 -40
View File
@@ -212,60 +212,43 @@ class brave{
private function get_js(){
$script_disc =
$script_bodies =
$this->fuckhtml
->getElementsByTagName(
"script"
);
$data = null;
foreach($script_disc as &$discs){
$datas = [];
foreach($script_bodies as &$script){
if(
preg_match(
'/kit\.start\(/',
$discs["innerHTML"]
)
){
$jsons = explode("{type:\"data\"", $script["innerHTML"]);
array_shift($jsons);
$data =
explode(
"function(",
$discs["innerHTML"]
);
$data = $data[count($data) - 1];
$data =
explode(
"return ",
$data,
2
)[1];
break;
}
}
if($data === null){
throw new Exception("Could not grep JavaScript object");
}
foreach($jsons as $json){
$data =
$this->fuckhtml
->parseJsObject(
$this->fuckhtml
->extract_json(
$data
"{type:\"data\"" .
$json
)
);
if($data === null){
if($data !== []){
throw new Exception("Failed to decode JavaScript object");
$datas[] = $data;
}
}
}
return $data;
if(count($datas) === 0){
throw new Exception("Could not grep any JavaScript object");
}
return $datas;
}
public function web($get){
@@ -440,8 +423,11 @@ class brave{
// do some magic
$this->fuckhtml->load($html);
$data = $this->get_js();
$datas = $this->get_js();
$has_found_body = false;
foreach($datas as $data){
if(
isset($data[2]["data"]["title"]) &&
stripos($data[2]["data"]["title"], "PoW Captcha") !== false
@@ -452,9 +438,10 @@ class brave{
if(!isset($data["data"]["body"]["response"])){
throw new Exception("Brave did not return a result object");
continue;
}
$has_found_body = true;
$data = $data["data"]["body"]["response"];
/*
@@ -1123,6 +1110,12 @@ class brave{
// append discussions at position 2
array_splice($out["web"], 1, 0, $disc_out);
}
if($has_found_body === false){
throw new Exception("Brave did not return a search result object");
}
return $out;
}
@@ -1223,8 +1216,9 @@ class brave{
);
$this->fuckhtml->load($html);
$json = $this->get_js();
$jsons = $this->get_js();
foreach($jsons as $json){
foreach(
$json["data"]["response"]["news"]["results"]
as $news
@@ -1256,6 +1250,7 @@ class brave{
"url" => $news["url"]
];
}
}
return $out;
}
@@ -1309,8 +1304,9 @@ class brave{
$this->detect_captcha($html);
$json = $this->get_js();
$jsons = $this->get_js();
foreach($jsons as $json){
foreach(
$json
["data"]
@@ -1337,6 +1333,7 @@ class brave{
"url" => $result["url"]
];
}
}
return $out;
}
@@ -1442,8 +1439,9 @@ class brave{
fclose($handle);*/
$this->fuckhtml->load($html);
$json = $this->get_js();
$jsons = $this->get_js();
foreach($jsons as $json){
foreach(
$json
["data"]
@@ -1494,6 +1492,7 @@ class brave{
"url" => $result["url"]
];
}
}
return $out;
}