lolcat
/
4get
6
13
Fork 4
Code Issues 13 Pull Requests Packages Projects Releases Wiki Activity

fixed brave news

This commit is contained in:
lolcat 2023-11-09 08:50:53 -05:00
parent 165d80f80b
commit 9fd993b47b
1 changed files with 38 additions and 115 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
scraper/brave.php
View File

@ -1139,131 +1139,54 @@ class brave{
$proxy $proxy
); );
$news = preg_match(
$this->fuckhtml '/const data ?= ?(\[{.*}]);/',
->getElementsByClassName( $html,
"snippet inline gap-standard", $json
"div" );
);
foreach($news as $article){ if(!isset($json[1])){
$data = [ throw new Exception("Failed to grep javascript object");
"title" => null, }
"author" => null,
"description" => null,
"date" => null,
"thumb" =>
[
"url" => null,
"ratio" => null
],
"url" => null
];
$this->fuckhtml->load($article); $json = $this->fuckhtml->parseJsObject($json[1], true);
$elems =
$this->fuckhtml
->getElementsByTagName("*");
// get title if($json === null){
$data["title"] =
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByClassName(
"snippet-title",
$elems
)
[0]
["innerHTML"]
);
// get description throw new Exception("Failed to parse javascript object");
$data["description"] = }
$this->titledots(
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByClassName(
"snippet-description",
$elems
)
[0]
["innerHTML"]
)
);
// get date foreach(
$date = $json[1]["data"]["body"]["response"]["news"]["results"]
explode( as $news
"•", ){
$this->fuckhtml
->getTextContent(
$this->fuckhtml
->getElementsByClassName(
"snippet-url",
$elems
)[0]
)
);
if( if(
count($date) !== 1 && !isset($news["thumbnail"]["src"]) ||
trim($date[1]) != "" $news["thumbnail"]["src"] == "void 0"
){ ){
$data["date"] = $thumb = [
strtotime( "url" => null,
$date[1] "ratio" => null
); ];
} }else{
// get URL $thumb = [
$data["url"] = "url" => $this->unshiturl($news["thumbnail"]["src"]),
$this->fuckhtml->getTextContent(
$this->unshiturl(
$this->fuckhtml
->getElementsByClassName(
"result-header",
$elems
)
[0]
["attributes"]
["href"]
)
);
// get thumbnail
$thumb =
$this->fuckhtml
->getElementsByTagName(
"img"
);
if(
count($thumb) === 2 &&
trim(
$thumb[1]
["attributes"]
["src"]
) != ""
){
$data["thumb"] = [
"url" =>
$this->fuckhtml->getTextContent(
$this->unshiturl(
$thumb[1]
["attributes"]
["src"]
)
),
"ratio" => "16:9" "ratio" => "16:9"
]; ];
} }
$out["news"][] = $data; $out["news"][] = [
"title" => $news["title"],
"author" => null,
"description" => $news["description"],
"date" => !isset($news["age"]) || $news["age"] == "void 0" ? null : strtotime($news["age"]),
"thumb" => $thumb,
"url" => $news["url"]
];
} }
return $out; return $out;