scrape google sublinks, which are encrypted pieces of shit
This commit is contained in:
+44
-2
@@ -733,7 +733,7 @@ class google{
|
|||||||
$container = &$data["container"];
|
$container = &$data["container"];
|
||||||
|
|
||||||
//file_put_contents("scraper/google-jp.html", $html);
|
//file_put_contents("scraper/google-jp.html", $html);
|
||||||
//$html = file_get_contents("scraper/dmca.html");
|
//$html = file_get_contents("scraper/sublink.html");
|
||||||
//$container = "";
|
//$container = "";
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -772,9 +772,10 @@ class google{
|
|||||||
$html = &$data2["data"];
|
$html = &$data2["data"];
|
||||||
|
|
||||||
$this->fuckhtml->load($html);
|
$this->fuckhtml->load($html);
|
||||||
$this->parsestyles();
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
$this->parsestyles();
|
||||||
|
|
||||||
$this->scrape_dimg($html);
|
$this->scrape_dimg($html);
|
||||||
$this->scrape_imagearr($html);
|
$this->scrape_imagearr($html);
|
||||||
|
|
||||||
@@ -1269,6 +1270,47 @@ class google{
|
|||||||
|
|
||||||
$sublinks = [];
|
$sublinks = [];
|
||||||
|
|
||||||
|
// get sublinks (rare)
|
||||||
|
$as =
|
||||||
|
$this->fuckhtml
|
||||||
|
->getElementsByClassName(
|
||||||
|
$this->getstyle([
|
||||||
|
"padding-right" => "6px",
|
||||||
|
"margin-top" => "4px",
|
||||||
|
"display" => "inline-block"
|
||||||
|
]),
|
||||||
|
"a"
|
||||||
|
);
|
||||||
|
|
||||||
|
foreach($as as $sublink){
|
||||||
|
|
||||||
|
$sublink_url =
|
||||||
|
$this->fuckhtml
|
||||||
|
->getTextContent(
|
||||||
|
$sublink["attributes"]["href"]
|
||||||
|
);
|
||||||
|
|
||||||
|
if(
|
||||||
|
preg_match(
|
||||||
|
'/^\/goto\?url=(.*)/',
|
||||||
|
$sublink_url,
|
||||||
|
$piece_of_shit
|
||||||
|
)
|
||||||
|
){
|
||||||
|
|
||||||
|
// encrypted piece of shit url
|
||||||
|
$sublink_url = "/resolver?scraper=google&target=" . $piece_of_shit[1];
|
||||||
|
}
|
||||||
|
|
||||||
|
$sublinks[] = [
|
||||||
|
"title" =>
|
||||||
|
$this->fuckhtml
|
||||||
|
->getTextContent($sublink),
|
||||||
|
"date" => null,
|
||||||
|
"url" => $sublink_url
|
||||||
|
];
|
||||||
|
}
|
||||||
|
|
||||||
$out["web"][] = [
|
$out["web"][] = [
|
||||||
"title" => $title,
|
"title" => $title,
|
||||||
"description" => $description,
|
"description" => $description,
|
||||||
|
|||||||
Reference in New Issue
Block a user