url redirect bypass
This commit is contained in:
lolcat committed
1 parent
9f0437d949
commit
57a72e5b5e
1 file changed
+268
-30
+268
-30
@@ -10,8 +10,9 @@ class google{
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("google");
|
||||
|
||||
$this->proc = curl_multi_init();
|
||||
$this->handles = [];
|
||||
$this->urls_to_resolve = [];
|
||||
|
||||
$this->init_multi_curl();
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
@@ -617,23 +618,57 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
private function redirect_add_url($url){
|
||||
private function init_multi_curl(){
|
||||
|
||||
$this->proc = curl_multi_init();
|
||||
$this->handles = [];
|
||||
}
|
||||
|
||||
private function redirect_add_url($url, $replace = null){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/goto\?url=([^&]+)/',
|
||||
$url,
|
||||
$slug
|
||||
) === 0
|
||||
) === 1
|
||||
){
|
||||
|
||||
// not a google redirect
|
||||
// normal goto
|
||||
$this->urls_to_resolve[] = [
|
||||
"replace" => $url,
|
||||
"slug" => $slug[1],
|
||||
"type" => "goto"
|
||||
];
|
||||
}
|
||||
|
||||
elseif(
|
||||
preg_match(
|
||||
'/https?:\/\/share\.google\/([^"]+)+/',
|
||||
$url,
|
||||
$link_regex
|
||||
) === 1
|
||||
){
|
||||
|
||||
// share.google
|
||||
$this->urls_to_resolve[] = [
|
||||
"replace" => $replace,
|
||||
"slug" => $link_regex[1],
|
||||
"type" => "share"
|
||||
];
|
||||
}
|
||||
|
||||
// rest is ignored, since its not a redirect
|
||||
|
||||
// always return url
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function urlfuck_goto($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . rawurlencode($slug));
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
@@ -658,7 +693,7 @@ class google{
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
@@ -669,7 +704,92 @@ class google{
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function resolve_urls(&$scraped_data){
|
||||
private function urlfuck_shareservice($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/httpservice/retry/SearchApiService/GetShortenedKpSharingUrl?udm=14&reqpld=[[[\"/goto?url%3D" . rawurlencode($slug) . "\"],1,null,null,null,null,35,[\"st\"]]]&msc=gwsrpc");
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
"replacement" => $url,
|
||||
"handle" => $curlproc
|
||||
];
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function urlfuck_share($url, $slug){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/share.google?q=" . rawurlencode($slug));
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $this->backend->get_ip(null, true));
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
"replacement" => $url,
|
||||
"handle" => $curlproc
|
||||
];
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function resolve_gotos(&$scraped_data){
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_goto($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
@@ -741,6 +861,140 @@ class google{
|
||||
curl_multi_close($this->proc);
|
||||
}
|
||||
|
||||
private function resolve_share(&$scraped_data){
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_shareservice($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
$status = curl_multi_exec($this->proc, $active);
|
||||
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||
|
||||
if($active){
|
||||
curl_multi_select($this->proc);
|
||||
}
|
||||
|
||||
}while($active && $status === CURLM_OK);
|
||||
|
||||
//
|
||||
// if we reach this, we're done downloading garbage
|
||||
//
|
||||
|
||||
// get the share links
|
||||
$this->urls_to_resolve = [];
|
||||
|
||||
for($i=0; $i<count($this->handles); $i++){
|
||||
|
||||
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/https?:\/\/share\.google\/(?:[^"]+)/',
|
||||
$html,
|
||||
$link_regex
|
||||
) !== 1
|
||||
){
|
||||
|
||||
// failed to find link, will serve a resolver target as fallback
|
||||
$fallback = true;
|
||||
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||
}else{
|
||||
|
||||
$fallback = false;
|
||||
$link = $link_regex[0];
|
||||
}
|
||||
|
||||
if(
|
||||
(
|
||||
$fallback === false &&
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
) ||
|
||||
$fallback === true
|
||||
){
|
||||
|
||||
$this->redirect_add_url($link, $this->handles[$i]["replacement"]);
|
||||
}
|
||||
}
|
||||
|
||||
// resolve shareable urls
|
||||
$this->init_multi_curl();
|
||||
|
||||
foreach($this->urls_to_resolve as $item){
|
||||
|
||||
$this->urlfuck_share($item["replace"], $item["slug"]);
|
||||
}
|
||||
|
||||
do{
|
||||
do{
|
||||
$status = curl_multi_exec($this->proc, $active);
|
||||
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||
|
||||
if($active){
|
||||
curl_multi_select($this->proc);
|
||||
}
|
||||
|
||||
}while($active && $status === CURLM_OK);
|
||||
|
||||
// parse downloaded garbage
|
||||
$keys = [];
|
||||
$replacements = [];
|
||||
|
||||
for($i=0; $i<count($this->handles); $i++){
|
||||
|
||||
$html = curl_multi_getcontent($this->handles[$i]["handle"]);
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$a =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"a"
|
||||
);
|
||||
|
||||
if(count($a) === 0){
|
||||
|
||||
// failed to find link, will serve a resolver target as fallback
|
||||
$fallback = true;
|
||||
$slug = explode("url=", $this->handles[$i]["replacement"]);
|
||||
$link = "/resolver?scraper=google&target=" . rawurlencode($slug[1]);
|
||||
}else{
|
||||
|
||||
$fallback = false;
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
}
|
||||
|
||||
if(
|
||||
(
|
||||
$fallback === false &&
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
) ||
|
||||
$fallback === true
|
||||
){
|
||||
|
||||
$keys[] = $this->handles[$i]["replacement"];
|
||||
$replacements[] = $link;
|
||||
}
|
||||
}
|
||||
|
||||
$this->recursive_replace($keys, $replacements, $scraped_data);
|
||||
|
||||
curl_multi_close($this->proc);
|
||||
}
|
||||
|
||||
private function recursive_replace(&$keys, &$replacements, &$pointer){
|
||||
|
||||
foreach($pointer as $key => &$value){
|
||||
@@ -794,22 +1048,6 @@ class google{
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
/*
|
||||
$session = $this->backend->session_get();
|
||||
|
||||
if(config::FPLAY_ENABLE_API === false){
|
||||
|
||||
throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information");
|
||||
}
|
||||
|
||||
if($session === false){
|
||||
|
||||
throw new Exception("Session database is empty");
|
||||
}
|
||||
|
||||
$proxy = &$session["proxy"];
|
||||
$headers = &$session["headers"];
|
||||
*/
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
@@ -1475,7 +1713,7 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -1603,8 +1841,6 @@ class google{
|
||||
}
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
$data = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
@@ -1615,6 +1851,8 @@ class google{
|
||||
$html = &$data["data"];
|
||||
$container = &$data["container"];
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
@@ -1721,7 +1959,7 @@ class google{
|
||||
);
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_gotos($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -2129,7 +2367,7 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
@@ -2495,7 +2733,7 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
$this->resolve_share($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user