lolcat
/
4get
6
13
Fork 4
Code Issues 13 Pull Requests Packages Projects Releases Wiki Activity

http2 on ddg

This commit is contained in:
lolcat 2024-11-07 23:37:43 -05:00
parent e83865be49
commit 9cd369ac08
1 changed files with 28 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
scraper/ddg.php
View File

@ -28,6 +28,9 @@ class ddg{
curl_setopt($curlproc, CURLOPT_URL, $url); curl_setopt($curlproc, CURLOPT_URL, $url);
// http2 bypass
curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
switch($reqtype){ switch($reqtype){
case self::req_web: case self::req_web:
$headers = $headers =
@ -36,27 +39,33 @@ class ddg{
"Accept-Encoding: gzip", "Accept-Encoding: gzip",
"Accept-Language: en-US,en;q=0.5", "Accept-Language: en-US,en;q=0.5",
"DNT: 1", "DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive", "Connection: keep-alive",
"Upgrade-Insecure-Requests: 1", "Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document", "Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate", "Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: cross-site", "Sec-Fetch-Site: same-origin",
"Upgrade-Insecure-Requests: 1"]; "Sec-Fetch-User: ?1",
"Priority: u=0, i",
"TE: trailers"];
break; break;
case self::req_xhr: case self::req_xhr:
$headers = $headers =
["User-Agent: " . config::USER_AGENT, ["User-Agent: " . config::USER_AGENT,
"Accept: */*", "Accept: application/json, text/javascript, */*; q=0.01",
"Accept-Encoding: gzip", "Accept-Encoding: gzip",
"Accept-Language: en-US,en;q=0.5", "Accept-Language: en-US,en;q=0.5",
"Connection: keep-alive", "Connection: keep-alive",
"Referer: https://duckduckgo.com/", "Referer: https://duckduckgo.com/",
"X-Requested-With: XMLHttpRequest", "X-Requested-With: XMLHttpRequest",
"DNT: 1", "DNT: 1",
"Sec-Fetch-Dest: script", "Sec-GPC: 1",
"Sec-Fetch-Mode: no-cors", "Connection: keep-alive",
"Sec-Fetch-Site: same-site"]; "Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"TE: trailers"];
break; break;
} }
@ -1889,12 +1898,12 @@ class ddg{
[$npt, $proxy] = $this->backend->get($get["npt"], "images"); [$npt, $proxy] = $this->backend->get($get["npt"], "images");
try{ try{
$json = json_decode($this->get( $json = $this->get(
$proxy, $proxy,
"https://duckduckgo.com/i.js?" . $npt, "https://duckduckgo.com/i.js?" . $npt,
[], [],
ddg::req_xhr ddg::req_xhr
), true); );
}catch(Exception $err){ }catch(Exception $err){
@ -1920,6 +1929,7 @@ class ddg{
$filter = []; $filter = [];
$get_filters = [ $get_filters = [
"hps" => "1",
"q" => $search, "q" => $search,
"iax" => "images", "iax" => "images",
"ia" => "images" "ia" => "images"
@ -1970,7 +1980,7 @@ class ddg{
} }
$vqd = $vqd[1]; $vqd = $vqd[1];
// @TODO: s param = image offset // @TODO: s param = image offset
$js_params = [ $js_params = [
"l" => $country, "l" => $country,
@ -1994,12 +2004,12 @@ class ddg{
} }
try{ try{
$json = json_decode($this->get( $json = $this->get(
$proxy, $proxy,
"https://duckduckgo.com/i.js", "https://duckduckgo.com/i.js",
$js_params, $js_params,
ddg::req_xhr ddg::req_xhr
), true); );
}catch(Exception $err){ }catch(Exception $err){
@ -2007,6 +2017,13 @@ class ddg{
} }
} }
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$out = [ $out = [
"status" => "ok", "status" => "ok",
"npt" => null, "npt" => null,