From 5d8c9a579832f15f536f92fa09326a65e36de403 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 8 Nov 2026 15:55:20 -0500 Subject: [PATCH] startpage fix --- scraper/startpage.php | 198 ++++++++++++++++++++++++++++++++++++------ 1 file changed, 170 insertions(+), 28 deletions(-) diff --git a/scraper/startpage.php b/scraper/startpage.php index a721574..9d706da 100644 --- a/scraper/startpage.php +++ b/scraper/startpage.php @@ -545,17 +545,20 @@ class startpage{ switch($category["display_type"]){ case "web-google": + case "web-bing": foreach($category["results"] as $result){ $sublinks = []; - foreach($result["siteLinks"] as $sublink){ - - $sublinks[] = [ - "title" => $sublink["title"], - "description" => null, - "url" => $sublink["clickUrl"] - ]; + if(isset($result["siteLinks"])){ + foreach($result["siteLinks"] as $sublink){ + + $sublinks[] = [ + "title" => $sublink["title"], + "description" => null, + "url" => $sublink["clickUrl"] + ]; + } } $description = @@ -596,15 +599,17 @@ class startpage{ $out["web"][] = [ "title" => - $this->titledots( - html_entity_decode( - $this->fuckhtml - ->getTextContent( - $result["title"] + $this->remove_penguins( + $this->titledots( + html_entity_decode( + $this->fuckhtml + ->getTextContent( + $result["title"] + ) ) ) ), - "description" => $description, + "description" => $this->remove_penguins($description), "url" => $result["clickUrl"], "date" => $date, "type" => "web", @@ -619,8 +624,43 @@ class startpage{ break; case "images-qi-top": + case "images-bing-top": foreach($category["results"] as $result){ + if(isset($result["mdThumbnailWidth"])){ + + $thumb_width = $result["mdThumbnailWidth"]; + }elseif(isset($result["thumbnailWidth"])){ + + $thumb_width = $result["thumbnailWidth"]; + }else{ + + $thumb_width = null; + } + + if(isset($result["mdThumbnailHeight"])){ + + $thumb_height = $result["mdThumbnailHeight"]; + }elseif(isset($result["thumbnailHeight"])){ + + $thumb_height = $result["thumbnailHeight"]; + }else{ + + $thumb_height = null; + } + + if(isset($result["mdThumbnailUrl"])){ + + $thumb_source = $result["mdThumbnailUrl"]; + }elseif(isset($result["thumbnailUrl"])){ + + $thumb_source = $result["thumbnailUrl"]; + }else{ + + // should not happen + continue; + } + $out["image"][] = [ "title" => $this->titledots( @@ -638,19 +678,17 @@ class startpage{ "height" => (int)$result["height"] ], [ - "url" => $this->unshitimage($result["mdThumbnailUrl"]), - "width" => (int)$result["mdThumbnailWidth"], - "height" => (int)$result["mdThumbnailHeight"] + "url" => $this->unshitimage($thumb_source), + "width" => (int)$thumb_width, + "height" => (int)$thumb_height ] ], - "url" => - $result["altClickUrl"] + "url" => $result["altClickUrl"] ]; } break; case "spellsuggest-google": - if(isset($category["results"][0]["query"])){ $out["spelling"] = @@ -662,6 +700,60 @@ class startpage{ } break; + case "spelladjust-bing": + if(isset($category["results"][0]["alterationDisplayQuery"])){ + + $out["spelling"] = + [ + "type" => "including", + "using" => $this->remove_penguins($category["results"][0]["alterationDisplayQuery"]), + "correction" => $category["results"][0]["originalQuery"] + ]; + } + break; + + case "aylf-bing-top": + if(isset($category["results"][0]["query"])){ + + foreach($category["results"] as $ddm){ + + $out["related"][] = $ddm["query"]; + } + } + break; + + case "video-bing-top": + if(isset($category["results"][0]["clickUrl"])){ + + foreach($category["results"] as $video){ + + if(isset($video["thumbnailUrl"])){ + + $thumb = [ + "ratio" => "16:9", + "url" => $this->unshitimage($video["thumbnailUrl"]) + ]; + }else{ + + $thumb = [ + "ratio" => null, + "url" => null + ]; + } + + $out["video"][] = [ + "title" => $video["title"], + "description" => $this->remove_penguins($video["description"]), + "date" => isset($video["date"]) ? strtotime($video["date"]) : null, + "duration" => (int)$video["duration"] !== 0 ? (int)$video["duration"] / 1000 : null, + "views" => $video["viewCount"], + "thumb" => $thumb, + "url" => $video["clickUrl"] + ]; + } + } + break; + case "dictionary-qi": foreach($category["results"] as $result){ @@ -732,6 +824,36 @@ class startpage{ $out["answer"][] = $answer; } break; + + case "news-bing-top": + if(isset($category["results"][0]["clickUrl"])){ + + foreach($category["results"] as $news){ + + if(isset($news["thumbnailUrl"])){ + + $thumb = [ + "ratio" => "16:9", + "url" => $news["thumbnailUrl"] + ]; + }else{ + + $thumb = [ + "ratio" => null, + "url" => null + ]; + } + + $out["news"][] = [ + "title" => $this->remove_penguins($news["title"]), + "description" => $news["description"], + "date" => isset($news["date"]) ? strtotime($news["date"]) : null, + "thumb" => $thumb, + "url" => $news["clickUrl"], + ]; + } + } + break; } } @@ -1246,14 +1368,10 @@ class startpage{ // get npt $out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies); + //print_r($json); + foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){ - if($category["display_type"] != "news-bing"){ - - // unsupported category - continue; - } - foreach($category["results"] as $news){ if( @@ -1276,10 +1394,10 @@ class startpage{ $out["news"][] = [ "title" => $this->titledots($this->remove_penguins($news["title"])), "author" => $news["source"], - "description" => $this->titledots($this->remove_penguins($news["description"])), - "date" => (int)substr((string)$news["date"], 0, -3), + "description" => isset($news["description"]) ? $this->titledots($this->remove_penguins($news["description"])) : null, + "date" => isset($news["date"]) ? strtotime($news["date"]) : null, "thumb" => $thumb, - "url" => $news["clickUrl"] + "url" => $this->remove_tracking_link($news["clickUrl"]) ]; } } @@ -1327,6 +1445,30 @@ class startpage{ return null; } + private function remove_tracking_link($url){ + + // https://api.airfind.com/stats/adrequest/v1?clientid=51776&type=click&adCampaign=organicNewsArticle-prg-top&provider=prg&redirectURL=https%3A%2F%2Fvancouverisland.ctvnews.ca%2Fvideo%2F2026%2F11%2F02%2Ftrump-claims-concept-of-a-deal-with-canada%2F&brand=startpage.udog + $query = parse_url($url); + + if( + !isset($query["host"]) || + $query["host"] != "api.airfind.com" || + !isset($query["query"]) + ){ + + return $url; + } + + parse_str($query["query"], $res); + + if(isset($res["redirectURL"])){ + + return $res["redirectURL"]; + } + + return $url; + } + private function unshitimage($url){ $query = parse_url($url, PHP_URL_QUERY);