From a43b3e394961ade8979caae65e1209c39e7e3361 Mon Sep 17 00:00:00 2001 From: lolcat Date: Sat, 25 Jul 2026 20:15:27 -0400 Subject: [PATCH] add purili scraper --- lib/frontend.php | 3 + scraper/purili.php | 350 +++++++++++++++++++++++++++++++++++++++++++++ settings.php | 8 ++ 3 files changed, 361 insertions(+) create mode 100644 scraper/purili.php diff --git a/lib/frontend.php b/lib/frontend.php index 9509ae7..88b34bf 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -634,6 +634,7 @@ class frontend{ $filters["scraper"] = [ "display" => "Scraper", "option" => [ + //"fget" => "fget", "ddg" => "DuckDuckGo", //"yahoo" => "Yahoo!", "brave" => "Brave", @@ -652,6 +653,7 @@ class frontend{ "coccoc" => "Cốc Cốc", "solofield" => "Solofield", "marginalia" => "Marginalia", + "purili" => "Purili", "wiby" => "wiby" ] ]; @@ -707,6 +709,7 @@ class frontend{ "naver" => "Naver", "baidu" => "Baidu", "coccoc" => "Cốc Cốc", + "purili" => "Purili", "solofield" => "Solofield" ] ]; diff --git a/scraper/purili.php b/scraper/purili.php new file mode 100644 index 0000000..5879419 --- /dev/null +++ b/scraper/purili.php @@ -0,0 +1,350 @@ +backend = new backend("purili"); + } + + public function getfilters($page){ + + switch($page){ + case "web": + return [ + "nsfw" => [ + "display" => "NSFW", + "option" => [ + "yes" => "Yes", + "maybe" => "Maybe", + "no" => "No" + ] + ], + "language" => [ + "display" => "Language", + "option" => [ + "any" => "Any language", + "en" => "English only", + "nl" => "Dutch preferred", + "de" => "German preferred", + "fr" => "French preferred", + "es" => "Spanish preferred", + "it" => "Italian preferred", + "pt" => "Portuguese preferred" + ] + ] + ]; + break; + + case "videos": + return []; + break; + } + } + + private function get($proxy, $url, $get = [], $nsfw = null, $language = null){ + + $curlproc = curl_init(); + + switch($nsfw){ + + case "yes": + case null: + $get["fv"] = "0"; + $nsfw = "off"; + break; + + case "maybe": + $get["fv"] = "1"; + $nsfw = "moderate"; + break; + + case "no": + $get["fv"] = "2"; + $nsfw = "strict"; + break; + } + + switch($language){ + + case "any": + case null: + $language_cookie = ""; + break; + + default: + $language_cookie = "; purili-language={$language}"; + break; + } + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, + ["User-Agent: " . config::USER_AGENT, + "Accept: */*", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br, zstd", + "DNT: 1", + "Sec-GPC: 1", + "Connection: keep-alive", + "Cookie: purili-safesearch={$nsfw}{$language_cookie}", + "Sec-Fetch-Dest: empty", + "Sec-Fetch-Mode: cors", + "Sec-Fetch-Site: same-origin", + "Priority: u=4"] + ); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $this->backend->assign_proxy($curlproc, $proxy); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function web($get){ + + if($get["npt"]){ + + [$q, $proxy] = $this->backend->get($get["npt"], "web"); + $q = json_decode($q, true); + + $nsfw = $q["nsfw"]; + unset($q["nsfw"]); + + $language = $q["language"]; + unset($q["language"]); + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $proxy = $this->backend->get_ip(); + + $q = [ + "q" => $search, + "page" => 1, + // added at $this->get() + // "fv" => 0 + ]; + + $nsfw = $get["nsfw"]; + $language = $get["language"]; + } + + // https://puri.li/api/search?q=4get&page=1&fv=0 + try{ + $json = $this->get( + $proxy, + "https://puri.li/api/search", + $q, + $nsfw, + $language + ); + }catch(Exception $error){ + + throw new Exception("Failed to hit API"); + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["results"])){ + + throw new Exception("Failed to access results array"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + foreach($json["results"] as $result){ + + $out["web"][] = [ + "title" => $result["title"], + "description" => $result["description"], + "url" => $result["url"], + "date" => null, + "type" => "web", + "thumb" => [ + "url" => null, + "ratio" => null + ], + "sublink" => [], + "table" => [] + ]; + } + + // get next page + if($json["hasNext"]){ + + $q["page"]++; + $q["nsfw"] = $nsfw; + $q["language"] = $language; + + $out["npt"] = + $this->backend->store( + json_encode($q), + "web", + $proxy + ); + } + + return $out; + } + + public function video($get){ + + + if($get["npt"]){ + + [$q, $proxy] = $this->backend->get($get["npt"], "videos"); + $q = json_decode($q, true); + + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $proxy = $this->backend->get_ip(); + + $q = [ + "q" => $search, + "offset" => 0, + "limit" => 12 + // added at $this->get() + // "fv" => 0 + ]; + } + + // https://puri.li/api/videos/search?q=higurashi&limit=12&fv=1 + // hidden &offset parameter only accessible through the API + try{ + $json = $this->get( + $proxy, + "https://puri.li/api/videos/search", + $q, + null, + null + ); + }catch(Exception $error){ + + throw new Exception("Failed to hit API"); + } + + $json = json_decode($json, true); + + if($json === null){ + + throw new Exception("Failed to decode JSON"); + } + + if(!isset($json["results"])){ + + throw new Exception("Failed to access results array"); + } + + $out = [ + "status" => "ok", + "npt" => null, + "video" => [], + "author" => [], + "livestream" => [], + "playlist" => [], + "reel" => [] + ]; + + foreach($json["results"] as $result){ + + if(isset($result["thumbnail"])){ + + $thumb = [ + "ratio" => "16:9", + "url" => $result["thumbnail"] + ]; + }else{ + + $thumb = [ + "ratio" => null, + "url" => null + ]; + } + + $out["video"][] = [ + "title" => $result["title"], + "description" => + str_replace( + "\n", " ", + mb_substr( + $result["description"], 0, 255, "utf-8" + ) + ), + "author" => [ + "name" => null, + "url" => isset($result["channel_id"]) ? "https://www.youtube.com/channel/" . $result["channel_id"] : null, + "avatar" => null + ], + "date" => strtotime($result["upload_date"]), + "duration" => $result["duration"], + "views" => $result["view_count"], + "thumb" => $thumb, + "url" => $result["url"] + ]; + } + + // get next page + if($json["limit"] + $json["offset"] < $json["total"]){ + + $q["offset"] = $q["offset"] + $json["limit"]; + + $out["npt"] = + $this->backend->store( + json_encode($q), + "videos", + $proxy + ); + } + + return $out; + } +} diff --git a/settings.php b/settings.php index 11146f3..9253586 100644 --- a/settings.php +++ b/settings.php @@ -185,6 +185,10 @@ $settings = [ "value" => "marginalia", "text" => "Marginalia" ], + [ + "value" => "purili", + "text" => "Purili" + ], [ "value" => "wiby", "text" => "wiby" @@ -341,6 +345,10 @@ $settings = [ "value" => "coccoc", "text" => "Cốc Cốc" ], + [ + "value" => "purili", + "text" => "Purili" + ], [ "value" => "solofield", "text" => "Solofield"