diff --git a/lib/frontend.php b/lib/frontend.php index b45c5a6..fd54e7d 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -937,14 +937,12 @@ class frontend{ "display" => "Scraper", "option" => [ "ddg" => "DuckDuckGo", - //"yahoo" => "Yahoo!", + "yahoo" => "Yahoo!", "brave" => "Brave", - "mullvad_brave" => "Mullvad (Brave)", "yandex" => "Yandex", "google" => "Google", "google_api" => "Google API", "google_cse" => "Google CSE", - "mullvad_google" => "Mullvad (Google)", "startpage" => "Startpage", "qwant" => "Qwant", "ghostery" => "Ghostery", diff --git a/scraper/mullvad.php b/scraper/mullvad.php deleted file mode 100644 index c143cd5..0000000 --- a/scraper/mullvad.php +++ /dev/null @@ -1,342 +0,0 @@ -engine = $engine; - - include "lib/backend.php"; - $this->backend = new backend("mullvad_{$this->engine}"); - } - - public function getfilters($page){ - return [ - "country" => [ // &country= - "display" => "Country", - "option" => [ - "any" => "Any country", - "ar" => "Argentina", - "au" => "Australia", - "at" => "Austria", - "be" => "Belgium", - "br" => "Brazil", - "ca" => "Canada", - "cl" => "Chile", - "cn" => "China", - "dk" => "Denmark", - "fi" => "Finland", - "fr" => "France", - "de" => "Germany", - "hk" => "Hong Kong", - "in" => "India", - "id" => "Indonesia", - "it" => "Italy", - "jp" => "Japan", - "kr" => "Korea, Republic", - "my" => "Malaysia", - "mx" => "Mexico", - "nl" => "Netherlands", - "nz" => "New Zealand", - "no" => "Norway", - "ph" => "Philippines", - "pl" => "Poland", - "pt" => "Portugal", - "ru" => "Russian Federation", - "sa" => "Saudi Arabia", - "za" => "South Africa", - "es" => "Spain", - "se" => "Sweden", - "ch" => "Switzerland", - "tw" => "Taiwan", - "tr" => "Turkey", - "uk" => "United Kingdom", - "us" => "United States" - ] - ], - "language" => [ // &language= - "display" => "Language", - "option" => [ - "any" => "Any language", - "ar" => "Arabic", - "bg" => "Bulgarian", - "ca" => "Catalan", - "zh-hans" => "Chinese (Simplified)", - "zh-hant" => "Chinese (Traditional)", - "hr" => "Croatian", - "cs" => "Czech", - "da" => "Danish", - "nl" => "Dutch", - "en" => "English", - "et" => "Estonian", - "fi" => "Finnish", - "fr" => "French", - "de" => "German", - "he" => "Hebrew", - "hu" => "Hungarian", - "is" => "Icelandic", - "it" => "Italian", - "jp" => "Japanese", - "ko" => "Korean", - "lv" => "Latvian", - "lt" => "Lithuanian", - "nb" => "Norwegian", - "pl" => "Polish", - "pt" => "Portuguese", - "ro" => "Romanian", - "ru" => "Russian", - "sr" => "Serbian", - "sk" => "Slovak", - "sl" => "Slovenian", - "es" => "Spanish", - "sv" => "Swedish", - "tr" => "Turkish" - ] - ], - "time" => [ // &lastUpdated= - "display" => "Time posted", - "option" => [ - "any" => "Any time", - "d" => "Past day", - "w" => "Past week", - "m" => "Past month", - "y" => "Past year" - ] - ] - ]; - } - - private function get($proxy, $url, $get = []){ - - $curlproc = curl_init(); - - if($get !== []){ - $get = http_build_query($get); - $url .= "?" . $get; - } - - curl_setopt($curlproc, CURLOPT_URL, $url); - - // http2 bypass - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - curl_setopt($curlproc, CURLOPT_HTTPHEADER, - ["User-Agent: " . config::USER_AGENT, - "Accept: */*", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip, deflate, br, zstd", - "Referer: https://leta.mullvad.net/search", - "DNT: 1", - "Sec-GPC: 1", - "Connection: keep-alive", - "Cookie: engine=brave", - "Sec-Fetch-Dest: empty", - "Sec-Fetch-Mode: cors", - "Sec-Fetch-Site: same-origin", - "Priority: u=0", - "TE: trailers"] - ); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - $this->backend->assign_proxy($curlproc, $proxy); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ - - throw new Exception(curl_error($curlproc)); - } - - curl_close($curlproc); - return $data; - } - - public function web($get){ - - if($get["npt"]){ - - [$params, $proxy] = $this->backend->get($get["npt"], "web"); - $params = json_decode($params, true); - - }else{ - - if(strlen($get["s"]) === 0){ - - throw new Exception("Search term is empty!"); - } - - // generate filters - $params = [ - "q" => $get["s"], - "engine" => $this->engine, - "page" => 1 - ]; - - if($get["country"] != "any"){ - - $params["country"] = $get["country"]; - } - - if($get["language"] != "any"){ - - $params["language"] = $get["language"]; - } - - if($get["time"] != "any"){ - - $params["lastUpdated"] = $get["time"]; - } - - $proxy = $this->backend->get_ip(); - } - - try{ - $json = $this->get( - $proxy, - "https://leta.mullvad.net/search/__data.json", - $params - ); - }catch(Exception $error){ - - throw new Exception("Failed to fetch search page"); - } - - $json = json_decode($json, true); - - if($json === null){ - - throw new Exception("Failed to decode JSON"); - } - - if(!isset($json["nodes"])){ - - throw new Exception("Mullvad did not return a nodes object"); - } - - $out = [ - "status" => "ok", - "spelling" => [ - "type" => "no_correction", - "using" => null, - "correction" => null - ], - "npt" => $nextpage, - "answer" => [], - "web" => [], - "image" => [], - "video" => [], - "news" => [], - "related" => [] - ]; - - // parse json payload - foreach($json["nodes"] as $node){ - - if(!isset($node["data"][0]["q"])){ - - // not iterating through the query object - continue; - } - - // node 0 contains pointers to what we need to iterate through - $node0 = &$node["data"][0]; - - if(!isset($node["data"][$node0["success"]])){ - - throw new Exception("Mullvad did not return a success object"); - } - - $success = &$node["data"][$node0["success"]]; - - if($success === false){ - - throw new Exception("Mullvad flagged the response as unsuccessful"); - } - - if(!isset($node["data"][$node0["items"]])){ - - throw new Exception("Mullvad did not return an items object"); - } - - $search_pointers = &$node["data"][$node0["items"]]; - - // - // Iterate over results - // - foreach($search_pointers as $pointer){ - - $pointer = &$node["data"][$pointer]; - - $link = &$node["data"][$pointer["link"]]; - $title = &$node["data"][$pointer["title"]]; - $description = &$node["data"][$pointer["snippet"]]; - - $date = null; - if($this->engine == "google"){ - - // attempt to extract date - // Jan 12, 2017 - $date_parts = explode(" ... ", $description, 2); - - if( - count($date_parts) === 2 && - strlen($date_parts[0]) < 15 - ){ - - $date = strtotime(trim($date_parts[0])); - - if($date === false){ - - $date = null; - }else{ - - $description = trim($date_parts[1]); - } - } - } - - $out["web"][] = [ - "title" => $this->titledots($title), - "description" => $this->titledots($description), - "url" => $link, - "date" => $date, - "type" => "web", - "thumb" => [ - "url" => null, - "ratio" => null - ], - "sublink" => [], - "table" => [] - ]; - } - - // - // Get nextpage - // - if(isset($node["data"][$node0["next"]])){ - - $params["page"] = (int)$node["data"][$node0["next"]]; - - $out["npt"] = - $this->backend->store( - json_encode($params), - "web", - $proxy - ); - } - } - - return $out; - } - - private function titledots($title){ - - return trim($title, " .\t\n\r\0\x0B…"); - } -} diff --git a/scraper/mullvad_brave.php b/scraper/mullvad_brave.php deleted file mode 100644 index 958733b..0000000 --- a/scraper/mullvad_brave.php +++ /dev/null @@ -1,20 +0,0 @@ -mullvad = new mullvad("brave"); - } - - public function getfilters($page){ - - return $this->mullvad->getfilters($page); - } - - public function web($get){ - - return $this->mullvad->web($get); - } -} diff --git a/scraper/mullvad_google.php b/scraper/mullvad_google.php deleted file mode 100644 index 9804942..0000000 --- a/scraper/mullvad_google.php +++ /dev/null @@ -1,20 +0,0 @@ -mullvad = new mullvad("google"); - } - - public function getfilters($page){ - - return $this->mullvad->getfilters($page); - } - - public function web($get){ - - return $this->mullvad->web($get); - } -} diff --git a/settings.php b/settings.php index b1949f6..79cf947 100644 --- a/settings.php +++ b/settings.php @@ -125,10 +125,6 @@ $settings = [ "value" => "brave", "text" => "Brave" ], - [ - "value" => "mullvad_brave", - "text" => "Mullvad (Brave)" - ], [ "value" => "yandex", "text" => "Yandex" @@ -145,10 +141,6 @@ $settings = [ "value" => "google_cse", "text" => "Google CSE" ], - [ - "value" => "mullvad_google", - "text" => "Mullvad (Google)" - ], [ "value" => "startpage", "text" => "Startpage"