backend = new backend("yep"); include "lib/fuckhtml.php"; $this->fuckhtml = new fuckhtml(); } public function getfilters($page){ return [ "country" => [ "display" => "Country", "option" => [ "all" => "All regions", "af" => "Afghanistan", "al" => "Albania", "dz" => "Algeria", "as" => "American Samoa", "ad" => "Andorra", "ao" => "Angola", "ai" => "Anguilla", "ag" => "Antigua and Barbuda", "ar" => "Argentina", "am" => "Armenia", "aw" => "Aruba", "au" => "Australia", "at" => "Austria", "az" => "Azerbaijan", "bs" => "Bahamas", "bh" => "Bahrain", "bd" => "Bangladesh", "bb" => "Barbados", "by" => "Belarus", "be" => "Belgium", "bz" => "Belize", "bj" => "Benin", "bt" => "Bhutan", "bo" => "Bolivia", "ba" => "Bosnia and Herzegovina", "bw" => "Botswana", "br" => "Brazil", "bn" => "Brunei Darussalam", "bg" => "Bulgaria", "bf" => "Burkina Faso", "bi" => "Burundi", "cv" => "Cabo Verde", "kh" => "Cambodia", "cm" => "Cameroon", "ca" => "Canada", "ky" => "Cayman Islands", "cf" => "Central African Republic", "td" => "Chad", "cl" => "Chile", "cn" => "China", "co" => "Colombia", "cg" => "Congo", "cd" => "Congo, Democratic Republic", "ck" => "Cook Islands", "cr" => "Costa Rica", "hr" => "Croatia", "cu" => "Cuba", "cy" => "Cyprus", "cz" => "Czechia", "ci" => "Côte d'Ivoire", "dk" => "Denmark", "dj" => "Djibouti", "dm" => "Dominica", "do" => "Dominican Republic", "ec" => "Ecuador", "eg" => "Egypt", "sv" => "El Salvador", "gq" => "Equatorial Guinea", "ee" => "Estonia", "et" => "Ethiopia", "fo" => "Faroe Islands", "fj" => "Fiji", "fi" => "Finland", "fr" => "France", "gf" => "French Guiana", "pf" => "French Polynesia", "ga" => "Gabon", "gm" => "Gambia", "ge" => "Georgia", "de" => "Germany", "gh" => "Ghana", "gi" => "Gibraltar", "gr" => "Greece", "gl" => "Greenland", "gd" => "Grenada", "gp" => "Guadeloupe", "gu" => "Guam", "gt" => "Guatemala", "gg" => "Guernsey", "gn" => "Guinea", "gy" => "Guyana", "ht" => "Haiti", "hn" => "Honduras", "hk" => "Hong Kong", "hu" => "Hungary", "is" => "Iceland", "in" => "India", "id" => "Indonesia", "iq" => "Iraq", "ie" => "Ireland", "im" => "Isle of Man", "il" => "Israel", "it" => "Italy", "jm" => "Jamaica", "jp" => "Japan", "je" => "Jersey", "jo" => "Jordan", "kz" => "Kazakhstan", "ke" => "Kenya", "ki" => "Kiribati", "kw" => "Kuwait", "kg" => "Kyrgyzstan", "la" => "Lao People's Democratic Republic", "lv" => "Latvia", "lb" => "Lebanon", "ls" => "Lesotho", "ly" => "Libya", "li" => "Liechtenstein", "lt" => "Lithuania", "lu" => "Luxembourg", "mk" => "Macedonia", "mg" => "Madagascar", "mw" => "Malawi", "my" => "Malaysia", "mv" => "Maldives", "ml" => "Mali", "mt" => "Malta", "mq" => "Martinique", "mr" => "Mauritania", "mu" => "Mauritius", "yt" => "Mayotte", "mx" => "Mexico", "fm" => "Micronesia, Federated States of", "md" => "Moldova", "mc" => "Monaco", "mn" => "Mongolia", "me" => "Montenegro", "ms" => "Montserrat", "ma" => "Morocco", "mz" => "Mozambique", "mm" => "Myanmar", "na" => "Namibia", "nr" => "Nauru", "np" => "Nepal", "nl" => "Netherlands", "nc" => "New Caledonia", "nz" => "New Zealand", "ni" => "Nicaragua", "ne" => "Niger", "ng" => "Nigeria", "nu" => "Niue", "no" => "Norway", "om" => "Oman", "pk" => "Pakistan", "ps" => "Palestine, State of", "pa" => "Panama", "pg" => "Papua New Guinea", "py" => "Paraguay", "pe" => "Peru", "ph" => "Philippines", "pn" => "Pitcairn", "pl" => "Poland", "pt" => "Portugal", "pr" => "Puerto Rico", "qa" => "Qatar", "ro" => "Romania", "ru" => "Russian Federation", "rw" => "Rwanda", "re" => "Réunion", "sh" => "Saint Helena", "kn" => "Saint Kitts and Nevis", "lc" => "Saint Lucia", "vc" => "Saint Vincent and the Grenadines", "ws" => "Samoa", "sm" => "San Marino", "st" => "Sao Tome and Principe", "sa" => "Saudi Arabia", "sn" => "Senegal", "rs" => "Serbia", "sc" => "Seychelles", "sl" => "Sierra Leone", "sg" => "Singapore", "sk" => "Slovakia", "si" => "Slovenia", "sb" => "Solomon Islands", "so" => "Somalia", "kr" => "Sourth Korea", "za" => "South Africa", "es" => "Spain", "lk" => "Sri Lanka", "sr" => "Suriname", "se" => "Sweden", "ch" => "Switzerland", "tw" => "Taiwan", "tj" => "Tajikistan", "tz" => "Tanzania", "th" => "Thailand", "tl" => "Timor-Leste", "tg" => "Togo", "tk" => "Tokelau", "to" => "Tonga", "tt" => "Trinidad and Tobago", "tn" => "Tunisia", "tr" => "Turkey", "tm" => "Turkmenistan", "ug" => "Uganda", "ua" => "Ukraine", "ae" => "United Arab Emirates", "gb" => "United Kingdom", "us" => "United States", "uy" => "Uruguay", "uz" => "Uzbekistan", "vu" => "Vanuatu", "ve" => "Venezuela", "vn" => "Vietnam", "vg" => "Virgin Islands, British", "vi" => "Virgin Islands, U.S.", "ye" => "Yemen", "zm" => "Zambia", "zw" => "Zimbabwe" ] ], "nsfw" => [ "display" => "NSFW", "option" => [ "yes" => "Yes", "maybe" => "Maybe", "no" => "No" ] ] ]; } private function get($proxy, $url, $get = []){ $curlproc = curl_init(); if($get !== []){ $get = http_build_query($get); $url .= "?" . $get; } curl_setopt($curlproc, CURLOPT_URL, $url); // use http2 curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding curl_setopt($curlproc, CURLOPT_HTTPHEADER, ["User-Agent: " . config::USER_AGENT, "Accept: */*", "Accept-Language: en-US,en;q=0.5", "Accept-Encoding: gzip, deflate, br, zstd", "Referer: https://yep.com/", "Origin: https://yep.com", "DNT: 1", "Connection: keep-alive", "Sec-Fetch-Dest: empty", "Sec-Fetch-Mode: cors", "Sec-Fetch-Site: same-site", "Priority: u=4", "TE: trailers"] ); curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); $this->backend->assign_proxy($curlproc, $proxy); $data = curl_exec($curlproc); if(curl_errno($curlproc)){ throw new Exception(curl_error($curlproc)); } curl_close($curlproc); return $data; } public function web($get){ $search = $get["s"]; if(strlen($search) === 0){ throw new Exception("Search term is empty!"); } $country = $get["country"]; $nsfw = $get["nsfw"]; switch($nsfw){ case "yes": $nsfw = "off"; break; case "maybe": $nsfw = "moderate"; break; case "no": $nsfw = "strict"; break; } $out = [ "status" => "ok", "spelling" => [ "type" => "no_correction", "using" => null, "correction" => null ], "npt" => null, "answer" => [], "web" => [], "image" => [], "video" => [], "news" => [], "related" => [] ]; try{ // https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web $json = $this->get( $this->backend->get_ip(), "https://api.yep.com/fs/2/search", [ "client" => "web", "gl" => $country == "all" ? $country : strtoupper($country), "limit" => "99999", "no_correct" => "false", "q" => $search, "safeSearch" => $nsfw, "type" => "web" ] ); }catch(Exception $error){ throw new Exception("Failed to fetch JSON"); } $this->detect_cf($json); $json = json_decode($json, true); //$json = json_decode(file_get_contents("scraper/yep.json"), true); if($json === null){ throw new Exception("Failed to decode JSON"); } if(isset($json[1]["correction"])){ $out["spelling"] = [ "type" => "not_many", "using" => $search, "correction" => $json[1]["correction"][1] ]; } if(isset($json[1]["results"])){ foreach($json[1]["results"] as $item){ switch(strtolower($item["type"])){ case "organic": $sublinks = []; if(isset($item["sitelinks"]["full"])){ foreach($item["sitelinks"]["full"] as $link){ $sublinks[] = [ "title" => $link["title"], "date" => null, "description" => $this->titledots( strip_tags( html_entity_decode( $link["snippet"] ) ) ), "url" => $link["url"] ]; } } $out["web"][] = [ "title" => $item["title"], "description" => $this->titledots( strip_tags( html_entity_decode( $item["snippet"] ) ) ), "url" => $item["url"], "date" => strtotime($item["first_seen"]), "type" => "web", "thumb" => [ "url" => null, "ratio" => null ], "sublink" => $sublinks, "table" => [] ]; break; } } } if(isset($json[1]["featured_news"])){ foreach($json[1]["featured_news"] as $news){ $out["news"][] = [ "title" => $news["title"], "description" => $this->titledots( strip_tags( html_entity_decode( $news["snippet"] ) ) ), "date" => strtotime($news["first_seen"]), "thumb" => isset($news["img"]) ? [ "url" => $this->unshiturl($news["img"]), "ratio" => "16:9" ] : [ "url" => null, "ratio" => null ], "url" => $news["url"] ]; } } if(isset($json[1]["featured_images"])){ foreach($json[1]["featured_images"] as $image){ if( $image["width"] !== 0 && $image["height"] !== 0 ){ $thumb_width = $image["width"] >= 260 ? 260 : $image["width"]; $thumb_height = ceil($image["height"] * ($thumb_width / $image["width"])); $width = $image["width"]; $height = $image["height"]; }else{ $thumb_width = null; $thumb_height = null; $width = null; $height = null; } $out["image"][] = [ "title" => $image["title"], "source" => [ [ "url" => $image["image_id"], "width" => $width, "height" => $height ], [ "url" => $image["src"], "width" => $thumb_width, "height" => $thumb_height ] ], "url" => $image["host_page"] ]; } } return $out; } public function image($get){ $search = $get["s"]; if(strlen($search) === 0){ throw new Exception("Search term is empty!"); } $country = $get["country"]; $nsfw = $get["nsfw"]; switch($nsfw){ case "yes": $nsfw = "off"; break; case "maybe": $nsfw = "moderate"; break; case "no": $nsfw = "strict"; break; } $out = [ "status" => "ok", "npt" => null, "image" => [] ]; try{ $json = $this->get( $this->backend->get_ip(), // no nextpage! "https://api.yep.com/fs/2/search", [ "client" => "web", "gl" => $country == "all" ? $country : strtoupper($country), "no_correct" => "false", "q" => $search, "safeSearch" => $nsfw, "type" => "images" ] ); }catch(Exception $error){ throw new Exception("Failed to fetch JSON"); } $this->detect_cf($json); $json = json_decode($json, true); if($json === null){ throw new Exception("Failed to decode JSON"); } if(isset($json[1]["results"])){ foreach($json[1]["results"] as $item){ if( $item["width"] !== 0 && $item["height"] !== 0 ){ $thumb_width = $item["width"] >= 260 ? 260 : $item["width"]; $thumb_height = ceil($item["height"] * ($thumb_width / $item["width"])); $width = $item["width"]; $height = $item["height"]; }else{ $thumb_width = null; $thumb_height = null; $width = null; $height = null; } $out["image"][] = [ "title" => $item["title"], "source" => [ [ "url" => $item["image_id"], "width" => $width, "height" => $height ], [ "url" => $item["src"], "width" => $thumb_width, "height" => $thumb_height ] ], "url" => $item["host_page"] ]; } } return $out; } public function news($get){ $search = $get["s"]; if(strlen($search) === 0){ throw new Exception("Search term is empty!"); } $country = $get["country"]; $nsfw = $get["nsfw"]; switch($nsfw){ case "yes": $nsfw = "off"; break; case "maybe": $nsfw = "moderate"; break; case "no": $nsfw = "strict"; break; } $out = [ "status" => "ok", "npt" => null, "news" => [] ]; try{ // https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web $json = $this->get( $this->backend->get_ip(), "https://api.yep.com/fs/2/search", [ "client" => "web", "gl" => $country == "all" ? $country : strtoupper($country), "limit" => "99999", "no_correct" => "false", "q" => $search, "safeSearch" => $nsfw, "type" => "news" ] ); }catch(Exception $error){ throw new Exception("Failed to fetch JSON"); } $this->detect_cf($json); $json = json_decode($json, true); //$json = json_decode(file_get_contents("scraper/yep.json"), true); if($json === null){ throw new Exception("Failed to decode JSON"); } if(isset($json[1]["results"])){ foreach($json[1]["results"] as $item){ $out["news"][] = [ "title" => $item["title"], "author" => null, "description" => $this->titledots( strip_tags( html_entity_decode( $item["snippet"] ) ) ), "date" => strtotime($item["first_seen"]), "thumb" => isset($item["img"]) ? [ "url" => $this->unshiturl($item["img"]), "ratio" => "16:9" ] : [ "url" => null, "ratio" => null ], "url" => $item["url"] ]; } } return $out; } private function detect_cf($payload){ // detect cloudflare page $this->fuckhtml->load($payload); if( count( $this->fuckhtml ->getElementsByClassName( "cf-wrapper", "div" ) ) !== 0 ){ throw new Exception("Blocked by Cloudflare. Please follow curl-impersonate installation instructions"); } } private function titledots($title){ $substr = substr($title, -4); if( strpos($substr, "...") !== false || strpos($substr, "…") !== false ){ return trim(substr($title, 0, -4)); } return trim($title); } private function unshiturl($url){ $newurl = parse_url($url, PHP_URL_QUERY); parse_str($newurl, $newurl); if(isset($newurl["url"])){ return $newurl["url"]; } return $url; } }