diff --git a/data/config.php b/data/config.php index 06b834b..fdf2aff 100644 --- a/data/config.php +++ b/data/config.php @@ -24,15 +24,21 @@ class config{ const API_ENABLED = true; // - // 4play (session provider) + // 4play settings // - // Enable 4play API? - const FPLAY_ENABLE_API = true; + // Enable 4get's /api/v2/provide_sesh endpoint + const FPLAY_ENABLE_API = false; - // 4play password. Please set this to something secure if you enable the 4play API. + // 4play password. Please set this to something secure if you use this extension. // This password is used to POST sessions to /api/v2/provide_sesh + // It's also used as the external endpoint password. const FPLAY_PASSWORD = "cnc"; + // Endpoint 4get hits to render webpages + // Eg: null=no endpoint provided + // Or "https://your-endpoint.com/my-endpoint" (excluding get parameters like &pw) + const FPLAY_EXTERNAL_ENDPOINT = null; + // // BOT PROTECTION // diff --git a/docs/configure.md b/docs/configure.md index d65caaa..2b6c246 100644 --- a/docs/configure.md +++ b/docs/configure.md @@ -79,3 +79,16 @@ Done! The scraper you chose should now be using the rotating proxies. When askin ## Important! If you ever test out a `socks5` proxy locally on your machine and find out it works but doesn't on your server, try supplying the `socks5_hostname` protocol instead. Hopefully this tip can save you 3 hours of your life! + +# 4play +4play is a botfaggotry Firefox extension. I made it, and it allows me to control a Firefox browser through code. More information about the project [here](https://git.lolcat.ca/lolcat/4play). It allows you to scrape Google (& probably more sites in the future) + +1. Install the [firefox extension](https://addons.mozilla.org/en-US/firefox/addon/4play/) onto a clean Firefox profile. Make sure you have a real display connected and that you use real hardware with an user-grade GPU. Websites detect this shit now. +2. Navigate to `/extra/4play/` +3. Install the 4play server: `npm install @lawlers/4play` +4. Edit the `render-server.js` file. Change the line `var password = "cnc";` so that it uses a secure password. +5. Edit `/data/config.php` and update the 4play password. Also update the `FPLAY_EXTERNAL_ENDPOINT` so that it points to the render HTTP server. + +Now, if you did everything correctly, you should be able to render search pages on scrapers that require it. If your render server sits on a different server, I highly recommend setting up a reverse proxy to add TLS support. + +In my case, the 4get website & scrapers sits at the OVH datacenter, while the render server is at my home running on a mini-PC I bought for the occasion. This wouldn't have been possible without your financial support, thank you all!! diff --git a/extra/4play/page-render.js b/extra/4play/page-render.js new file mode 100644 index 0000000..7fb59d4 --- /dev/null +++ b/extra/4play/page-render.js @@ -0,0 +1,341 @@ +const fplay = require("@lawlers/4play"); +const http = require("http"); + +// 4play settings +var port = 3030; +var timeout = 30000; +var password = "cnc"; + +// ================================================== + +// webserver + +var global_ws = null; +const webserver_port = 3000; + +var callback_q = new Map(); + +const server = http.createServer(async function(req, res){ + + const parsed = new URL(req.url, "http://localhost"); + + const url = parsed.searchParams.get("url"); + const pw = parsed.searchParams.get("pw"); + var proxy = parsed.searchParams.get("proxy"); + const url_match = parsed.searchParams.get("url_match"); + const content_match = parsed.searchParams.get("content_match"); + const fail_url_match = parsed.searchParams.get("fail_url_match"); + const fail_content_match = parsed.searchParams.get("fail_content_match"); + + var container = parsed.searchParams.get("container"); + + if(fplay.browser_connected === false){ + + res.writeHead( + 400, + { + "Content-Type": "text/plain" + } + ); + + res.end("No browser available to render the motherfucking page"); + return; + } + + var errors = []; + + if( + pw === null || + pw != password + ){ + + errors.push("pw"); + } + + if(url === null){ errors.push("url"); } + if(proxy === null){ errors.push("proxy"); } + + if(errors.length !== 0){ + + res.writeHead( + 400, + { + "Content-Type": "text/plain" + } + ); + + res.end("The following parameters are missing or invalid: " + errors.join(", ")); + return; + } + + console.log("Rendering " + url); + + if( + container === null || + await fplay.container_exists(global_ws, container) === false + ){ + + container = await fplay.container_create(global_ws); + + proxy = get_proxy(proxy); + + // assign proxy + if(proxy.config !== false){ + + await fplay.container_attach_proxy( + global_ws, + container, + proxy.config + ); + } + }else{ + + proxy = {raw: proxy}; + } + + // open tab + const newtab = await fplay.tab_open(global_ws, url, false, container); + + if(newtab === false){ + + // failed to open tab + res.writeHead( + 400, + { + "Content-Type": "text/plain", + "X-Proxy": proxy.raw, + "X-Container-ID": typeof container.id == "undefined" ? container : container.id + } + ); + + res.end("Failed to open tab"); + return; + } + + // callback will be handled in "web_response" + callback_q.set( + newtab.id, + { + http_res: res, + proxy: proxy, + container: container, + url_match: url_match, + content_match: content_match, + fail_url_match: fail_url_match, + fail_content_match: fail_content_match + } + ); + + // close tab in 30 seconds + await fplay.wait_random(15000, 35000); + await fplay.tab_close(global_ws, newtab); +}); + +server.listen(webserver_port, function(){ + + console.log("http server running"); +}); + +function get_proxy(parts){ + + parts = parts.split(":"); + // format: type:address:port:username:password + // PHP explode(":", $ip, 5) splits into max 5 parts + var type = parts[0]; + const address = parts[1]; + const port = parts[2]; + const username = parts[3]; + const password = parts[4]; + + var proxy_dns = true; + + switch(type){ + case "raw_ip": + case "ip": + // no proxy, use direct connection + return { + config: false, + raw: "raw_ip::::" + } + break; + + case "socks5": + type = "socks"; + proxy_dns = false; + break; + + case "socks4a": + type = "socks4"; + break; + + case "socks4": + proxy_dns = false; + break; + + case "socks5_hostname": + case "socks5h": + case "socks5a": + type = "socks"; + break; + + case "http": + case "https": + // do nothing + break; + + default: + // invalid proxy provided + return { + config: false, + raw: "raw_ip::::" + } + } + + return { + config: { + type: type, + host: address, + port: port, + username: username, + password: password, + proxyDNS: proxy_dns + }, + raw: proxy_line + }; +} + +fplay.event.on("server_ready", function(){ + + console.log("listening on port " + port + " (timeout=" + timeout + ")"); +}); + +fplay.event.on("browser_connect", async function(ws){ + + global_ws = ws; + + const ua = await fplay.get_ua(ws); + console.log("New browser instance connected: " + ua); + + await fplay.close_all_tabs(ws); + await fplay.delete_all_containers(ws); + + fplay.web_response_whitelist(ws, ["main_frame"]); + + // clean up + const blanktab = await fplay.close_all_tabs(ws); + await fplay.delete_all_containers(ws); +}); + +fplay.event.on("web_response", async function(page){ + + var callback_data = callback_q.get(page.id); + + if(typeof callback_data == "undefined"){ return; } + + var body = page.body.toString(); + + var url_match = page.url.match(new RegExp(callback_data.url_match, "i")); + var content_match = body.match(new RegExp(callback_data.content_match, "i")); + var fail_url_match = page.url.match(new RegExp(callback_data.fail_url_match, "i")); + var fail_content_match = body.match(new RegExp(callback_data.fail_content_match, "i")); + + // handle positive response + if( + ( + callback_data.url_match === null && + callback_data.content_match === null + ) || + ( + url_match && + content_match + ) || + ( + url_match && + callback_data.content_match === null + ) || + ( + callback_data.url_match === null && + content_match + ) + ){ + + callback_data.http_res.writeHead( + 200, + { + "Content-Type": "text/plain;charset=UTF-8", + "X-Proxy": callback_data.proxy.raw, + "X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id + } + ); + + callback_data.http_res.end(body); + + callback_q.delete(page.id); // cleanup callback only + return; + } + + // handle negative response + if( + ( + callback_data.fail_url_match !== null && + callback_data.fail_content_match !== null + ) && + ( + ( + fail_url_match && + fail_content_match + ) || + ( + fail_url_match && + callback_data.fail_content_match === null + ) || + ( + callback_data.fail_url_match === null && + fail_content_match + ) + ) + ){ + + callback_data.http_res.writeHead( + 400, + { + "Content-Type": "text/plain", + "X-Proxy": callback_data.proxy.raw, + "X-Container-ID": callback_data.container.id + } + ); + + callback_data.http_res.end("Reached fail state (matched fail_url_match or fail_content_match)"); + + // cleanup browser + callback + await fplay.tab_close(global_ws, page.id); + await fplay.container_delete(global_ws, callback_data.container); + callback_q.delete(page.id); + } +}); + +fplay.event.on("dom_load_fail", async function(page){ + + var callback_data = callback_q.get(page.id); + + if(typeof callback_data == "undefined"){ return; } + + callback_q.delete(page.id); + + callback_data.http_res.writeHead( + 400, + { + "Content-Type": "text/plain", + "X-Proxy": callback_data.proxy.raw, + "X-Container-ID": typeof callback_data.container.id == "undefined" ? callback_data.container : callback_data.container.id + } + ); + + callback_data.http_res.end("Network error (" + page.error + ")"); + + // cleanup browser + await fplay.tab_close(global_ws, page.id); + await fplay.container_delete(global_ws, callback_data.container); +}); + +fplay.init(port, password, timeout); diff --git a/lib/backend.php b/lib/backend.php index 68ac270..8a523bb 100644 --- a/lib/backend.php +++ b/lib/backend.php @@ -4,6 +4,9 @@ class backend{ public function __construct($scraper){ $this->scraper = $scraper; + $this->db = null; + $this->db_path_prefix = "data/sessions/"; + $this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite"; } /* @@ -208,4 +211,127 @@ class backend{ $payload[1] // proxy ]; } + + // + // Sessions + // + public function session_init_db($force_init = false){ + + if( + $force_init || + $this->db === null + ){ + + $this->db = new SQLite3($this->db_path); + + // create table if it doesnt exist + $this->db->exec( + "CREATE TABLE IF NOT EXISTS sessions ( " . + "id INTEGER PRIMARY KEY AUTOINCREMENT, " . + "created_at INTEGER NOT NULL, " . + "proxy TEXT NOT NULL, " . + "headers_json TEXT NOT NULL" . + ")" + ); + + return true; + } + + return false; + } + + public function session_count(){ + + $this->session_init_db(); + + $stmt = $this->db->prepare( + "SELECT COUNT(*) as count FROM sessions" + ); + + $result = $stmt->execute(); + $row = $result->fetchArray(SQLITE3_ASSOC); + + return (int)($row["count"] ?? 0); + } + + public function session_write($proxy, $headers){ + + $this->session_init_db(); + + $stmt = $this->db->prepare( + "INSERT INTO sessions ( " . + "created_at, " . + "proxy, " . + "headers_json " . + ") VALUES ( " . + ":created_at, " . + ":proxy, " . + ":headers_json" . + ")" + ); + + $stmt->bindValue(':created_at', time(), SQLITE3_INTEGER); + $stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT); + $stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT); + + $stmt->execute(); + + return $this->db->lastInsertRowID(); + } + + public function session_get(){ + + $count = $this->session_count(); + + if($count === 0){ + + return false; + } + + $offset = apcu_inc("d." . $this->scraper) % $count; + + $stmt = $this->db->prepare( + "SELECT * " . + "FROM sessions " . + "ORDER BY id ASC " . + "LIMIT 1 OFFSET :offset" + ); + + $stmt->bindValue(':offset', $offset, SQLITE3_INTEGER); + + $result = $stmt->execute(); + $row = $result->fetchArray(SQLITE3_ASSOC); + + return [ + "id" => $row["id"], + "created_at" => $row["created_at"], + "proxy" => $row["proxy"], + "headers" => json_decode($row["headers_json"], true) + ]; + } + + public function session_destroy_list(){ + + if( + !preg_match( + '/^[A-Za-z0-9-]+$/', + $this->scraper + ) + ){ + + // path traversal + return false; + } + + if( + file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") && + is_file("{$this->db_path_prefix}{$this->scraper}.sqlite") + ){ + + unlink("{$this->db_path_prefix}{$this->scraper}.sqlite"); + return true; + } + + return false; + } } diff --git a/scraper/google.php b/scraper/google.php index f528fb1..49f575c 100644 --- a/scraper/google.php +++ b/scraper/google.php @@ -1,8 +1,5 @@ backend = new backend("google"); - - $this->message = "Still working on a Google scraper that uses a headful browser. It will require Firefox + a webExtension running on a dedicated server. Waiting for my EDID adapter and we can get the show going. In the meantime, use the Google CSE/API or Yahoo JP/Startpage scrapers. They're all crippled in their own special ways but they're serviceable I guess."; } public function getfilters($page){ @@ -507,79 +502,1635 @@ class google{ } } - private function get($proxy, $url, $get = []){ + private function get($proxy, $url, $get = [], $container = null){ - $curlproc = curl_init(); - - if($get !== []){ - $get = http_build_query($get); - $url .= "?" . $get; - } - - curl_setopt($curlproc, CURLOPT_URL, $url); - - curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding - - curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); - curl_setopt($curlproc, CURLOPT_HTTPHEADER, [ - "User-Agent: " . config::USER_AGENT, - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", - "Accept-Language: en-US,en;q=0.5", - "Accept-Encoding: gzip", - "DNT: 1", - "Connection: keep-alive", - "Upgrade-Insecure-Requests: 1", - "Sec-Fetch-Dest: document", - "Sec-Fetch-Mode: navigate", - "Sec-Fetch-Site: none", - "Sec-Fetch-User: ?1", - "Priority: u=1", - "TE: trailers" - ]); - - curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); - curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); - curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); - curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); - - // follow redirects - curl_setopt($curlproc, CURLOPT_FOLLOWLOCATION, true); - - $this->backend->assign_proxy($curlproc, $proxy); - - $data = curl_exec($curlproc); - - if(curl_errno($curlproc)){ + if(config::FPLAY_EXTERNAL_ENDPOINT === null){ - throw new Exception(curl_error($curlproc)); + throw new Exception("This scraper requires 4play, and the instance maintainer did not configure it. Please refer to the documentation for more information"); } - curl_close($curlproc); + try{ + $curlproc = curl_init(); + + if($get !== []){ + $get = http_build_query($get); + $url .= "?" . $get; + } + + if($container === null){ + + $url = + config::FPLAY_EXTERNAL_ENDPOINT . + "?url=" . urlencode($url) . + "&pw=" . urlencode(config::FPLAY_PASSWORD) . + "&container=" . urlencode($container) . + "&proxy=" . urlencode($proxy) . + "&url_match=" . urlencode("&sei=") . + "&fail_url_match=" . urlencode("\/sorry\/"); + }else{ + + $url = + config::FPLAY_EXTERNAL_ENDPOINT . + "?url=" . urlencode($url) . + "&pw=" . urlencode(config::FPLAY_PASSWORD) . + "&container=" . urlencode($container) . + "&proxy=" . urlencode($proxy); + } + + $header_out = []; + curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$header_out){ + + $length = strlen($header); + $header_out[] = trim($header); + + return $length; + }); + + curl_setopt($curlproc, CURLOPT_URL, $url); + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + + curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0); + curl_setopt($curlproc, CURLOPT_HTTPHEADER, [ + "User-Agent: " . config::USER_AGENT_FRIENDLY, + "Accept: text/plain" + ]); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + curl_setopt($curlproc, CURLOPT_FOLLOWLOCATION, true); + + //$this->backend->assign_proxy($curlproc, $proxy); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + }catch(Exception $error){ + + throw new Exception("4play API is currently unreachable. Render server offline (probably) woopsies"); + } - return $data; + // check for failstate + if( + str_contains( + $header_out[0], + "400" + ) + ){ + + throw new Exception("Page renderer returned an error: " . $data); + } + + // parse headers + $container = null; + + foreach($header_out as $header){ + + $header = explode(":", $header, 2); + + if(strtolower($header[0]) == "x-container-id"){ + + $container = trim($header[1]); + break; + } + } + + if($container === null){ + + throw new Exception("The 4play API did not return a valid container ID. Make sure your configuration forwards X-Container-ID headers properly"); + } + + return [ + "data" => $data, + "container" => $container + ]; } public function web($get){ - throw new Exception($this->message); + if($get["npt"]){ + + [$get, $proxy] = $this->backend->get($get["npt"], "web"); + + $get = json_decode($get, true); + $container = $get["c"]; + $get = $get["u"]; + + $data = + $this->get( + $proxy, + "https://www.google.com" . $get, + [], + $container + ); + + $html = &$data["data"]; + $container = &$data["container"]; + + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + /* + $session = $this->backend->session_get(); + + if(config::FPLAY_ENABLE_API === false){ + + throw new Exception("This scraper requires the 4play session generator. Check the documentation for more information"); + } + + if($session === false){ + + throw new Exception("Session database is empty"); + } + + $proxy = &$session["proxy"]; + $headers = &$session["headers"]; + */ + + $proxy = $this->backend->get_ip(); + + // + // Generate params + // + + $params = [ + "q" => $get["s"], + "hl" => "en", + "udm" => 14 + ]; + + // country + if($get["country"] != "any"){ + + $params["gl"] = $country; + } + + // nsfw + $params["safe"] = $get["nsfw"] == "yes" ? "off" : "active"; + + // language + if($get["lang"] != "any"){ + + $params["lr"] = "lang_" . $lang; + } + + // generate tbs + $tbs = []; + + // get date + $get["older"] = $get["older"] === false ? null : date("m/d/Y", $older); + $get["newer"] = $get["newer"] === false ? null : date("m/d/Y", $newer); + + if( + $get["older"] !== null || + $get["newer"] !== null + ){ + + $tbs["cdr"] = "1"; + $tbs["cd_min"] = $get["newer"]; + $tbs["cd_max"] = $get["older"]; + } + + // spellcheck filter + if($get["spellcheck"] == "no"){ + + $params["nfpr"] = "1"; + } + + if(count($tbs) !== 0){ + + $params["tbs"] = ""; + + foreach($tbs as $key => $value){ + + $params["tbs"] .= $key . ":" . $value . ","; + } + + $params["tbs"] = rtrim($params["tbs"], ","); + } + + $data = $this->get( + $proxy, + "https://www.google.com/search", + $params, + null + ); + + $html = &$data["data"]; + $container = &$data["container"]; + + //file_put_contents("scraper/google-jp.html", $html); + //$html = file_get_contents("scraper/google-jp.html"); + } + + $out = [ + "status" => "ok", + "spelling" => [ + "type" => "no_correction", + "using" => null, + "correction" => null + ], + "npt" => null, + "answer" => [], + "web" => [], + "image" => [], + "video" => [], + "news" => [], + "related" => [] + ]; + + $this->fuckhtml->load($html); + + // init + $this->detect_sorry(); + $this->parsestyles(); + $this->scrape_dimg($html); + $this->scrape_imagearr($html); + + $content = + $this->fuckhtml + ->getElementById( + "rso" + ); + + if($content === null){ + + throw new Exception("Failed to grep search div"); + } + + // get encoded links + $encoded_links = + explode( + "function(){var m={", + $html, + 2 + ); + + if(count($encoded_links) === 2){ + + $encoded_links = + json_decode( + $this->fuckhtml + ->extract_json( + "{" . $encoded_links[1] + ), + true + ); + }else{ + + $encoded_links = []; + } + + // get the "did you mean" bullshit + $taw = + $this->fuckhtml + ->getElementById("taw"); + + if($taw !== null){ + $this->fuckhtml->load($taw); + + $as = + $this->fuckhtml + ->getElementsByTagName("a"); + + $links = []; + + foreach($as as $a){ + + $href = + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ); + + if(preg_match('/^\/search\?/', $href)){ + + $links[] = + $this->fuckhtml + ->getTextContent( + $a + ); + } + } + + switch(count($links)){ + + case 1: + $out["spelling"] = [ + "type" => "including", + "using" => $params["q"], + "correction" => $links[0] + ]; + break; + + case 2: + $out["spelling"] = [ + "type" => "including", + "using" => $links[0], + "correction" => $links[1] + ]; + break; + } + + // reset + $this->fuckhtml->load($html); + } + + $bottom = + $this->fuckhtml + ->getElementById("botstuff"); + + if($bottom !== null){ + + $this->fuckhtml->load($bottom); + + // get next page thingy + $npt = + $this->fuckhtml + ->getElementById( + "pnnext" + ); + + if($npt !== false){ + + $out["npt"] = + $this->backend->store( + json_encode([ + "u" => + $this->fuckhtml + ->getTextContent( + $npt["attributes"]["href"] + ), + "c" => $container + ]), + "web", + $proxy + ); + } + + // get lumendatabase notices + $notices = []; + $as = + $this->fuckhtml + ->getElementsByTagName("a"); + + foreach($as as $a){ + + if(!isset($a["attributes"]["href"])){ continue; } + + $link = + $this->fuckhtml + ->getTextContent( + $a["attributes"]["href"] + ); + + if( + preg_match( + '/^https?:\/\/lumendatabase\.org/', + $link + ) + ){ + + $notices[] = $link; + } + } + + // push notices to dom + if(count($notices) !== 0){ + + $notices_dom = []; + + $i = 0; + $c = count($notices); + + $notices_dom = [ + [ + "type" => "text", + "value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: " + ] + ]; + + foreach($notices as $n){ + + $i++; + + $notices_dom[] = [ + "type" => "link", + "value" => "Notice #{$i}", + "url" => $n + ]; + + if($i !== count($notices)){ + + $notices_dom[] = [ + "type" => "text", + "value" => ", " + ]; + } + } + + $out["answer"][] = [ + "title" => "LumenDB notices", + "description" => $notices_dom, + "url" => null, + "thumb" => null, + "table" => [], + "sublink" => [] + ]; + } + } + + $this->fuckhtml->load($content); + + // get search results + $results = + $this->fuckhtml + ->getElementsByAttributeName( + "data-rpos", + "div" + ); + + foreach($results as $result){ + + $this->fuckhtml->load($result); + + // get title + $title = + $this->fuckhtml + ->getElementsByTagName("h3"); + + if(count($title) === 0){ + + // should not happen + continue; + } + + $title = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $title[0] + ) + ); + + $url = + $this->fuckhtml + ->getElementsByTagName( + "a" + ); + + if( + count($url) === 0 || + !isset($url[0]["attributes"]["href"]) + ){ + + continue; + } + + $url = + $this->fuckhtml + ->getTextContent( + $url[0]["attributes"]["href"] + ); + + if( + preg_match( + '/^\/goto\?url/', + $url + ) + ){ + + // encrypted url detected + // get reference div + $refdivs = + $this->fuckhtml + ->getElementsByAttributeName( + "id", + "div" + ); + + foreach($refdivs as $r){ + + if( + preg_match( + '/^atritem-/', + $r["attributes"]["id"] + ) && + isset($r["attributes"]["jsdata"]) + ){ + + $ref = explode(";", $r["attributes"]["jsdata"]); + $ref = $ref[count($ref) - 1]; + + if(isset($encoded_links[$ref][32][3][0])){ + + $url = $encoded_links[$ref][32][3][0]; + } + break; + } + } + } + + // get citation text. contains table elements + $table = []; + $date = null; + + $cite = + $this->fuckhtml + ->getElementsByTagName("cite"); + + if(isset($cite[0])){ + + $texts = + explode( + " · ", + $this->fuckhtml + ->getTextContent($cite[0]) + ); + + foreach($texts as $text){ + + $text = trim($text); + + preg_match( + '/([0-9+kmb,.]+) +(likes?|views?|comments?|followers?|subscribers?|(?:hours?|days?|months?|years?) +ago)/i', + $text, + $probe + ); + + if(isset($probe[1])){ + + // is it date? + if( + preg_match( + '/ago$/i', + $text + ) + ){ + + $time = strtotime($text); + + if($time !== false){ $date = $time; } + continue; + } + + // anything else? + $table[ucfirst($probe[2])] = trim($probe[1]); + } + } + } + + // get description/image wrapper + $desc_img_wrappers = + $this->fuckhtml + ->getElementsByAttributeName( + "data-sncf", + "div" + ); + + $description = null; + $thumb = [ + "url" => null, + "ratio" => null + ]; + + foreach($desc_img_wrappers as $wrapper){ + + $this->fuckhtml->load($wrapper); + + // probe for image + $image = + $this->fuckhtml + ->getElementsByTagName( + "img" + ); + + if(count($image) !== 0){ + + // handle thumbnails + $i = null; + + if(isset($image[0]["attributes"]["id"])){ + + $i = $this->getdimg($image[0]["attributes"]["id"]); + } + + if($i !== null){ + + $thumb = [ + "ratio" => "1:1", + "url" => $i + ]; + } + }else{ + + if($description === null){ + + // parse it as description + $description = + $this->titledots( + $this->fuckhtml + ->getTextContent( + $wrapper + ) + ); + }else{ + + // parse table elements + $ts = + explode( + " · ", + $this->fuckhtml + ->getTextContent( + $wrapper + ) + ); + + foreach($ts as $t){ + + $t = trim(html_entity_decode($t)); + + if(preg_match("/\p{Sc}/u", $t)){ + + $table["Price"] = $t; + continue; + } + + if(str_contains($t, "stock")){ + + $table["Stock"] = $t; + continue; + } + + if(str_contains($t, "return")){ + + $table["Returns"] = $t; + continue; + } + + if(str_contains($t, "delivery")){ + + $table["Delivery"] = $t; + continue; + } + + // 5.0(590,004) + if(preg_match('/^([0-9kmb\/%.,]+) *(\([0-9kmb,.]+\))?/', $t, $match)){ + + if(isset($match[2])){ + + $table["Ratings"] = "{$match[1]} {$match[2]}"; + continue; + } + } + + if(str_contains($t, "rating")){ + + $table["Ratings"] = $t; + continue; + } + } + } + } + } + + // extract date from description + // Nov 6, 2017 — Keira Hagai from Jak and Daxter. I took off her belt/suspenders/goggles. Originally posted by DemonBoy on Models Resource, and edited and posted to Sketchfab + if($date === null){ + $date_probe = + explode( + " — ", + $description, + 2 + ); + + if(count($date_probe) === 2){ + + $time = strtotime($date_probe[0]); + + if($time !== false){ + + $description = $date_probe[1]; + $date = $time; + } + } + } + + $sublinks = []; + + $out["web"][] = [ + "title" => $title, + "description" => $description, + "url" => $url, + "date" => $date, + "type" => "web", + "thumb" => $thumb, + "sublink" => $sublinks, + "table" => $table + ]; + } + + return $out; } + public function image($get){ - throw new Exception($this->message); + + // generate parameters + if($get["npt"]){ + + [$params, $proxy] = + $this->backend->get( + $get["npt"], + "images" + ); + + $params = json_decode($params, true); + $container = $params["c"]; + unset($params["c"]); + }else{ + + $search = $get["s"]; + if(strlen($search) === 0){ + + throw new Exception("Search term is empty!"); + } + + $proxy = $this->backend->get_ip(); + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $time = $get["time"]; + $size = $get["size"]; + $ratio = $get["ratio"]; + $color = $get["color"]; + $type = $get["type"]; + $format = $get["format"]; + $rights = $get["rights"]; + + $params = [ + "q" => $search, + "udm" => "2", // get images + "start" => 0 + ]; + + // country (image search uses cr instead of gl) + if($country != "any"){ + + $params["cr"] = "country" . strtoupper($country); + } + + // nsfw + $params["safe"] = $nsfw == "yes" ? "off" : "active"; + + // generate tbs + $tbs = []; + + // time + if($time != "any"){ + + $tbs["qdr"] = $time; + } + + // size + if($size != "any"){ + + $params["imgsz"] = $size; + } + + // ratio + if($ratio != "any"){ + + $params["imgar"] = $ratio; + } + + // color + if($color != "any"){ + + if( + $color == "color" || + $color == "trans" + ){ + + $params["imgc"] = $color; + }elseif($color == "bnw"){ + + $params["imgc"] = "gray"; + }else{ + + $tbs["ic"] = "specific"; + $tbs["isc"] = $color; + } + } + + // type + if($type != "any"){ + + $tbs["itp"] = $type; + } + + // format + if($format != "any"){ + + $params["as_filetype"] = $format; + } + + // rights (tbs) + if($rights != "any"){ + + $tbs["sur"] = $rights; + } + + // append tbs + if(count($tbs) !== 0){ + + $params["tbs"] = ""; + + foreach($tbs as $key => $value){ + + $params["tbs"] .= $key . ":" . $value . ","; + } + + $params["tbs"] = rtrim($params["tbs"], ","); + } + } + + //$html = file_get_contents("scraper/google.html"); + + $data = $this->get( + $proxy, + "https://www.google.com/search", + $params + ); + + $html = &$data["data"]; + $container = &$data["container"]; + + $this->fuckhtml->load($html); + + $this->detect_sorry(); + + // get javascript images + $this->scrape_imagearr($html); + + $out = [ + "status" => "ok", + "npt" => null, + "image" => [] + ]; + + $images = + $this->fuckhtml + ->getElementsByClassName( + "ivg-i", + "div" + ); + + foreach($images as $div){ + + $this->fuckhtml->load($div); + + $image = + $this->fuckhtml + ->getElementsByTagName("img")[0]; + + // make sure we dont attempt to show an image we dont have data for + if( + isset($div["attributes"]["data-docid"]) && + isset($this->image_arr[$div["attributes"]["data-docid"]]) + ){ + + $source = + $this->image_arr[ + $div["attributes"]["data-docid"] + ]; + }else{ + + continue; + } + + $out["image"][] = [ + "title" => + $this->titledots( + $this->fuckhtml + ->getTextContent( + $image["attributes"]["alt"] + ) + ), + "source" => $source, + "url" => + $this->fuckhtml + ->getTextContent( + $div["attributes"]["data-lpage"] + ) + ]; + } + + // as usual, no way to check if there is a next page reliably + if(count($out["image"]) > 50){ + + if(!isset($params["start"])){ + + $params["start"] = 10; + }else{ + + $params["start"] += 10; + } + + $out["npt"] = + $this->backend + ->store( + json_encode($params), + "image", + $proxy + ); + } + + return $out; } + public function video($get){ - throw new Exception($this->message); + + if($get["npt"]){ + + [$params, $proxy] = $this->backend->get($get["npt"], "video"); + $params = json_decode($params, true); + + $container = $params["c"]; + unset($params["c"]); + + }else{ + $search = $get["s"]; + $country = $get["country"]; + $nsfw = $get["nsfw"]; + $older = $get["older"]; + $newer = $get["newer"]; + $duration = $get["duration"]; + $quality = $get["quality"]; + $captions = $get["captions"]; + + $container = null; + $proxy = $this->backend->get_ip(); + + $params = [ + "q" => $search, + "udm" => "7", + "hl" => "en" + ]; + + // country + if($country != "any"){ + + $params["gl"] = $country; + } + + // nsfw + $params["safe"] = $nsfw == "yes" ? "off" : "active"; + + $tbs = []; + + // get date + $older = $older === false ? null : date("m/d/Y", $older); + $newer = $newer === false ? null : date("m/d/Y", $newer); + + if( + $older !== null || + $newer !== null + ){ + + $tbs["cdr"] = "1"; + $tbs["cd_min"] = $newer; + $tbs["cd_max"] = $older; + } + + // duration + if($duration != "any"){ + + $tbs[] = "dur:" . $duration; + } + + // quality + if($quality != "any"){ + + $tbs[] = "hq:" . $quality; + } + + // captions + if($captions != "any"){ + + $tbs[] = "cc:" . $captions; + } + + // append tbs + if(count($tbs) !== 0){ + + $params["tbs"] = + implode(",", $tbs); + } + } + + //$html = file_get_contents("scraper/google.html"); + $data = $this->get( + $proxy, + "https://www.google.com/search", + $params, + $container + ); + + $html = &$data["data"]; + $container = &$data["container"]; + + if(!isset($params["start"])){ + + $params["start"] = 0; + } + $params["start"] += 20; + + $this->fuckhtml->load($html); + + // + // Parse web video page + // + $this->detect_sorry(); + + // parse all