scraper = $scraper; $this->db = null; $this->db_path_prefix = "data/sessions/"; $this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite"; } /* Proxy stuff */ public function get_ip($proxy_index_raw = null){ $pool = constant("config::PROXY_" . strtoupper($this->scraper)); if($pool === false){ // we don't want a proxy, fuck off! return 'raw_ip::::'; } // indent if($proxy_index_raw === null){ $proxy_index_raw = apcu_inc("p." . $this->scraper); } $proxylist = file_get_contents("data/proxies/" . $pool . ".txt"); $proxylist = explode("\n", $proxylist); // ignore empty or commented lines $proxylist = array_filter($proxylist, function($entry){ $entry = ltrim($entry); return strlen($entry) > 0 && substr($entry, 0, 1) != "#"; }); $proxylist = array_values($proxylist); if(count($proxylist) === 0){ throw new Exception("A proxy list was specified but it's empty!"); } //echo $proxylist[$proxy_index_raw % count($proxylist)]; return $proxylist[$proxy_index_raw % count($proxylist)]; } // this function is also called directly on nextpage public function assign_proxy(&$curlproc, string $ip){ // parse proxy line [ $type, $address, $port, $username, $password ] = explode(":", $ip, 5); switch($type){ case "raw_ip": return; break; case "http": case "https": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_HTTP); curl_setopt($curlproc, CURLOPT_PROXY, $type . "://" . $address . ":" . $port); break; case "socks4": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; case "socks5": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; case "socks4a": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; case "socks5_hostname": case "socks5h": case "socks5a": curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME); curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port); break; } if($username != ""){ curl_setopt($curlproc, CURLOPT_PROXYUSERPWD, $username . ":" . $password); } } // API key rotation public function get_key(){ $keys = file_get_contents("data/api_keys/" . $this->scraper . ".txt"); $keys = explode("\n", $keys); $keys = array_filter($keys, function($entry){ $entry = ltrim($entry); return strlen($entry) > 0 && substr($entry, 0, 1) != "#"; }); $keys = array_values($keys); if(count($keys) === 0){ throw new Exception("Please specify API keys in data/api_keys/" . $this->scraper . ".txt"); } $increment = apcu_inc("s." . $this->scraper) % count($keys); return [ "key" => $keys[$increment], "increment" => $increment ]; } /* Next page stuff */ public function store(string $payload, string $page, string $proxy){ $key = sodium_crypto_secretbox_keygen(); $nonce = random_bytes(SODIUM_CRYPTO_SECRETBOX_NONCEBYTES); $requestid = apcu_inc("requestid"); apcu_store( $page[0] . "." . // first letter of page name $this->scraper . // scraper name $requestid, [ $nonce, $proxy, // compress and encrypt sodium_crypto_secretbox( gzdeflate($payload), $nonce, $key ) ], 900 // cache information for 15 minutes ); return $this->scraper . $requestid . "." . rtrim(strtr(base64_encode($key), '+/', '-_'), '='); } public function get(string $npt, string $page){ $page = $page[0]; $explode = explode(".", $npt, 2); if(count($explode) !== 2){ throw new Exception("Malformed nextPageToken!"); } $apcu = $page . "." . $explode[0]; $key = $explode[1]; $payload = apcu_fetch($apcu); if($payload === false){ throw new Exception("The next page token is invalid or has expired!"); } $key = base64_decode( str_pad( strtr($key, '-_', '+/'), strlen($key) % 4, '=', STR_PAD_RIGHT ) ); // decrypt and decompress data $payload[2] = gzinflate( sodium_crypto_secretbox_open( $payload[2], // data $payload[0], // nonce $key ) ); if($payload[2] === false){ throw new Exception("The next page token is invalid or has expired!"); } // remove the key after using successfully apcu_delete($apcu); return [ $payload[2], // data $payload[1] // proxy ]; } // // Sessions // public function session_init_db($force_init = false){ if( $force_init || $this->db === null ){ $this->db = new SQLite3($this->db_path); // create table if it doesnt exist $this->db->exec( "CREATE TABLE IF NOT EXISTS sessions ( " . "id INTEGER PRIMARY KEY AUTOINCREMENT, " . "created_at INTEGER NOT NULL, " . "proxy TEXT NOT NULL, " . "headers_json TEXT NOT NULL" . ")" ); return true; } return false; } public function session_count(){ $this->session_init_db(); $stmt = $this->db->prepare( "SELECT COUNT(*) as count FROM sessions" ); $result = $stmt->execute(); $row = $result->fetchArray(SQLITE3_ASSOC); return (int)($row["count"] ?? 0); } public function session_write($proxy, $headers){ $this->session_init_db(); $stmt = $this->db->prepare( "INSERT INTO sessions ( " . "created_at, " . "proxy, " . "headers_json " . ") VALUES ( " . ":created_at, " . ":proxy, " . ":headers_json" . ")" ); $stmt->bindValue(':created_at', time(), SQLITE3_INTEGER); $stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT); $stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT); $stmt->execute(); return $this->db->lastInsertRowID(); } public function session_get(){ $count = $this->session_count(); if($count === 0){ return false; } $offset = apcu_inc("d." . $this->scraper) % $count; $stmt = $this->db->prepare( "SELECT * " . "FROM sessions " . "ORDER BY id ASC " . "LIMIT 1 OFFSET :offset" ); $stmt->bindValue(':offset', $offset, SQLITE3_INTEGER); $result = $stmt->execute(); $row = $result->fetchArray(SQLITE3_ASSOC); return [ "id" => $row["id"], "created_at" => $row["created_at"], "proxy" => $row["proxy"], "headers" => json_decode($row["headers_json"], true) ]; } public function session_destroy_list(){ if( !preg_match( '/^[A-Za-z0-9-]+$/', $this->scraper ) ){ // path traversal return false; } if( file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") && is_file("{$this->db_path_prefix}{$this->scraper}.sqlite") ){ unlink("{$this->db_path_prefix}{$this->scraper}.sqlite"); return true; } return false; } }