338 lines
6.9 KiB
PHP
338 lines
6.9 KiB
PHP
<?php
|
|
class backend{
|
|
|
|
public function __construct($scraper){
|
|
|
|
$this->scraper = $scraper;
|
|
$this->db = null;
|
|
$this->db_path_prefix = "data/sessions/";
|
|
$this->db_path = "{$this->db_path_prefix}{$scraper}.sqlite";
|
|
}
|
|
|
|
/*
|
|
Proxy stuff
|
|
*/
|
|
public function get_ip($proxy_index_raw = null){
|
|
|
|
$pool = constant("config::PROXY_" . strtoupper($this->scraper));
|
|
if($pool === false){
|
|
|
|
// we don't want a proxy, fuck off!
|
|
return 'raw_ip::::';
|
|
}
|
|
|
|
// indent
|
|
if($proxy_index_raw === null){
|
|
|
|
$proxy_index_raw = apcu_inc("p." . $this->scraper);
|
|
}
|
|
|
|
$proxylist = file_get_contents("data/proxies/" . $pool . ".txt");
|
|
$proxylist = explode("\n", $proxylist);
|
|
|
|
// ignore empty or commented lines
|
|
$proxylist = array_filter($proxylist, function($entry){
|
|
$entry = ltrim($entry);
|
|
return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
|
|
});
|
|
|
|
$proxylist = array_values($proxylist);
|
|
|
|
if(count($proxylist) === 0){
|
|
|
|
throw new Exception("A proxy list was specified but it's empty!");
|
|
}
|
|
|
|
//echo $proxylist[$proxy_index_raw % count($proxylist)];
|
|
return $proxylist[$proxy_index_raw % count($proxylist)];
|
|
}
|
|
|
|
// this function is also called directly on nextpage
|
|
public function assign_proxy(&$curlproc, string $ip){
|
|
|
|
// parse proxy line
|
|
[
|
|
$type,
|
|
$address,
|
|
$port,
|
|
$username,
|
|
$password
|
|
] = explode(":", $ip, 5);
|
|
|
|
switch($type){
|
|
|
|
case "raw_ip":
|
|
return;
|
|
break;
|
|
|
|
case "http":
|
|
case "https":
|
|
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_HTTP);
|
|
curl_setopt($curlproc, CURLOPT_PROXY, $type . "://" . $address . ":" . $port);
|
|
break;
|
|
|
|
case "socks4":
|
|
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4);
|
|
curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
|
|
break;
|
|
|
|
case "socks5":
|
|
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5);
|
|
curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
|
|
break;
|
|
|
|
case "socks4a":
|
|
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS4A);
|
|
curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
|
|
break;
|
|
|
|
case "socks5_hostname":
|
|
case "socks5h":
|
|
case "socks5a":
|
|
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
|
|
curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
|
|
break;
|
|
}
|
|
|
|
if($username != ""){
|
|
|
|
curl_setopt($curlproc, CURLOPT_PROXYUSERPWD, $username . ":" . $password);
|
|
}
|
|
}
|
|
|
|
// API key rotation
|
|
public function get_key(){
|
|
|
|
$keys = file_get_contents("data/api_keys/" . $this->scraper . ".txt");
|
|
$keys = explode("\n", $keys);
|
|
|
|
$keys = array_filter($keys, function($entry){
|
|
$entry = ltrim($entry);
|
|
return strlen($entry) > 0 && substr($entry, 0, 1) != "#";
|
|
});
|
|
|
|
$keys = array_values($keys);
|
|
|
|
if(count($keys) === 0){
|
|
|
|
throw new Exception("Please specify API keys in data/api_keys/" . $this->scraper . ".txt");
|
|
}
|
|
|
|
$increment = apcu_inc("s." . $this->scraper) % count($keys);
|
|
return [
|
|
"key" => $keys[$increment],
|
|
"increment" => $increment
|
|
];
|
|
}
|
|
|
|
|
|
/*
|
|
Next page stuff
|
|
*/
|
|
public function store(string $payload, string $page, string $proxy){
|
|
|
|
$key = sodium_crypto_secretbox_keygen();
|
|
$nonce = random_bytes(SODIUM_CRYPTO_SECRETBOX_NONCEBYTES);
|
|
|
|
$requestid = apcu_inc("requestid");
|
|
|
|
apcu_store(
|
|
$page[0] . "." . // first letter of page name
|
|
$this->scraper . // scraper name
|
|
$requestid,
|
|
[
|
|
$nonce,
|
|
$proxy,
|
|
// compress and encrypt
|
|
sodium_crypto_secretbox(
|
|
gzdeflate($payload),
|
|
$nonce,
|
|
$key
|
|
)
|
|
],
|
|
900 // cache information for 15 minutes
|
|
);
|
|
|
|
return
|
|
$this->scraper . $requestid . "." .
|
|
rtrim(strtr(base64_encode($key), '+/', '-_'), '=');
|
|
}
|
|
|
|
public function get(string $npt, string $page){
|
|
|
|
$page = $page[0];
|
|
$explode = explode(".", $npt, 2);
|
|
|
|
if(count($explode) !== 2){
|
|
|
|
throw new Exception("Malformed nextPageToken!");
|
|
}
|
|
|
|
$apcu = $page . "." . $explode[0];
|
|
$key = $explode[1];
|
|
|
|
$payload = apcu_fetch($apcu);
|
|
|
|
if($payload === false){
|
|
|
|
throw new Exception("The next page token is invalid or has expired!");
|
|
}
|
|
|
|
$key =
|
|
base64_decode(
|
|
str_pad(
|
|
strtr($key, '-_', '+/'),
|
|
strlen($key) % 4,
|
|
'=',
|
|
STR_PAD_RIGHT
|
|
)
|
|
);
|
|
|
|
// decrypt and decompress data
|
|
$payload[2] =
|
|
gzinflate(
|
|
sodium_crypto_secretbox_open(
|
|
$payload[2], // data
|
|
$payload[0], // nonce
|
|
$key
|
|
)
|
|
);
|
|
|
|
if($payload[2] === false){
|
|
|
|
throw new Exception("The next page token is invalid or has expired!");
|
|
}
|
|
|
|
// remove the key after using successfully
|
|
apcu_delete($apcu);
|
|
|
|
return [
|
|
$payload[2], // data
|
|
$payload[1] // proxy
|
|
];
|
|
}
|
|
|
|
//
|
|
// Sessions
|
|
//
|
|
public function session_init_db($force_init = false){
|
|
|
|
if(
|
|
$force_init ||
|
|
$this->db === null
|
|
){
|
|
|
|
$this->db = new SQLite3($this->db_path);
|
|
|
|
// create table if it doesnt exist
|
|
$this->db->exec(
|
|
"CREATE TABLE IF NOT EXISTS sessions ( " .
|
|
"id INTEGER PRIMARY KEY AUTOINCREMENT, " .
|
|
"created_at INTEGER NOT NULL, " .
|
|
"proxy TEXT NOT NULL, " .
|
|
"headers_json TEXT NOT NULL" .
|
|
")"
|
|
);
|
|
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
public function session_count(){
|
|
|
|
$this->session_init_db();
|
|
|
|
$stmt = $this->db->prepare(
|
|
"SELECT COUNT(*) as count FROM sessions"
|
|
);
|
|
|
|
$result = $stmt->execute();
|
|
$row = $result->fetchArray(SQLITE3_ASSOC);
|
|
|
|
return (int)($row["count"] ?? 0);
|
|
}
|
|
|
|
public function session_write($proxy, $headers){
|
|
|
|
$this->session_init_db();
|
|
|
|
$stmt = $this->db->prepare(
|
|
"INSERT INTO sessions ( " .
|
|
"created_at, " .
|
|
"proxy, " .
|
|
"headers_json " .
|
|
") VALUES ( " .
|
|
":created_at, " .
|
|
":proxy, " .
|
|
":headers_json" .
|
|
")"
|
|
);
|
|
|
|
$stmt->bindValue(':created_at', time(), SQLITE3_INTEGER);
|
|
$stmt->bindValue(':proxy', $proxy, SQLITE3_TEXT);
|
|
$stmt->bindValue(':headers_json', json_encode($headers), SQLITE3_TEXT);
|
|
|
|
$stmt->execute();
|
|
|
|
return $this->db->lastInsertRowID();
|
|
}
|
|
|
|
public function session_get(){
|
|
|
|
$count = $this->session_count();
|
|
|
|
if($count === 0){
|
|
|
|
return false;
|
|
}
|
|
|
|
$offset = apcu_inc("d." . $this->scraper) % $count;
|
|
|
|
$stmt = $this->db->prepare(
|
|
"SELECT * " .
|
|
"FROM sessions " .
|
|
"ORDER BY id ASC " .
|
|
"LIMIT 1 OFFSET :offset"
|
|
);
|
|
|
|
$stmt->bindValue(':offset', $offset, SQLITE3_INTEGER);
|
|
|
|
$result = $stmt->execute();
|
|
$row = $result->fetchArray(SQLITE3_ASSOC);
|
|
|
|
return [
|
|
"id" => $row["id"],
|
|
"created_at" => $row["created_at"],
|
|
"proxy" => $row["proxy"],
|
|
"headers" => json_decode($row["headers_json"], true)
|
|
];
|
|
}
|
|
|
|
public function session_destroy_list(){
|
|
|
|
if(
|
|
!preg_match(
|
|
'/^[A-Za-z0-9-]+$/',
|
|
$this->scraper
|
|
)
|
|
){
|
|
|
|
// path traversal
|
|
return false;
|
|
}
|
|
|
|
if(
|
|
file_exists("{$this->db_path_prefix}{$this->scraper}.sqlite") &&
|
|
is_file("{$this->db_path_prefix}{$this->scraper}.sqlite")
|
|
){
|
|
|
|
unlink("{$this->db_path_prefix}{$this->scraper}.sqlite");
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
}
|