libsys/4get
1
0
Fork 0
forked from lolcat/4get
Code Pull Requests Activity

v8

Browse Source
This commit is contained in:
lolcat
2024-04-21 19:31:56 -04:00
parent 9e18327df6
commit 130358a9e0
16 changed files with 1385 additions and 457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
lib/backend.php
View File

@@ -93,31 +93,31 @@ class backend{
*/
public function store($payload, $page, $proxy){
$page = $page[0];
$password = random_bytes(256); // 2048 bit
$salt = random_bytes(16);
$key = hash_pbkdf2("sha512", $password, $salt, 20000, 32, true);
$iv =
random_bytes(
openssl_cipher_iv_length("aes-256-gcm")
);
$tag = "";
$out = openssl_encrypt($payload, "aes-256-gcm", $key, OPENSSL_RAW_DATA, $iv, $tag, "", 16);
$key = sodium_crypto_secretbox_keygen();
$nonce = random_bytes(SODIUM_CRYPTO_SECRETBOX_NONCEBYTES);
$requestid = apcu_inc("requestid");
apcu_store(
$page . "." .
$this->scraper .
$page[0] . "." . // first letter of page name
$this->scraper . // scraper name
$requestid,
gzdeflate($proxy . "," . $salt.$iv.$out.$tag),
900 // cache information for 15 minutes blaze it
[
$nonce,
$proxy,
// compress and encrypt
sodium_crypto_secretbox(
gzdeflate($payload),
$nonce,
$key
)
],
900 // cache information for 15 minutes
);
return
$this->scraper . $requestid . "." .
rtrim(strtr(base64_encode($password), '+/', '-_'), '=');
rtrim(strtr(base64_encode($key), '+/', '-_'), '=');
}
public function get($npt, $page){
@@ -137,7 +137,7 @@ class backend{
if($payload === false){
throw new Exception("The nextPageToken is invalid or has expired!");
throw new Exception("The next page token is invalid or has expired!");
}
$key =
@@ -150,47 +150,27 @@ class backend{
)
);
$payload = gzinflate($payload);
// get proxy
[
$proxy,
$payload
] = explode(",", $payload, 2);
$key =
hash_pbkdf2(
"sha512",
$key,
substr($payload, 0, 16), // salt
20000,
32,
true
);
$ivlen = openssl_cipher_iv_length("aes-256-gcm");
$payload =
openssl_decrypt(
substr(
$payload,
16 + $ivlen,
-16
),
"aes-256-gcm",
$key,
OPENSSL_RAW_DATA,
substr($payload, 16, $ivlen),
substr($payload, -16)
// decrypt and decompress data
$payload[2] =
gzinflate(
sodium_crypto_secretbox_open(
$payload[2], // data
$payload[0], // nonce
$key
)
);
if($payload === false){
if($payload[2] === false){
throw new Exception("The nextPageToken is invalid or has expired!");
throw new Exception("The next page token is invalid or has expired!");
}
// remove the key after using
// remove the key after using successfully
apcu_delete($apcu);
return [$payload, $proxy];
return [
$payload[2], // data
$payload[1] // proxy
];
}
}

63
lib/curlproxy.php
View File

@@ -290,30 +290,24 @@ class proxy{
if(isset($headers["content-type"])){
if($headers["content-type"] == "text/html"){
if(stripos($headers["content-type"], "text/html") !== false){
throw new Exception("Server returned an html document instead of image");
throw new Exception("Server returned html");
}
$tmp = explode(";", $headers["content-type"]);
for($i=0; $i<count($tmp); $i++){
if(
preg_match(
'/image\/([^ ]+)/i',
$headers["content-type"],
$match
)
){
if(
preg_match(
'/^image\/([^ ]+)/i',
$tmp[$i],
$match
)
){
$format = strtolower($match[1]);
if(substr(strtolower($format), 0, 2) == "x-"){
$format = strtolower($match[1]);
if(substr($format, 0, 2) == "x-"){
$format = substr($format, 2);
}
break;
$format = substr($format, 2);
}
}
}
@@ -351,6 +345,8 @@ class proxy{
private function stream($url, $referer, $format){
$this->clientcache();
$this->url = $url;
$this->format = $format;
@@ -360,8 +356,6 @@ class proxy{
throw new Exception("Invalid URL");
}
$this->clientcache();
$curl = curl_init();
// set headers
@@ -490,11 +484,14 @@ class proxy{
// get content type
if(isset($this->headers["content-type"])){
$filetype = explode("/", $this->headers["content-type"]);
$octet_check = stripos($this->headers["content-type"], "octet-stream");
if(strtolower($filetype[0]) != $this->format){
if(
stripos($this->headers["content-type"], $this->format) === false &&
$octet_check === false
){
throw new Exception("Resource is not an {$this->format} (Found {$filetype[0]} instead)");
throw new Exception("Resource reported invalid Content-Type");
}
}else{
@@ -502,6 +499,18 @@ class proxy{
throw new Exception("Resource is not an {$this->format} (no Content-Type)");
}
$filetype = explode("/", $this->headers["content-type"]);
if(!isset($filetype[1])){
throw new Exception("Malformed Content-Type header");
}
if($octet_check !== false){
$filetype[1] = "jpeg";
}
header("Content-Type: {$this->format}/{$filetype[1]}");
// give payload size
@@ -541,7 +550,7 @@ class proxy{
if(isset($filename[1])){
header("Content-Disposition: filename=" . $filename[1] . "." . $filetype);
header("Content-Disposition: filename=\"" . trim($filename[1], "\"'") . "." . $filetype . "\"");
return;
}
}
@@ -552,7 +561,7 @@ class proxy{
if($filename === null){
// everything failed! rename file to domain name
header("Content-Disposition: filename=" . parse_url($url, PHP_URL_HOST) . "." . $filetype);
header("Content-Disposition: filename=\"" . parse_url($url, PHP_URL_HOST) . "." . $filetype . "\"");
return;
}
@@ -569,7 +578,7 @@ class proxy{
$filename = implode(".", $filename);
header("Content-Disposition: inline; filename=" . $filename . "." . $filetype);
header("Content-Disposition: inline; filename=\"" . $filename . "." . $filetype . "\"");
return;
}

100
lib/frontend.php
View File

@@ -923,6 +923,7 @@ class frontend{
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google",
"qwant" => "Qwant",
"yep" => "Yep",
"crowdview" => "Crowdview",
"mwmbl" => "Mwmbl",
@@ -942,6 +943,7 @@ class frontend{
"yandex" => "Yandex",
"brave" => "Brave",
"google" => "Google",
"qwant" => "Qwant",
"yep" => "Yep",
//"pinterest" => "Pinterest",
"imgur" => "Imgur",
@@ -959,7 +961,8 @@ class frontend{
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"yandex" => "Yandex",
"google" => "Google"
"google" => "Google",
"qwant" => "Qwant"
]
];
break;
@@ -971,6 +974,7 @@ class frontend{
"ddg" => "DuckDuckGo",
"brave" => "Brave",
"google" => "Google",
"qwant" => "Qwant",
"yep" => "Yep",
"mojeek" => "Mojeek"
]
@@ -1010,98 +1014,8 @@ class frontend{
$scraper_out = $first;
}
switch($scraper_out){
case "ddg":
include "scraper/ddg.php";
$lib = new ddg();
break;
case "brave":
include "scraper/brave.php";
$lib = new brave();
break;
case "yt";
include "scraper/youtube.php";
$lib = new youtube();
break;
case "yandex":
include "scraper/yandex.php";
$lib = new yandex();
break;
case "google":
include "scraper/google.php";
$lib = new google();
break;
/*
case "fb":
include "scraper/facebook.php";
$lib = new facebook();
break;*/
case "crowdview":
include "scraper/crowdview.php";
$lib = new crowdview();
break;
case "mwmbl":
include "scraper/mwmbl.php";
$lib = new mwmbl();
break;
case "mojeek":
include "scraper/mojeek.php";
$lib = new mojeek();
break;
case "marginalia":
include "scraper/marginalia.php";
$lib = new marginalia();
break;
case "wiby":
include "scraper/wiby.php";
$lib = new wiby();
break;
case "curlie":
include "scraper/curlie.php";
$lib = new curlie();
break;
case "yep":
include "scraper/yep.php";
$lib = new yep();
break;
case "sc":
include "scraper/sc.php";
$lib = new sc();
break;
case "spotify":
include "scraper/spotify.php";
$lib = new spotify();
break;
case "pinterest":
include "scraper/pinterest.php";
$lib = new pinterest();
break;
case "imgur":
include "scraper/imgur.php";
$lib = new imgur();
break;
case "ftm":
include "scraper/ftm.php";
$lib = new ftm();
break;
}
include "scraper/$scraper_out.php";
$lib = new $scraper_out();
// set scraper on $_GET
$_GET["scraper"] = $scraper_out;