Compare commits

2 Commits
Author SHA1 Message Date
throwaway 0efbb81d23 remove parseInt on password env 2026-08-04 23:32:43 -07:00
throwaway 4b83c47637 read values from env and add Dockerfile 2026-08-04 23:03:47 -07:00
51 changed files with 3482 additions and 6240 deletions
+1
View File
@@ -26,3 +26,4 @@ data/captcha/minecraft/
!banner/*default*
scraper/curlie.html
icons/*
node_modules/
+1 -1
View File
@@ -4,7 +4,7 @@ FROM alpine:3.21
WORKDIR /var/www/html/4get
RUN apk update && apk upgrade
RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates ghostscript-fonts
RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates
COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
+13 -12
View File
@@ -37,20 +37,21 @@ tl;dr 4get is the best way to browse for shit.
| Brave | Yandex | Vimeo | Brave | Swisscows | DuckDuckGo |
| Yandex | Brave | Sepia Search | Google | | Yandex |
| Google | Google | DuckDuckGo | Yahoo! JAPAN | | Google |
| Google API | Google API | Brave | Startpage | | Qwant |
| Google CSE | Google CSE | Yandex | Baidu | | Marginalia |
| Yahoo! JAPAN | Yahoo! JAPAN | Google | | | YouTube |
| Startpage | Startpage | Yahoo! JAPAN | | | SoundCloud |
| Yep | Naver | Startpage | | | Startpage |
| Mwmbl | Baidu | Naver | | | Kagi |
| Mojeek | Solofield | Baidu | | | |
| Naver | Pinterest | Cc Cc | | | |
| Baidu | Flickr | Purili | | | |
| Cốc Cốc | Pexels | Solofield | | | |
| Google API | Google API | Brave | Startpage | | Startpage |
| Google CSE | Google CSE | Yandex | Qwant | | Kagi |
| Yahoo! JAPAN | Yahoo! JAPAN | Google | Mojeek | | Qwant |
| Startpage | Startpage | Yahoo! JAPAN | Baidu | | Ghostery |
| Qwant | Qwant | Startpage | | | Yep |
| Ghostery | Baidu | Qwant | | | Marginalia |
| Yep | Solofield | Baidu | | | YouTube |
| Mwmbl | Pinterest | Coc Coc | | | SoundCloud |
| Mojeek | Cara | Solofield | | | |
| Baidu | Flickr | | | | |
| Coc Coc | Pexels | | | | |
| Solofield | Pixabay | | | | |
| Marginalia | Unsplash | | | | |
| Purili | 500px | | | | |
| wiby | VSCO | | | | |
| wiby | 500px | | | | |
| | VSCO | | | | |
| | Imgur | | | | |
| | FindThatMeme | | | | |
+4 -16
View File
@@ -5,20 +5,8 @@ header("Access-Control-Allow-Origin: *");
include "data/config.php";
// get request count in last 24 hours
$bot_requests = 0;
$real_requests = 0;
$time = time();
for($i=0; $i<24; $i++){
$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
}
for($i=0; $i<24; $i++){
$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
}
$real_requests = apcu_fetch("real_requests");
$bot_requests = apcu_fetch("captcha_gen");
echo json_encode(
[
@@ -28,8 +16,8 @@ echo json_encode(
"name" => config::SERVER_NAME,
"description" => config::SERVER_LONG_DESCRIPTION,
"bot_protection" => config::BOT_PROTECTION,
"real_requests" => $real_requests,
"bot_requests" => $bot_requests,
"real_requests" => $real_requests === false ? 0 : $real_requests,
"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
"api_enabled" => config::API_ENABLED,
"alt_addresses" => config::ALT_ADDRESSES,
"version" => config::VERSION
+3 -1
View File
@@ -15,11 +15,13 @@ class autocomplete{
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
"yep" => "https://api.yep.com/ac/?query={searchTerms}",
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
"sc" => "",
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
];
/*
-3
View File
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "images", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->image($get),
-3
View File
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "music", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->music($get),
-3
View File
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "news", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->news($get),
-3
View File
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "videos", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->video($get),
-3
View File
@@ -38,9 +38,6 @@ if(
$get["extendedsearch"] = "no";
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo
-100
View File
@@ -1,100 +0,0 @@
<?php
header("Content-Type: image/png");
date_default_timezone_set('America/Toronto');
include "../../data/config.php";
function get_requests($slug){
$time = time();
$requests = [];
for($i=0; $i<73; $i++){
$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
$requests[] = [$i, ($data === false ? 0 : $data)];
}
// rtrim() out all entries that are 0
$out = [];
$found = false;
for($i=count($requests) - 1; $i>=0; $i--){
if($requests[$i][1] !== 0){
$found = $i;
break;
}
}
if($found === false){
return [];
}
$requests = array_slice($requests, 0, $found + 1);
return $requests;
}
$bot_requests = get_requests("bot_requests");
$real_requests = get_requests("real_requests");
$real_reqs_8h =
array_sum(
array_column(
array_slice(
$real_requests,
0,
8
),
1
)
);
$real_reqs_24h =
array_sum(
array_column(
array_slice(
$real_requests,
0,
24
),
1
)
);
$real_reqs_72h =
array_sum(
array_column(
$real_requests,
1
)
);
include "../../lib/graph_gen.php";
$graph = new graph();
$c = count($real_requests);
if($c > 25){
$c = 25;
}elseif($c <= 2){
$c = 2;
}
echo $graph->render([
"title" => "Number of requests within the last 72 hours (non-cummulative)",
"subtitle" => date("jS M y @ g:ia", time()) . " || 8h=" . number_format($real_reqs_8h) . " - 24h=" . number_format($real_reqs_24h) . " - 72h=" . number_format($real_reqs_72h) . " || " . config::SERVER_NAME,
"x_label" => "<= Hours passed =>",
"y_label" => "<= Request count/hour =>",
"grad_x" => $c,
"grad_y" => 20,
"data" => [
"Bot requests" => $bot_requests,
"Real requests" => $real_requests,
]
]);
+18 -17
View File
@@ -5,7 +5,7 @@ class config{
// any parameters.
// 4get version. Please keep this updated
const VERSION = 9;
const VERSION = 8;
// Will be shown pretty much everywhere.
const SERVER_NAME = "4get";
@@ -23,9 +23,6 @@ class config{
// Enable the API?
const API_ENABLED = true;
// Display errors. This should ONLY be enabled for debugging and constitutes a security risk.
const DISPLAY_ERRORS = false;
//
// 4play settings
//
@@ -109,31 +106,35 @@ class config{
// eachother your serber should appear everywhere.
const INSTANCES = [
"https://4get.ca",
"https://search.fischbytes.de",
"https://4get.nadeko.net",
"https://4get.zzls.xyz",
"https://4getus.zzls.xyz",
"https://4get.silly.computer",
"https://4get.konakona.moe",
"https://4get.lvkaszus.pl",
"https://4g.ggtyler.dev",
"https://4get.perennialte.ch",
"https://4get.sijh.net",
"https://4get.hbubli.cc",
"https://4get.plunked.party",
"https://4get.etenie.pl",
"https://4get.lunar.icu",
"https://4get.dcs0.hu",
"https://4get.kizuki.lol",
"https://4get.psily.garden",
"https://search.milivojevic.in.rs",
"https://4get.snine.nl",
"https://4get.datura.network",
"https://4get.neco.lol",
"https://4get.lol",
"https://4get.ch",
"https://4get.edmateo.site",
"https://4get.sudovanilla.org",
"https://search.mint.lgbt",
"https://4get.aishiteiru.moe",
"https://4.nboeck.de",
"https://search.yonderly.org",
"https://4get.lurx.net",
"https://4get.canine.tools",
"https://4get.sny.sh",
"https://4g.opnxng.com",
"https://s.307200.xyz",
"https://4get.privadency.com/"
"https://search.mint.lgbt"
];
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
// User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
+1 -1
View File
@@ -104,7 +104,7 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
</Directory>
# deny access to private resources
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
Require all denied
</DirectoryMatch>
-6
View File
@@ -152,8 +152,6 @@ This should expose a webservice located at `http://localhost:3000`. I highly rec
If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
## Setting up 4get
Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
@@ -165,7 +163,3 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
In case it isn't clear, here is a visual representation of what you want to achieve:
![Network map](https://i.imgur.com/jFjbBtG.png)
-29
View File
@@ -5,32 +5,3 @@ Start the PHP webserver like this, assuming you have curl-impersonate installed
```sh
LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
```
Make sure to add `.php` to paths, the embedded PHP webserver doesnt support `mod_rewrite`-like directives.
If, for some reason, you're in an environment where accessing the debug logs is impossible (like those shitty shared webhosts), you can set `DISPLAY_ERRORS` to `true` in `data/config.php`.
# Debug guide for docker
Install:
```sh
git clone https://git.lolcat.ca/lolcat/4get
# make changes to docker/apache/http/httpd.conf (comment out the lines that disable logging)
docker build -t luuul/4get .
```
Run:
```sh
docker run --name 4get -p 80:80 -p 443:443 luuul/4get
```
Get debug logs:
```sh
docker exec -it `docker ps -qf ancestor=luuul/4get:latest` sh # browse the container
docker exec -it `docker ps -qf ancestor=luuul/4get:latest` tail -f /var/log/apache2/error_log
```
Restart the container:
```sh
docker restart 4get
```
+4
View File
@@ -150,3 +150,7 @@ services:
##### Tor
You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
##### 4play
You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)
+48
View File
@@ -0,0 +1,48 @@
### Running 4get and 4play via docker guide
This setup will run both 4get and the 4play sidecar
# Docker compose
```
# docker-compose.yaml
services:
fourget:
image: luuul/4get:latest
restart: unless-stopped
environment:
- FOURGET_PROTO=http
- FOURGET_SERVER_NAME=4get.ca
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
- FOURGET_FPLAY_PASSWORD=cnc
- FOURGET_FPLAY_TIMEOUT=30000
- FOURGET_FPLAY_ENABLE_API=true
ports:
- "80:80"
fourplay:
image: luuul/4get-4play:latest
environment:
- FOURPLAY_PORT=3030
- FOURPLAY_PASSWORD=cnc
- FOURPLAY_COMMAND_TIMEOUT=5000
ports:
- "3000:3000"
- "3030:3030"
```
There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
# Steps
1. start the services with
```
docker compose up -d
```
2. configure firefox
folllow the steps in [configure](./configure.md) for firefox setup
+19
View File
@@ -0,0 +1,19 @@
FROM alpine:3.21
WORKDIR /app
RUN apk update && apk upgrade
RUN apk add nodejs npm
COPY package*.json ./
RUN npm ci --only=production && npm cache clean --force
COPY . .
EXPOSE 3030
ENTRYPOINT ["node"]
CMD ["page-render.js"]
+24
View File
@@ -0,0 +1,24 @@
# docker-compose.yaml
services:
fourget:
image: luuul/4get:latest
restart: unless-stopped
environment:
- FOURGET_PROTO=http
- FOURGET_SERVER_NAME=4get.ca
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
- FOURGET_FPLAY_PASSWORD=cnc
- FOURGET_FPLAY_TIMEOUT=30000
- FOURGET_FPLAY_ENABLE_API=true
ports:
- "80:80"
fourplay:
image: luuul/4get-4play:latest
environment:
- FOURPLAY_PORT=3030
- FOURPLAY_PASSWORD=cnc
- FOURPLAY_COMMAND_TIMEOUT=5000
ports:
- "3000:3000"
- "3030:3030"
+47
View File
@@ -0,0 +1,47 @@
{
"name": "4play",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "4play",
"version": "1.0.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@lawlers/4play": "^1.2.5",
"ws": "^8.21.2"
}
},
"node_modules/@lawlers/4play": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
"integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
"license": "AGPL-3.0-only",
"dependencies": {
"ws": "^8.21.1"
}
},
"node_modules/ws": {
"version": "8.21.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
"integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
}
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"name": "4play",
"version": "1.0.0",
"description": "",
"repository": {
"type": "git",
"url": "https://git.lolcat.ca/lolcat/4get/"
},
"license": "AGPL-3.0-only",
"author": "lolcat",
"type": "commonjs",
"main": "page-render.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@lawlers/4play": "^1.2.5",
"ws": "^8.21.2"
}
}
+3 -3
View File
@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
const http = require("http");
// 4play settings
var port = 3030;
var timeout = 30000;
var password = "cnc";
var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
// ==================================================
-151
View File
@@ -1,151 +0,0 @@
<?php
$scrapers = [
"web" => [
"DuckDuckGo",
"Brave",
"Yandex",
"Google",
"Google API",
"Google CSE",
"Yahoo! JAPAN",
"Startpage",
"Yep",
"Mwmbl",
"Mojeek",
"Naver",
"Baidu",
"Cốc Cốc",
"Solofield",
"Marginalia",
"Purili",
"wiby"
],
"images" => [
"DuckDuckGo",
"Yandex",
"Brave",
"Google",
"Google API",
"Google CSE",
"Yahoo! JAPAN",
"Startpage",
"Naver",
"Baidu",
"Solofield",
"Pinterest",
"Flickr",
"Pexels",
"Pixabay",
"Unsplash",
"500px",
"VSCO",
"Imgur",
"FindThatMeme"
],
"videos" => [
"YouTube",
"Vimeo",
"Sepia Search",
"DuckDuckGo",
"Brave",
"Yandex",
"Google",
"Yahoo! JAPAN",
"Startpage",
"Naver",
"Baidu",
"Cốc Cốc",
"Purili",
"Solofield"
],
"news" => [
"DuckDuckGo",
"Brave",
"Google",
"Yahoo! JAPAN",
"Startpage",
"Baidu"
],
"music" => [
"SoundCloud",
"Swisscows"
],
"autocomplete" => [
"Brave",
"DuckDuckGo",
"Yandex",
"Google",
"Qwant",
"Marginalia",
"YouTube",
"SoundCloud",
"Startpage",
"Kagi"
]
];
$lines = [
0 => "",
1 => ""
];
// get longest scraper string
// also get list of scrapers with most entries
$biggest_category_len = 0;
foreach($scrapers as $category_name => $category){
$count = count($category);
if($count > $biggest_category_len){
$biggest_category_len = $count;
}
}
foreach($scrapers as $category_name => $category){
// get longest string of the category
$longest_len = mb_strlen($category_name);
foreach($category as $scraper){
$strlen = mb_strlen($scraper);
if($strlen > $longest_len){
$longest_len = $strlen;
}
}
// add header
$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
for($i=2; $i<count($category) + 2; $i++){
if(!isset($lines[$i])){
$lines[$i] = "";
}
$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
}
if($i - 2 < $biggest_category_len){
for($k=$i; $k<$biggest_category_len + 2; $k++){
if(!isset($lines[$k])){
$lines[$k] = "";
}
$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
}
}
}
foreach($lines as $line){
echo $line . "|\n";
}
-3
View File
@@ -24,9 +24,6 @@ $payload = [
"nextpage" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->image($get);
+5 -8
View File
@@ -45,15 +45,15 @@ class anubis{
if(
!isset($script["challenge"]) ||
!is_string($script["challenge"]["randomData"]) ||
!isset($script["rules"]["difficulty"]) ||
!is_int($script["rules"]["difficulty"])
!is_int($script["rules"]["difficulty"]) ||
!is_string($script["challenge"])
){
throw new Exception("Found invalid challenge data");
}
return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
}
private function is_valid_hash($hash, $difficulty){
@@ -74,10 +74,9 @@ class anubis{
return true;
}
public function rape($id, $data, $difficulty = 5){
public function rape($data, $difficulty = 5){
$nonce = 0;
$microtime = microtime(true);
while(true){
@@ -88,12 +87,10 @@ class anubis{
$hash_hex = bin2hex($hash_binary);
return [
"id" => $id,
"response" => $hash_hex,
//"data" => $data,
//"difficulty" => $difficulty,
"nonce" => $nonce,
"elapsedTime" => round((microtime(true) - $microtime) * 1000)
"nonce" => $nonce
];
}
+7 -5
View File
@@ -7,6 +7,7 @@ class bot_protection{
// check if we want captcha
if(config::BOT_PROTECTION !== 1){
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@@ -31,7 +32,7 @@ class bot_protection{
// does key exist
apcu_exists($_COOKIE["pass"])
){
// exists, increment counter
$inc = apcu_inc($_COOKIE["pass"]);
@@ -45,6 +46,8 @@ class bot_protection{
}else{
// the cookie is OK! dont die() and give results
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@@ -161,7 +164,8 @@ class bot_protection{
$key = "k" . $inc . "." . $this->randomchars();
apcu_inc($key, 1, $stupid, 86400);
apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
apcu_inc("real_requests");
setcookie(
"pass",
@@ -186,7 +190,6 @@ class bot_protection{
}
}
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200);
$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
$payload = [
@@ -249,8 +252,7 @@ class bot_protection{
$frontend->loadheader(
$get,
$filters,
$page,
false
$page
);
echo $frontend->load("search.html", $payload);
+18 -510
View File
@@ -118,7 +118,7 @@ class frontend{
return trim($html);
}
public function loadheader(array $get, array $filters, string $page, bool $increment_bot_counter = true){
public function loadheader(array $get, array $filters, string $page){
echo
$this->load("header.html", [
@@ -177,15 +177,13 @@ class frontend{
){
// bot detected !!
if($increment_bot_counter){
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
}
apcu_inc("captcha_gen");
$this->drawerror(
"Tshh, blocked!",
'Your browser, IP or IP range has been blocked from this 4get instance. If this is an error, please <a href="/about">contact the administrator</a>.'
);
die();
}
}
@@ -223,7 +221,7 @@ class frontend{
'This scraper returned an error:' .
'<div class="code">' . htmlspecialchars($error) . '</div>' .
'Things you can try:' .
'<ul>' .
'<ul>' .
'<li>Use a different scraper</li>' .
'<li>Remove keywords that could cause errors</li>' .
'<li><a href="/instances?target=' . $target . "&" . $this->buildquery($get, false) . '">Try your search on another 4get instance</a></li>' .
@@ -535,490 +533,14 @@ class frontend{
'</button>' .
'<div class="favicon-dropdown">';
$archives = [];
$fansites = [
[
"url" => "https://cum.st/creators?cq=",
"favicon" => "cum.st",
"favicon_alt" => "cu",
"title" => "OnlyHaven"
],
[
"url" => "https://pawchive.pw/artists?q=",
"favicon" => "pawchive.pw",
"favicon_alt" => "pa",
"title" => "Pawchive"
],
[
"url" => "https://bakemono.app/creators?q=",
"favicon" => "bakemono.app",
"favicon_alt" => "ba",
"title" => "Bakemono"
],
[
"url" => "https://kemono.cr/artists?q=",
"favicon" => "kemono.cr",
"favicon_alt" => "ke",
"title" => "Kemono"
],
[
"url" => "https://coomer.st/artists?q=",
"favicon" => "coomer.st",
"favicon_alt" => "co",
"title" => "Coomer"
]
];
// add website-specific archives
switch($host["host"]){
case "www.youtube.com":
case "music.youtube.com":
case "youtube.com":
case "m.youtube.com":
case "youtu.be":
if(
(
$host["host"] == "youtu.be" &&
isset($host["path"]) &&
preg_match(
'/^\/([A-Za-z0-9_-]+)/',
$host["path"],
$slug
)
) ||
(
isset($host["query"]) &&
preg_match(
'/v=([A-Za-z0-9_-]+)/',
$host["query"],
$slug
)
)
){
// for watch?v=slug
$archives[] = [
"url" => "https://findyoutubevideo.thetechrobo.ca/?q=" . $slug[1],
"favicon" => "findyoutubevideo.thetechrobo.ca",
"favicon_alt" => "fi",
"title" => "FindYouTubeVideo"
];
$archives[] = [
"url" => "https://filmot.com/video/" . $slug[1] . "/",
"favicon" => "filmot.com",
"favicon_alt" => "fi",
"title" => "Filmot (metadata only)"
];
}elseif(
preg_match(
'/^\/(?:channel)\/@?([A-Za-z0-9_-]+)/',
$host["path"],
$username
)
){
$archives[] = [
"url" => "https://filmot.com/channel/" . $username[1] . "/",
"favicon" => "filmot.com",
"favicon_alt" => "fi",
"title" => "Filmot (metadata only)"
];
}
break;
case "m.twitch.tv":
case "twitch.tv":
case "player.twitch.tv":
case "www.twitch.tv":
if(
isset($host["path"]) &&
preg_match(
'/^\/([A-Za-z0-9_.]+)/',
$host["path"],
$username
) &&
$username[1] != "videos"
){
$archives[] = [
"url" => "https://vodvod.top/channels/@" . $username[1],
"favicon" => "vodvod.top",
"favicon_alt" => "vo",
"title" => "vodvod"
];
$archives[] = [
"url" => "https://twitchtracker.com/" . $username[1],
"favicon" => "twitchtracker.com",
"favicon_alt" => "tw",
"title" => "TwitchTracker"
];
}
break;
case "kick.com":
case "player.kick.com":
if(
isset($host["path"]) &&
preg_match(
'/^\/([A-Za-z0-9_.]+)/',
$host["path"],
$username
)
){
$archives[] = [
"url" => "https://lick.lolcat.ca/channel?name=" . $username[1],
"favicon" => "lick.lolcat.ca",
"favicon_alt" => "li",
"title" => "lick"
];
$archives[] = [
"url" => "https://kicktracker.net/" . $username[1],
"favicon" => "kicktracker.net",
"favicon_alt" => "ki",
"title" => "Kick tracker"
];
}
break;
case "x.com":
case "twitter.com":
if(
isset($host["path"]) &&
preg_match(
'/^\/([A-Za-z0-9_.]+)/',
$host["path"],
$username
)
){
$archives[] = [
"url" => "https://web.archive.org/web/*/https://x.com/" . $username[1] . "/status*",
"favicon" => "archive.org",
"favicon_alt" => "ar",
"title" => "Archive.org: Tweets &gt;2023"
];
$archives[] = [
"url" => "https://web.archive.org/web/*/https://twitter.com/" . $username[1] . "/status*",
"favicon" => "archive.org",
"favicon_alt" => "ar",
"title" => "Archive.org: Tweets &lt;2023"
];
}
break;
case "www.instagram.com":
case "instagram.com":
if(
isset($host["path"]) &&
preg_match(
'/^\/([A-Za-z0-9_.]+)/',
$host["path"],
$username
) &&
$username[1] != "p"
){
$archives[] = [
"url" => "https://instarchiver.net/users?q=" . $username[1],
"favicon" => "instarchiver.net",
"favicon_alt" => "in",
"title" => "Instarchiver"
];
$archives[] = [
"url" => "https://www.storiesdb.ch/" . $username[1],
"favicon" => "www.storiesdb.ch",
"favicon_alt" => "st",
"title" => "StoriesDB"
];
}
break;
case "reddit.com":
case "old.reddit.com":
case "www.reddit.com":
if(isset($host["path"])){
// direct thread lookup
// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=pullpush
// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=artic_shift
if(
preg_match(
'/^\/r\/[^\/]+\/comments\/[^?&]+/',
$host["path"],
$slug
)
){
$archives[] = [
"url" => "https://ihsoyct.github.io{$slug[0]}?backend=artic_shift",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "Artic Shift"
];
$archives[] = [
"url" => "https://ihsoyct.github.io{$slug[0]}?backend=pullpush",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "PullPush"
];
}
// subreddit thread search
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=submissions&sort=desc
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=submissions&sort=desc
// subreddit comment search
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=comments&sort=desc
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=comments&sort=desc
elseif(
preg_match(
'/^\/r\/([^\/]+)\/(?:(?:search|wiki|about)\/?)?(?:$|\?|&)/',
$host["path"],
$slug
)
){
$archives[] = [
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=submissions&sort=desc",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "Artic Shift (Search threads)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=submissions&sort=desc",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "PullPush (Search threads)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=comments&sort=desc",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "Artic Shift (Search comments)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=comments&sort=desc",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "PullPush (Search comments)"
];
}
// user thread lookup
// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=artic_shift
// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=pullpush
// user comment lookup
// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=artic_shift
// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=pullpush
elseif(
preg_match(
'/^\/(?:u|user)\/([^\/]+)\/(?:(?:comments|submitted)\/?)?(?:$|\?|&)/',
$host["path"],
$slug
)
){
$archives[] = [
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=artic_shift",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "Artic Shift (Search threads)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=pullpush",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "PullPush (Search threads)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=artic_shift",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "Artic Shift (Search comments)"
];
$archives[] = [
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=pullpush",
"favicon" => "reddit.com",
"favicon_alt" => "re",
"title" => "PullPush (Search comments)"
];
}
}
break;
// https://cum.st/creators?cq= onlyfans fansly patreon
// https://pawchive.pw/artists?q= patreon pixiv/fanbox discord
// https://bakemono.app/creators?q= pixiv/fanbox fansly onlyfans patreon
// https://kemono.cr/artists?q= patreon pixiv/fanbox fantia afdian boosty gumroad subscribestar dlsite
// https://coomer.st/artists?q= onlyfans fansly candfans
// cant look up
/*
case "fantia.jp": // https://fantia.jp/fanclubs/ <integer>
*/
// normal
case "onlyfans.com": // https://onlyfans.com/ <username>
case "fansly.com": // https://fansly.com/ <username>
case "fanbox.cc":
case "patreon.com": // https://www.patreon.com/ <username>
case "www.patreon.com":
case "boosty.to": // https://boosty.to/ <username>
case "www.subscribestar.com": // https://www.subscribestar.com/ <username>
case "subscribestar.com":
case "www.dlsite.com": // https://www.dlsite.com/ <username> /
case "dlsite.com":
case "candfans.com": // https://candfans.com/ <username>
case "afdian.com": // https://afdian.com/a/ <username>
if(
isset($host["path"]) &&
(
(
$host["host"] == "afdian.com" &&
preg_match(
'/^\/a\/([A-Za-z0-9-_.]+)/',
$host["path"],
$username
)
) ||
preg_match(
'/^\/([A-Za-z0-9-_.]+)/',
$host["path"],
$username
)
)
){
foreach($fansites as $fansite){
$archives[] = [
"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
"favicon" => $fansite["favicon"],
"favicon_alt" => $fansite["favicon_alt"],
"title" => $fansite["title"]
];
}
}
break;
}
// special fansite cases
if(
preg_match(
'/^([A-Za-z0-9_]+)\.(?:fanbox\.cc|gumroad\.com)/',
$host["host"],
$username
)
){
foreach($fansites as $fansite){
$archives[] = [
"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
"favicon" => $fansite["favicon"],
"favicon_alt" => $fansite["favicon_alt"],
"title" => $fansite["title"]
];
}
}
// detect git
if(
(
stripos(
$host["host"],
"git."
) !== false ||
$host["host"] == "codeberg.org" ||
$host["host"] == "sourceforge.net" ||
$host["host"] == "github.com"
) &&
isset($host["path"]) &&
preg_match(
'/^(\/[^\/]+\/[^\/]+\/?)/',
$host["path"],
$edge
)
){
$archives[] = [
"url" => "https://archive.softwareheritage.org/browse/origin/directory/?origin_url=" . urlencode($host["scheme"] . "://" . $host["host"] . $edge[1]),
"favicon" => "www.softwareheritage.org",
"favicon_alt" => "so",
"title" => "SoftwareHeritage"
];
}
$archives =
array_merge(
$archives,
[
[
"url" => "https://web.archive.org/web/" . $urlencode,
"favicon" => "archive.org",
"favicon_alt" => "ar",
"title" => "Archive.org"
],
[
"url" => "https://archive.ph/newest/" . htmlspecialchars($link),
"favicon" => "archive.ph",
"favicon_alt" => "ar",
"title" => "Archive.ph"
],
[
"url" => "https://yandex.com/search/?text=" . urlencode("url:" . $link),
"favicon" => "yandex.com",
"favicon_alt" => "ya",
"title" => "Yandex cache"
],
[
"url" => "https://ghostarchive.org/search?term=" . $urlencode,
"favicon" => "ghostarchive.org",
"favicon_alt" => "gh",
"title" => "Ghostarchive"
],
[
"url" => "https://arquivo.pt/wayback/" . htmlspecialchars($link),
"favicon" => "arquivo.pt",
"favicon_alt" => "ar",
"title" => "Arquivo.pt"
],
[
"url" => "https://megalodon.jp/?url=" . $urlencode,
"favicon" => "megalodon.jp",
"favicon_alt" => "me",
"title" => "Megalodon"
],
[
"url" => "https://www.webcitation.org/query?url=" . $urlencode,
"favicon" => "webcitation.org",
"favicon_alt" => "we",
"title" => "Webcitation"
]
]
);
foreach($archives as $archive){
$payload .= '<a href="' . $archive["url"] . '" class="list" target="_BLANK"><img src="/favicon?s=https://' . $archive["favicon"] . '" alt="' . $archive["favicon_alt"] . '">' . $archive["title"] . '</a>';
}
$payload .= '</div>';
$payload .=
'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
'</div>';
}
/*
@@ -1071,13 +593,6 @@ class frontend{
public function getscraperfilters($page){
// hack: enable error reporting if configured
if(config::DISPLAY_ERRORS === true){
ini_set('display_errors', 1);
ini_set('display_startup_errors', 1);
}
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
if(
@@ -1129,6 +644,7 @@ class frontend{
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"yep" => "Yep",
"mwmbl" => "Mwmbl",
"mojeek" => "Mojeek",
@@ -1155,10 +671,12 @@ class frontend{
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"naver" => "Naver",
"baidu" => "Baidu",
"solofield" => "Solofield",
"pinterest" => "Pinterest",
"cara" => "Cara",
"flickr" => "Flickr",
"pexels" => "Pexels",
"pixabay" => "Pixabay",
@@ -1177,7 +695,6 @@ class frontend{
"option" => [
"yt" => "YouTube",
//"archiveorg" => "Archive.org",
//"dailymotion" => "Dailymotion",
"vimeo" => "Vimeo",
//"odysee" => "Odysee",
"sepiasearch" => "Sepia Search",
@@ -1188,6 +705,7 @@ class frontend{
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"naver" => "Naver",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
@@ -1206,7 +724,8 @@ class frontend{
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
//"mojeek" => "Mojeek",
"qwant" => "Qwant",
"mojeek" => "Mojeek",
"baidu" => "Baidu"
]
];
@@ -1538,17 +1057,6 @@ class frontend{
return http_build_query($out);
}
private function sanitize_slug($username){
return urlencode(preg_replace('/[-_]/', " ", $username));
}
public function increment_real_reqs($scraper){
apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
}
public function htmlimage($image, $format){
if(
-388
View File
@@ -1,388 +0,0 @@
<?php
class graph{
public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
$graph = array_merge([
"title" => "Title",
"subtitle" => "description",
"grad_x" => 25,
"grad_y" => 20,
"x_label" => "<= x =>",
"y_label" => "<= y =>",
"antialias" => false,
"data" => []
], $graph);
$size = array_merge([
"image_width" => 750,
"image_height" => 500,
"x" => 90,
"y" => 50,
"width" => 650,
"height" => 300,
"square" => 11,
"column" => 6
], $size);
$colors = array_merge([
"bg" => new ImagickPixel("#1d2021"),
"lines_bg" => new ImagickPixel("#3c3836"),
"outline" => new ImagickPixel("#a89984"),
"text" => new ImagickPixel("#ebdbb2"),
"transparent" => new ImagickPixel("transparent")
], $colors);
$colors_line = array_merge([
new ImagickPixel("#83a598"),
new ImagickPixel("#fb4934"),
new ImagickPixel("#b8bb26"),
new ImagickPixel("#fabd2f"),
new ImagickPixel("#d3869b"),
new ImagickPixel("#8ec07c"),
new ImagickPixel("#fe8019"),
new ImagickPixel("#458588"),
new ImagickPixel("#cc241d"),
new ImagickPixel("#98971a"),
new ImagickPixel("#d79921"),
new ImagickPixel("#b16286"),
new ImagickPixel("#689d6a"),
new ImagickPixel("#d65d0e"),
new ImagickPixel("#a89984") // other
], $colors_line);
$not_enough_data = false;
// compute datapoint max ranges
$max_range_x = 0;
$max_range_y = 0;
if(count($graph["data"]) === 0){
$not_enough_data = true;
}else{
foreach($graph["data"] as $datapoint => $value){
if(count($value) < 2){
$not_enough_data = true;
}else{
$nv1 = [];
$nv2 = [];
foreach($value as $k => $v){
$nv1[] = $v[0];
$nv2[] = $v[1];
}
$max1 = max($nv1);
$max2 = max($nv2);
if($max_range_x < $max1){ $max_range_x = $max1; }
if($max_range_y < $max2){ $max_range_y = $max2; }
}
}
}
$im = new Imagick();
$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
$im->setImageFormat("png");
$draw = new ImagickDraw();
//
// Draw background grid
//
$draw->setStrokeColor($colors["lines_bg"]);
$draw->setStrokeAntialias($graph["antialias"]);
$draw->setStrokeWidth(1);
$draw->setFillColor($colors["transparent"]);
$text = new ImagickDraw();
$text->setFontSize(11);
$text->setTextAntialias($graph["antialias"]);
$text->setFillColor($colors["text"]);
// vertical lines
$offset = $size["width"] / ($graph["grad_x"] - 1);
for($i=0; $i<$graph["grad_x"]; $i++){
$x = round($size["x"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_x"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$x,
$size["y"],
$x,
$size["y"] + $size["height"],
);
if($not_enough_data === false){
// add numbas
$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
$im->annotateImage(
$text,
round($x - ($number_width / 2)),
$size["y"] + $size["height"] + 14,
0,
$number
);
}
}
// horizontal lines
$text->setTextAlignment(Imagick::ALIGN_RIGHT);
$offset = $size["height"] / ($graph["grad_y"] - 1);
$vertical_text_size = 0;
for($i=0; $i<$graph["grad_y"]; $i++){
$y = round($size["y"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_y"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$size["x"] + 1,
$y,
$size["x"] + $size["width"] - 1,
$y,
);
if($not_enough_data === false){
$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
$query = $im->queryFontMetrics($text, $val);
if($query["textWidth"] > $vertical_text_size){
$vertical_text_size = $query["textWidth"];
}
// add numbas
$im->annotateImage(
$text,
$size["x"] - 4,
$y + 4,
0,
$val
);
}
}
//
// Draw legend
//
$text->setTextAlignment(Imagick::ALIGN_LEFT);
$top_padding = 44;
$c = count($colors_line);
$padding = $size["width"] / $size["column"];
$line = new ImagickDraw();
//$line->setStrokeWidth(1);
$line->setStrokeAntialias($graph["antialias"]);
if($not_enough_data === false){
$i = 0;
foreach($graph["data"] as $datafield => $data){
$used_color = $colors_line[$i % $c];
$draw->setStrokeColor($used_color);
$row = floor($i / $size["column"]);
$x = round($size["x"] + (($i % $size["column"]) * $padding));
$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
// draw legend
$draw->rectangle(
$x,
$y,
$x + $size["square"],
$y + $size["square"]
);
$im->annotateImage(
$text,
$x + $size["square"] + 4,
$y + 10,
0,
$datafield
);
//
// Draw lines
//
$line->setStrokeColor($used_color);
$old_x = null;
$old_y = null;
$continue = true;
foreach($data as $field){
// field 0 = hour (sorted)
// field 1 = datapoint
$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
if($continue){
$continue = false;
$old_x = $x;
$old_y = $y;
continue;
}
$line->line($old_x, $old_y, $x, $y);
$old_x = $x;
$old_y = $y;
}
$i++;
}
}
//
// add background labels
//
$text->setFontSize(13);
$text->setFillColor($colors["bg"]);
$draw->setFillColor($colors["text"]);
$draw->setStrokeColor($colors["transparent"]);
// horizontal
$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
$draw->rectangle(
$x_x - 2,
$x_y - $label_width_x["textHeight"] + 4,
$x_x + $label_width_x["textWidth"] + 2,
$x_y + 3,
);
// vertical
$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
$draw->rectangle(
$y_x + 4,
$y_y - 2,
$y_x + $label_width_y["textHeight"] + 3,
$y_y + $label_width_y["textWidth"] + 2,
);
$im->drawImage($draw);
$im->drawImage($line);
// horizontal text
$im->annotateImage(
$text,
$x_x,
$x_y,
0,
$graph["x_label"]
);
// vertical text
$im->annotateImage(
$text,
$y_x + $label_width_y["textHeight"],
$y_y + $label_width_y["textWidth"],
270,
$graph["y_label"]
);
//
// draw title + subtext
//
$text->setFontSize(20);
$text->setFontWeight(900);
$text->setFillColor($colors["text"]);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 25,
0,
$graph["title"]
);
$text->setFontSize(13);
$text->setFontWeight(100);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 10,
0,
$graph["subtitle"]
);
if($not_enough_data){
// add not enough data notice
$rect = new ImagickDraw();
$rect->setFillColor($colors["bg"]);
$rect->setStrokeColor($colors_line[1]);
$rect_w = 200;
$rect_h = 24;
$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
$rect->rectangle(
$rect_x,
$rect_y,
$rect_x + $rect_w,
$rect_y + $rect_h
);
$im->drawImage($rect);
$label = "Not enough data";
$text->setFontWeight(900);
$query = $im->queryFontMetrics($text, $label);
$im->annotateImage(
$text,
round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
0,
$label
);
}
return $im->getImageBlob();
}
}
-3
View File
@@ -26,9 +26,6 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->music($get);
-3
View File
@@ -26,9 +26,6 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->news($get);
-107
View File
@@ -1,107 +0,0 @@
<?php
class google{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("google");
include "lib/fuckhtml.php";
$this->fuckhtml = new fuckhtml();
}
private function get(/*$proxy, */$url, $get = []){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
// http2 bypass
curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
$headers =
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"];
// i dont think proxies are necessary here...
//$this->backend->assign_proxy($curlproc, $proxy);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
function resolve($id){
$html =
$this->get(
//$this->backend->get_ip(),
"https://www.google.com/goto",
[
"url" => $id
]
);
$this->fuckhtml->load($html);
$a =
$this->fuckhtml
->getElementsByTagName(
"a"
);
if(count($a) === 0){
throw new Exception("Failed to find link");
}
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
if(
preg_match(
'/^(https?:|)\/\//',
$link
)
){
return $link;
}
throw new Exception("Got malformed link: " . $link);
}
}
+4 -9
View File
@@ -5,8 +5,7 @@ $resolver = new resolver();
class resolver{
public const resolvers = [
"sc",
"google"
"sc"
];
public function __construct(){
@@ -41,13 +40,9 @@ class resolver{
$resolver = new $scraper();
$link = $resolver->resolve($target);
if(
is_string($link) &&
!empty($link)
){
if(is_string($link)){
header("Location: {$link}");
echo 'If your ass is not getting redirected, <a href="' . htmlspecialchars($link) . '">click here</a>.';
}
}catch(Exception $error){
@@ -59,13 +54,13 @@ class resolver{
header("Content-Type: text/plain");
http_response_code(400);
echo htmlspecialchars($message);
echo $message;
}
public function do404($message){
header("Content-Type: text/plain");
http_response_code(404);
echo htmlspecialchars($message);
echo $message;
}
}
+25 -13
View File
@@ -1,16 +1,28 @@
# 44
# 4444444 44
# 44444444 44444 444
# 44444444 444444 444444444
# 44444 44444444 444444444
# 444444444 4444444
# 4444444444 444444
# 4444444444444
# 444444444444444444
# 444444444444444
# 44444444
# 4444
# 44
# When the robots.txt is sus
# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌
# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌
# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌
# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪
# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡
# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐
# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔
# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘
# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎
# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗
# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷
# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟
# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰
# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊
# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌
# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁
# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀
# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀
# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟
# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾
# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽
User-agent: *
Disallow:
Host: 4get.ca
Sitemap: https://4get.ca/sitemap
+2 -2
View File
@@ -919,8 +919,8 @@ class baidu{
$description =
$this->fuckhtml
->getElementsByAttributeValue(
"data-module",
"abstract",
"data-sanssr-cmpt",
"card/www-summary-1",
$div
);
-39
View File
@@ -383,8 +383,6 @@ class brave{
// load html
$this->fuckhtml->load($html);
$this->detect_captcha($html);
/*
Get next page "token"
*/
@@ -1177,8 +1175,6 @@ class brave{
// load html
$this->fuckhtml->load($html);
$this->detect_captcha($html);
// get npt
$out["npt"] =
$this->generatenextpagetoken(
@@ -1274,9 +1270,6 @@ class brave{
fclose($handle);*/
$this->fuckhtml->load($html);
$this->detect_captcha($html);
$json = $this->get_js();
foreach(
@@ -1381,8 +1374,6 @@ class brave{
$this->fuckhtml->load($html);
$this->detect_captcha($html);
$out = [
"status" => "ok",
"npt" => null,
@@ -1862,34 +1853,4 @@ class brave{
)
);
}
private function detect_captcha(){
$title =
$this->fuckhtml
->getElementsByTagName(
"title"
);
if(count($title) === 0){
if(strtolower($this->fuckhtml->getloadedhtml()) == "this service is not available in your region"){
throw new Exception("Brave rangebanned the IP range");
}
throw new Exception("Brave returned a malformed page");
}
$title =
strtolower(
$this->fuckhtml
->getTextContent($title[0])
);
if($title == "human verification"){
throw new Exception("Brave returned a Captcha");
}
}
}
+847
View File
@@ -0,0 +1,847 @@
<?php
class cara{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("cara");
}
public function getfilters($page){
return [
"sort" => [
"display" => "Sort by",
"option" => [
"Top" => "Top",
"MostRecent" => "Most Recent"
]
],
"type" => [
"display" => "Post type",
"option" => [
"any" => "Any type",
"portfolio" => "Portfolio", // {"posts":["portfolio"]}
"timeline" => "Timeline" // {"posts":["timeline"]}
]
],
"fields" => [
"display" => "Field/Medium",
"option" => [
"any" => "Any field",
"2D" => "2D Work",
"3D" => "3D Work",
"3DPrinting" => "3D Printing",
"Acrylic" => "Acrylic",
"AlcoholMarkers" => "Alcohol Markers",
"Animation" => "Animation",
"Chalk" => "Chalk",
"Charcoal" => "Charcoal",
"Colored pencil" => "Colored pencil",
"Conte" => "Conte",
"Crayon" => "Crayon",
"Digital" => "Digital",
"Gouache" => "Gouache",
"Ink" => "Ink",
"MixedMedia" => "Mixed-Media",
"Oil" => "Oil",
"Oil-based Markers" => "Oil-based Markers",
"Other" => "Other",
"Pastels" => "Pastels",
"Photography" => "Photography",
"Sculpture" => "Sculpture",
"Sketches" => "Sketches",
"Tattoos" => "Tattoos",
"Traditional" => "Traditional",
"VFX" => "VFX",
"Watercolor" => "Watercolor"
]
],
"category" => [
"display" => "Category",
"option" => [
"any" => "Any category",
"3DScanning" => "3D Scanning",
"Abstract" => "Abstract",
"Adoptable" => "Adoptable",
"Anatomy" => "Anatomy",
"Animals" => "Animals",
"Anime" => "Anime",
"App" => "App",
"ArchitecturalConcepts" => "Architectural Concepts",
"ArchitecturalVisualization" => "Architectural Visualization",
"AugmentedReality" => "Augmented Reality",
"Automotive" => "Automotive",
"BoardGameArt" => "Board Game Art",
"BookIllustration" => "Book Illustration",
"CardGameArt" => "Card Game Art",
"CeramicsPottery" => "Ceramics/Pottery",
"CharacterAnimation" => "Character Animation",
"CharacterDesign" => "Character Design",
"CharacterModeling" => "Character Modeling",
"ChildrensArt" => "Children's Illustration",
"Collectibles" => "Collectibles",
"ColoringPage" => "Coloring Page",
"ComicArt" => "Comic Art",
"ConceptArt" => "Concept Art",
"Cosplay" => "Cosplay",
"CostumeDesign" => "Costume Design",
"CoverArt" => "Cover Art",
"Creatures" => "Creatures",
"Diorama" => "Diorama",
"EditorialIllustration" => "Editorial Illustration",
"EmbroiderySewing" => "Embroidery/Sewing",
"EnvironmentalConceptArt" => "Environmental Concept Art",
"EnvironmentalConceptDesign" => "Environmental Concept Design",
"FanArt" => "Fan Art",
"Fantasy" => "Fantasy",
"Fashion" => "Fashion",
"FashionStyling" => "Fashion Styling",
"FiberArts" => "Fiber Arts",
"Furry" => "Furry",
"GameArt" => "Game Art",
"GameplayDesign" => "Gameplay Design",
"GamesEnvironmentArt" => "Games Environment Art",
"Gem" => "Gem",
"GraphicDesign" => "Graphic Design",
"Handicraft" => "Handicraft",
"HairStyling" => "Hair Styling",
"HardSurface" => "Hard Surface",
"Horror" => "Horror",
"Illustration" => "Illustration",
"IllustrationVisualization" => "Illustration Visualization",
"IndustrialDesign" => "Industrial Design",
"Jewelry" => "Jewelry",
"KnittingCrochet" => "Knitting/Crochet",
"Landscape" => "Landscape",
"LevelDesign" => "Level Design",
"Lighting" => "Lighting",
"Makeup" => "Makeup",
"Manga" => "Manga",
"MapsCartography" => "Maps/Cartography",
"MattePainting" => "Matte Painting",
"Materials" => "Materials",
"MechanicalDesign" => "Mechanical Design",
"Medical" => "Medical",
"Mecha" => "Mecha",
"MiniatureArt" => "Miniature Art",
"MotionGraphics" => "Motion Graphics",
"FrescoMurals" => "Fresco/Murals",
"Natural" => "Natural",
"Original Character" => "Original Character",
"Overlay" => "Overlay",
"PleinAir" => "Plein Air",
"Photogrammetry" => "Photogrammetry",
"PixelArt" => "Pixel Art",
"Portraits" => "Portraits",
"Props" => "Props",
"ProductDesign" => "Product Design",
"PublicDomain" => "Public Domain or Royalty Free",
"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
"Realism" => "Realism",
"ScienceFiction" => "Science Fiction",
"ScientificVisualization" => "Scientific Visualization",
"Scripts" => "Scripts",
"StillLife" => "Still Life",
"Storyboards" => "Storyboards",
"Stylized" => "Stylized",
"Surreal" => "Surreal",
"TechnicalArt" => "Technical Art",
"Textures" => "Textures",
"Tools" => "Tools",
"Toys" => "Toys",
"ToyPackaging" => "Toy Packaging",
"Tutorials" => "Tutorials",
"UIArt" => "User Interface (UI) Art",
"UrbanSketch" => "Urban Sketch",
"VFXforAnimation" => "VFX for Animation",
"VFXforFilm" => "VFX for Film",
"VFXforGames" => "VFX for Games",
"VFXforRealTime" => "VFX for Real-Time",
"VFXforTV" => "VFX for TV",
"Vehicles" => "Vehicles",
"VirtualReality" => "Virtual Reality",
"VisualDevelopment" => "Visual Development",
"VoxelArt" => "Voxel Art",
"Vtubers" => "Vtubers",
"WIP" => "WIP (Work in Progress)",
"Web" => "Web",
"Weapons" => "Weapons",
"Wildlife" => "Wildlife",
"Woodcutting" => "Woodcutting"
]
],
"software" => [
"display" => "Software",
"option" => [
"any" => "Any software",
"123D" => "123D",
"123DCatch" => "123D Catch",
"3DBee" => "3DBee",
"3DCoat" => "3DCoat",
"3DCoatPrint" => "3DCoatPrint",
"3DCoatTextura" => "3DCoatTextura",
"3DEqualizer" => "3DEqualizer",
"3DFZephyr" => "3DF Zephyr",
"3Delight" => "3Delight",
"3dpeople" => "3dpeople",
"3dsMax" => "3ds Max",
"3DSPaint" => "3DS Paint",
"ACDSeeCanvas" => "ACDSee Canvas",
"AbletonLive" => "Ableton Live",
"Acrobat" => "Acrobat",
"AdobeDraw" => "Adobe Draw",
"AdobeFlash" => "Adobe Flash",
"AdobeFresco" => "Adobe Fresco",
"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
"AdobeXD" => "Adobe XD",
"AffinityDesigner" => "Affinity Designer",
"AffinityPhoto" => "Affinity Photo",
"AfterEffects" => "After Effects",
"Akeytsu" => "Akeytsu",
"Alchemy" => "Alchemy",
"AliasDesign" => "Alias Design",
"AlightMotion" => "Alight Motion",
"Amadine" => "Amadine",
"Amberlight" => "Amberlight",
"Animate" => "Animate",
"AnimationMaster" => "Animation:Master",
"AnimeStudio" => "Anime Studio",
"Apophysis" => "Apophysis",
"ArchiCAD" => "ArchiCAD",
"Arion" => "Arion",
"ArionFX" => "ArionFX",
"Arnold" => "Arnold",
"ArtEngine" => "ArtEngine",
"ArtFlow" => "ArtFlow",
"ArtRage" => "ArtRage",
"ArtstudioPro" => "Artstudio Pro",
"Artweaver" => "Artweaver",
"Aseprite" => "Aseprite",
"Audition" => "Audition",
"AutoCAD" => "AutoCAD",
"AutodeskSketchBook" => "Autodesk SketchBook",
"AvidMediaComposer" => "Avid Media Composer",
"AzPainter" => "AzPainter",
"babylonjs" => "babylon.js",
"BalsamiqMockup" => "Balsamiq Mockup",
"Bforartists" => "Bforartists",
"BlackInk" => "Black Ink",
"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
"Blender" => "Blender",
"Blender DeepPaint" => "Blender DeepPaint",
"BlenderGreasePencil" => "Blender Grease Pencil",
"Blockbench" => "Blockbench",
"BodyPaint" => "BodyPaint",
"Boxcutter" => "Boxcutter",
"BraidMaker" => "Braid Maker",
"BrickLinkStudio" => "BrickLink Studio",
"Bridge" => "Bridge",
"Brushifyio" => "Brushify.io",
"C" => "C",
"C#" => "C#",
"C++" => "C++",
"CACANi" => "CACANi",
"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
"CLO" => "CLO",
"CRYENGINE" => "CRYENGINE",
"Callipeg" => "Callipeg",
"Canva" => "Canva",
"CaptureOne" => "Capture One",
"CartoonAnimator" => "Cartoon Animator",
"Carveco" => "Carveco",
"Cavalry" => "Cavalry",
"Chaotica" => "Chaotica",
"CharacterAnimator" => "Character Animator",
"CharacterCreator" => "Character Creator",
"Cinema4D" => "Cinema 4D",
"ClarisseiFX" => "Clarisse iFX",
"Coiffure" => "Coiffure",
"ColorsLive" => "Colors Live",
"Combustion" => "Combustion",
"Construct2" => "Construct 2",
"Core" => "Core",
"CorelPainter" => "Corel Painter",
"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
"CoronaRenderer" => "Corona Renderer",
"ProMotionNG" => "Cosmigo Pro Motion NG",
"CrazyBump" => "CrazyBump",
"Crocotile3D" => "Crocotile 3D",
"Curvy3D" => "Curvy 3D",
"Cycles4D" => "Cycles 4D",
"Darkroom" => "Darkroom",
"DAZStudio" => "DAZ Studio",
"DDO" => "DDO",
"DECIMA" => "DECIMA",
"Darktable" => "Darktable",
"DaVinciResolve" => "DaVinci Resolve",
"Dimension" => "Dimension",
"DragonBones" => "DragonBones",
"Dragonframe" => "Dragonframe",
"Drawpile" => "Drawpile",
"Dreams" => "Dreams",
"Dreamweaver" => "Dreamweaver",
"DxOPhotoLab" => "DxO PhotoLab",
"ECycles" => "E-Cycles",
"EmberGen" => "EmberGen",
"Encore" => "Encore",
"Expresii" => "Expresii",
"FStorm" => "FStorm",
"FadeIn" => "FadeIn",
"Feather3D" => "Feather 3D",
"FiberShop" => "FiberShop",
"Figma" => "Figma",
"FilmoraWondershare" => "Filmora Wondershare",
"FilterForge" => "Filter Forge",
"FinalCutPro" => "Final Cut Pro",
"FinalDraft" => "Final Draft",
"finalRender" => "finalRender",
"FireAlpaca" => "FireAlpaca",
"Fireworks" => "Fireworks",
"FlamePainter" => "Flame Painter",
"Flash" => "Flash",
"FlipaClip" => "FlipaClip",
"FlipnoteStudio" => "Flipnote Studio",
"Fluent" => "Fluent",
"ForestPack" => "Forest Pack",
"FormZ" => "Form-Z",
"Fractorium" => "Fractorium",
"FreeCAD" => "FreeCAD",
"FreeHand" => "FreeHand",
"Forger" => "Forger",
"FrostbiteEngine" => "Frostbite Engine",
"fSpy" => "fSpy",
"FumeFX" => "FumeFX",
"Fusion360" => "Fusion 360",
"GIMP" => "GIMP",
"GSCurveTools" => "GS CurveTools",
"GSToolbox" => "GS Toolbox",
"Gaea" => "Gaea",
"GameTextures" => "Game Textures",
"GameMakerStudio" => "GameMaker: Studio",
"GarageFarmNET" => "GarageFarm.NET",
"GeoGlyph" => "GeoGlyph",
"GigapixelAl" => "Gigapixel Al",
"Glaxnimate" => "Glaxnimate",
"GnomePaint" => "Gnome Paint",
"Godot" => "Godot",
"Goxel" => "Goxel",
"Graphite" => "Graphite",
"Graswald" => "Graswald",
"GravitySketch" => "Gravity Sketch",
"GuerillaRender" => "GuerillaRender",
"HDRLightStudio" => "HDR Light Studio",
"HairStrandDesigner" => "Hair Strand Designer",
"HairTGHairFur" => "HairTG - Hair &amp; Fur",
"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
"Handplane" => "Handplane",
"Hansoft" => "Hansoft",
"HardOps" => "Hard Ops",
"HardMesh" => "HardMesh",
"Harmony" => "Harmony",
"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
"HelloPaint" => "HelloPaint",
"HeliconFocus" => "Helicon Focus",
"Hexels" => "Hexels",
"HiPaint" => "HiPaint",
"Houdini" => "Houdini",
"HydraRenderer" => "Hydra Renderer",
"iArtbook" => "iArtbook",
"IbisPaint" => "ibisPaint",
"Ideas" => "Ideas",
"IllustStudio" => "Illust Studio",
"Illustrator" => "Illustrator",
"IllustratorDraw" => "Illustrator Draw",
"InDesign" => "InDesign",
"Inochi2D" => "Inochi2D",
"InVision" => "InVision",
"InVisionCraft" => "InVision Craft",
"InfinitePainter" => "Infinite Painter",
"Inkscape" => "Inkscape",
"Inspirit" => "Inspirit",
"InstaLOD" => "InstaLOD",
"InstaMAT" => "InstaMAT",
"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
"InstantMeshes" => "Instant Meshes",
"InstantTerra" => "Instant Terra",
"Inventor" => "Inventor",
"Iray" => "Iray",
"JWildfire" => "JWildfire",
"Java" => "Java",
"Jira" => "Jira",
"JumpPaint" => "Jump Paint by MediBang",
"JSPaint" => "JS Paint",
"Katana" => "Katana",
"Keyshot" => "Keyshot",
"KidPix" => "Kid Pix",
"KitBash3D" => "KitBash3D",
"Knald" => "Knald",
"Kodon" => "Kodon",
"KolourPaint" => "KolourPaint",
"Krakatoa" => "Krakatoa",
"KRESKA" => "KRESKA",
"Krita" => "Krita",
"LensStudio" => "Lens Studio",
"LibreSprite" => "LibreSprite",
"LightWave3D" => "LightWave 3D",
"Lightroom" => "Lightroom",
"Linearity" => "Linearity",
"LiquiGen" => "LiquiGen",
"Live2DCubism" => "Live2D Cubism",
"LookatmyHair" => "Look at my Hair",
"Lotpixel" => "Lotpixel",
"Lumion" => "Lumion",
"LuxRender" => "LuxRender",
"MacPaint" => "MacPaint",
"MagicaCSG" => "MagicaCSG",
"MagicaVoxel" => "MagicaVoxel",
"Magma" => "Magma",
"MakeHuman" => "MakeHuman",
"Malmal" => "Malmal",
"Mandelbulb3D" => "Mandelbulb 3D",
"Mandelbulber" => "Mandelbulber",
"MangaStudio" => "Manga Studio",
"Mari" => "Mari",
"MarmosetToolbag" => "Marmoset Toolbag",
"MarvelousDesigner" => "Marvelous Designer",
"MasterpieceStudioPro" => "Masterpiece Studio Pro",
"MasterpieceVR" => "MasterpieceVR",
"Maverick" => "Maverick",
"MaxwellRender" => "Maxwell Render",
"Maya" => "Maya",
"MediBangPaint" => "MediBang Paint",
"MediumbyAdobe" => "Medium by Adobe",
"Megascans" => "Megascans",
"mentalray" => "mental ray",
"MeshLab" => "MeshLab",
"Meshroom" => "Meshroom",
"MetaHumanCreator" => "MetaHuman Creator",
"Metashape" => "Metashape",
"MightyBake" => "MightyBake",
"MikuMikuDance" => "MikuMikuDance",
"Minecraft" => "Minecraft",
"Mischief" => "Mischief",
"Mixamo" => "Mixamo",
"Mixer" => "Mixer",
"MoI3D" => "MoI3D",
"Mocha" => "Mocha",
"Modo" => "Modo",
"Moho" => "Moho",
"MotionBuilder" => "MotionBuilder",
"Mudbox" => "Mudbox",
"Muse" => "Muse",
"MSPaint" => "MS Paint",
"MyPaint" => "MyPaint",
"NDO" => "NDO",
"NX" => "NX",
"NdotCAD" => "NdotCAD",
"NintendoNotes" => "Nintendo Notes",
"NomadSculpt" => "Nomad Sculpt",
"Notability" => "Notability",
"Nuke" => "Nuke",
"Nvil" => "Nvil",
"OctaneRender" => "Octane Render",
"Omniverse" => "Omniverse",
"OmniverseCreate" => "Omniverse Create",
"ON1PhotoRAW" => "ON1 Photo RAW",
"Open3DEngine" => "Open 3D Engine",
"OpenCanvas" => "OpenCanvas",
"OpenGL" => "OpenGL",
"OpenToonz" => "OpenToonz",
"Ornatrix" => "Ornatrix",
"OsciRender" => "Osci-Render",
"OurPaint" => "Our Paint",
"PBRMAX" => "PBRMAX",
"PFTrack" => "PFTrack",
"PTGui" => "PTGui",
"Paintbrush" => "Paintbrush",
"PaintNET" => "Paint.NET",
"PaintShopPro" => "PaintShop Pro",
"PaintToolSAI" => "Paint Tool SAI",
"PaintstormStudio" => "Paintstorm Studio",
"Paper" => "Paper",
"Pencil2D" => "Pencil2D",
"Penpot" => "Penpot",
"PhoenixFD" => "Phoenix FD",
"Phonto" => "Phonto",
"PhotoLab2" => "PhotoLab 2",
"Photopea" => "Photopea",
"Photoscan" => "Photoscan",
"Photoshop" => "Photoshop",
"PhotoshopElements" => "Photoshop Elements",
"PicoCAD" => "picoCAD",
"PicoCAD2" => "picoCAD 2",
"Pinta" => "Pinta",
"Piskel" => "Piskel",
"Pixilart" => "Pixilart",
"Pixelitor" => "Pixelitor",
"Pixelmator" => "Pixelmator",
"Pixelorama" => "Pixelorama",
"PixivSketch" => "pixiv Sketch",
"Pixquare" => "Pixquare",
"PlantCatalog" => "PlantCatalog",
"PlantFactory" => "PlantFactory",
"Plasticity" => "Plasticity",
"PNGtuberPlus" => "PNGtuber Plus",
"Poliigon" => "Poliigon",
"Polybrush" => "Polybrush",
"PopcornFx" => "PopcornFx",
"Poser" => "Poser",
"Premiere" => "Premiere",
"PremiereElements" => "Premiere Elements",
"PresagisCreator" => "Presagis Creator",
"ProTools" => "Pro Tools",
"Procreate" => "Procreate",
"ProcreateDreams" => "Procreate Dreams",
"Producer" => "Producer",
"PrometheanAI" => "Promethean AI",
"PureRef" => "PureRef",
"Python" => "Python",
"PyxelEdit" => "PyxelEdit",
"QuadRemesher" => "Quad Remesher",
"QuarkXPress" => "QuarkXPress",
"Qubicle" => "Qubicle",
"Quill" => "Quill",
"QuixelBridge" => "Quixel Bridge",
"QuixelMegascans" => "Quixel Megascans",
"QuixelMixer" => "Quixel Mixer",
"QuixelSuite" => "Quixel Suite",
"R3DSWrap" => "R3DS Wrap",
"R3DSZWRAP" => "R3DS ZWRAP",
"RDTextures" => "RD-Textures",
"RailClone" => "RailClone",
"RealFlow" => "RealFlow",
"RealisticPaintStudio" => "Realistic Paint Studio",
"RealityCapture" => "RealityCapture",
"RealityScan" => "RealityScan",
"RealtimeBoard" => "Realtime Board",
"Rebelle" => "Rebelle",
"Redshift" => "Redshift",
"RenderMan" => "RenderMan",
"RenderNetwork" => "Render Network",
"Revit" => "Revit",
"Rhino" => "Rhino",
"Rhinoceros" => "Rhinoceros",
"RizomUV" => "RizomUV",
"RoughAnimator" => "Rough Animator",
"SamsungNotes" => "Samsung Notes",
"SamsungPENUP" => "Samsung PENUP",
"ScansLibrary" => "ScansLibrary",
"Scrivener" => "Scrivener",
"Sculpt+" => "Sculpt+",
"Sculptris" => "Sculptris",
"ShaveandaHaircut" => "Shave and a Haircut",
"ShiVa3D" => "ShiVa3D",
"Shotgun" => "Shotgun",
"Silo" => "Silo",
"Silugen" => "Silugen",
"Sketch" => "Sketch",
"SketchApp" => "Sketch App",
"SketchBookPro" => "SketchBook Pro",
"SketchClub" => "SketchClub",
"SketchUp" => "SketchUp",
"Sketchable" => "Sketchable",
"Sketchfab" => "Sketchfab",
"Skyshop" => "Skyshop",
"Snapseed" => "Snapseed",
"Snowdrop" => "Snowdrop",
"Softimage" => "Softimage",
"SolidWorks" => "SolidWorks",
"SonySketch" => "Sony Sketch",
"Soundbooth" => "Soundbooth",
"Source2" => "Source 2",
"SourceControl" => "Source Control",
"SourceFilmmaker" => "Source Filmmaker",
"SpeedTree" => "SpeedTree",
"Speedgrade" => "Speedgrade",
"SpeedyPainter" => "SpeedyPainter",
"Spine2D" => "Spine 2D",
"Spriter" => "Spriter",
"Stingray" => "Stingray",
"Storyboarder" => "Storyboarder",
"StoryboardPro" => "Storyboard Pro",
"SublimeText" => "Sublime Text",
"Substance3DDesigner" => "Substance 3D Designer",
"Substance3DModeler" => "Substance 3D Modeler",
"Substance3DPainter" => "Substance 3D Painter",
"Substance3DSampler" => "Substance 3D Sampler",
"Substance3DStager" => "Substance 3D Stager",
"SubstanceB2M" => "Substance B2M",
"SweetHome3D" => "Sweet Home 3D",
"SynthEyes" => "SynthEyes",
"TTools" => "TTools",
"TVPaint" => "TVPaint",
"TVPaintAnimation" => "TVPaint Animation",
"TayasuiSketches" => "Tayasui Sketches",
"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
"TayasuiSketchesPro" => "Tayasui Sketches Pro",
"Terragen" => "Terragen",
"Texturescom" => "Textures.com",
"Texturingxyz" => "Texturingxyz",
"TeyaConceptor" => "Teya Conceptor",
"TheGrove3D" => "The Grove 3D",
"TheaRender" => "Thea Render",
"Threejs" => "Three.js",
"Tiled" => "Tiled",
"TiltBrush" => "Tilt Brush",
"Tooll3" => "Tooll3",
"ToonBoomHarmony" => "Toon Boom Harmony",
"ToonBoomStudio" => "Toon Boom Studio",
"ToonSquid" => "ToonSquid",
"TopoGun" => "TopoGun",
"TuxPaint" => "Tux Paint",
"Tvori" => "Tvori",
"Twinmotion" => "Twinmotion",
"UNIGINEEngine" => "UNIGINE Engine",
"UVLayout" => "UVLayout",
"UltraFractal" => "Ultra Fractal",
"uMake" => "uMake",
"Unfold3D" => "Unfold 3D",
"Unity" => "Unity",
"UnrealEngine" => "Unreal Engine",
"Vengi" => "vengi",
"VRay" => "V-Ray",
"VRED" => "VRED",
"VTubeStudio" => "VTube Studio",
"Vectary" => "Vectary",
"VectorayGen" => "VectorayGen",
"Vectorworks" => "Vectorworks",
"VegasPro" => "Vegas Pro",
"VisualDesigner3D" => "Visual Designer 3D",
"VisualStudio" => "Visual Studio",
"VRoidStudio" => "VRoid Studio",
"Vue" => "Vue",
"Vuforia" => "Vuforia",
"WebGL" => "WebGL",
"WhiteboardFox" => "Whiteboard Fox",
"WickEditor" => "Wick Editor",
"Wings3D" => "Wings 3D",
"Word" => "Word",
"WorldCreator" => "World Creator",
"WorldMachine" => "World Machine",
"XParticles" => "X-Particles",
"Xfrog" => "Xfrog",
"Xgen" => "Xgen",
"xNormal" => "xNormal",
"xTex" => "xTex",
"XoliulShader" => "Xoliul Shader",
"Yafaray" => "Yafaray",
"Yeti" => "Yeti",
"ZBrush" => "ZBrush",
"ZBrushCore" => "ZBrushCore",
"ZenBrush" => "Zen Brush"
]
]
];
}
private function get($proxy, $url, $get = [], $search){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/plain, */*",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
"Referer: https://cara.app/search?q=" . urlencode($search),
//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"TE: trailers"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function image($get){
if($get["npt"]){
[$npt, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$npt = json_decode($npt, true);
}else{
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$npt = [
"q" => $get["s"],
"sortBy" => $get["sort"],
"take" => 24,
"skip" => 0,
"filters" => []
];
// parse filters
if($get["type"] != "any"){
$npt["filters"]["posts"] = [$get["type"]];
}
if($get["fields"] != "any"){
$npt["filters"]["fields"] = [$get["fields"]];
}
if($get["category"] != "any"){
$npt["filters"]["categories"] = [$get["category"]];
}
if($get["software"] != "any"){
$npt["filters"]["softwares"] = [$get["software"]];
}
if($npt["filters"] == []){
$npt["filters"] = "{}";
}else{
$npt["filters"] = json_encode($npt["filters"]);
}
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
try{
$json =
$this->get(
$proxy,
"https://cara.app/api/search/posts",
$npt,
$npt["q"]
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$imagecount = 0;
foreach($json as $image){
if(count($image["images"]) === 0){
// sometimes the api returns no images for an object
$imagecount++;
continue;
}
$cover = null;
$sources = [];
foreach($image["images"] as $source){
if($source["isCoverImg"]){
$cover = [
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
"width" => 500,
"height" => 500
];
}else{
$sources[] = [
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
"width" => null,
"height" => null
];
}
}
if($cover !== null){
$sources[] = $cover;
}
$out["image"][] = [
"title" => str_replace("\n", " ", $image["content"]),
"source" => $sources,
"url" => "https://cara.app/post/" . $image["id"]
];
$imagecount++;
}
if($imagecount === 24){
$npt["skip"] += 24;
$out["npt"] =
$this->backend->store(
json_encode($npt),
"images",
$proxy
);
}
return $out;
}
private function fix_url($url){
return
str_replace(
[" "],
["%20"],
$url
);
}
}
+50 -115
View File
File diff suppressed because one or more lines are too long
+35 -422
View File
@@ -9,9 +9,6 @@ class google{
include "lib/backend.php";
$this->backend = new backend("google");
$this->proc = curl_multi_init();
$this->handles = [];
}
public function getfilters($page){
@@ -526,7 +523,7 @@ class google{
config::FPLAY_EXTERNAL_ENDPOINT .
"?url=" . urlencode($url) .
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
"&container=" .// urlencode($container) .
"&container=" . urlencode($container) .
"&proxy=" . urlencode($proxy) .
"&url_match=" . urlencode("&sei=") .
"&fail_url_match=" . urlencode("\/sorry\/");
@@ -617,145 +614,6 @@ class google{
];
}
private function redirect_add_url($url){
if(
preg_match(
'/^\/goto\?url=([^&]+)/',
$url,
$slug
) === 0
){
// not a google redirect
return $url;
}
$curlproc = curl_init();
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1",
"Priority: u=0, i"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
//$this->backend->assign_proxy($curlproc, $proxy);
curl_multi_add_handle($this->proc, $curlproc);
$this->handles[] = [
"replacement" => $url,
"handle" => $curlproc
];
return $url;
}
private function resolve_urls(&$scraped_data){
do{
do{
$status = curl_multi_exec($this->proc, $active);
}while($status === CURLM_CALL_MULTI_PERFORM);
if($active){
curl_multi_select($this->proc);
}
}while($active && $status === CURLM_OK);
//
// if we reach this, we're done downloading garbage
//
// parse redirect values & create data shit
$keys = [];
$replacements = [];
for($i=0; $i<count($this->handles); $i++){
$this->fuckhtml->load(
curl_multi_getcontent(
$this->handles[$i]["handle"]
)
);
$a =
$this->fuckhtml
->getElementsByTagName(
"a"
);
if(count($a) === 0){
throw new Exception("Failed to find link");
}
$link =
$this->fuckhtml
->getTextContent(
$a[0]["attributes"]["href"]
);
if(
preg_match(
'/^(https?:|)\/\//',
$link
)
){
$keys[] = $this->handles[$i]["replacement"];
$replacements[] = $link;
}
}
$this->recursive_replace($keys, $replacements, $scraped_data);
curl_multi_close($this->proc);
}
private function recursive_replace(&$keys, &$replacements, &$pointer){
foreach($pointer as $key => &$value){
if($key == "url"){
$search =
array_search(
$value,
$keys
);
if($search !== false){
// replace url
$value = $replacements[$search];
}
}elseif(is_array($value)){
$this->recursive_replace($keys, $replacements, $value);
}
}
}
public function web($get){
@@ -875,8 +733,7 @@ class google{
$container = &$data["container"];
//file_put_contents("scraper/google-jp.html", $html);
//$html = file_get_contents("scraper/google.html");
//$container = "";
//$html = file_get_contents("scraper/google-jp.html");
}
$out = [
@@ -897,27 +754,9 @@ class google{
$this->fuckhtml->load($html);
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
}
// init
$this->detect_sorry();
$this->parsestyles();
$this->scrape_dimg($html);
$this->scrape_imagearr($html);
@@ -1048,8 +887,6 @@ class google{
$this->fuckhtml
->getElementsByTagName("a");
$illegal = false;
foreach($as as $a){
if(!isset($a["attributes"]["href"])){ continue; }
@@ -1060,35 +897,14 @@ class google{
$a["attributes"]["href"]
);
$link_text =
strtolower(
$this->fuckhtml
->getTextContent($a)
);
if(
str_contains(
$link_text,
"complaint"
) ||
str_contains(
$link_text,
"request"
preg_match(
'/^https?:\/\/lumendatabase\.org/',
$link
)
){
if(
str_contains(
$link_text,
"read more about the request"
)
){
$illegal = true;
}
// sometimes the link is an encrypted redirect, handle redirection securely
$notices[] = $this->redirect_add_url($link);
$notices[] = $link;
}
}
@@ -1099,24 +915,13 @@ class google{
$i = 0;
$c = count($notices);
if($illegal === false){
$notices_dom = [
[
"type" => "text",
"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
]
];
}else{
$notices_dom = [
[
"type" => "text",
"value" => "Google removed results from this page because they were illegal. You can view the takedown requests here: "
]
];
}
$notices_dom = [
[
"type" => "text",
"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
]
];
foreach($notices as $n){
@@ -1426,39 +1231,10 @@ class google{
$sublinks = [];
// get sublinks (rare)
$as =
$this->fuckhtml
->getElementsByClassName(
$this->getstyle([
"padding-right" => "6px",
"margin-top" => "4px",
"display" => "inline-block"
]),
"a"
);
foreach($as as $sublink){
$sublink_url =
$this->fuckhtml
->getTextContent(
$sublink["attributes"]["href"]
);
$sublinks[] = [
"title" =>
$this->fuckhtml
->getTextContent($sublink),
"date" => null,
"url" => $this->redirect_add_url($sublink_url)
];
}
$out["web"][] = [
"title" => $title,
"description" => $description,
"url" => $this->redirect_add_url($url),
"url" => $url,
"date" => $date,
"type" => "web",
"thumb" => $thumb,
@@ -1467,8 +1243,6 @@ class google{
];
}
$this->resolve_urls($out);
return $out;
}
@@ -1607,28 +1381,7 @@ class google{
$this->fuckhtml->load($html);
// parse all <style> tags
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
$this->detect_sorry();
// get javascript images
$this->scrape_imagearr($html);
@@ -1679,11 +1432,9 @@ class google{
),
"source" => $source,
"url" =>
$this->redirect_add_url(
$this->fuckhtml
->getTextContent(
$div["attributes"]["data-lpage"]
)
$this->fuckhtml
->getTextContent(
$div["attributes"]["data-lpage"]
)
];
}
@@ -1708,8 +1459,6 @@ class google{
);
}
$this->resolve_urls($out);
return $out;
}
@@ -1794,7 +1543,8 @@ class google{
implode(",", $tbs);
}
}
//$html = file_get_contents("scraper/google.html");
$data = $this->get(
$proxy,
"https://www.google.com/search",
@@ -1805,8 +1555,6 @@ class google{
$html = &$data["data"];
$container = &$data["container"];
//$html = file_get_contents("scraper/google.html");
if(!isset($params["start"])){
$params["start"] = 0;
@@ -1815,32 +1563,13 @@ class google{
$this->fuckhtml->load($html);
// parse all <style> tags
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
//
// Parse web video page
//
$this->detect_sorry();
// parse all <style> tags
$this->parsestyles();
// get javascript images
$this->scrape_dimg($html);
@@ -1849,29 +1578,6 @@ class google{
$params["c"] = $container;
// get encoded links
$encoded_links =
explode(
"function(){var m={",
$html,
2
);
if(count($encoded_links) === 2){
$encoded_links =
json_decode(
$this->fuckhtml
->extract_json(
"{" . $encoded_links[1]
),
true
);
}else{
$encoded_links = [];
}
$out = [
"status" => "ok",
"npt" =>
@@ -2050,50 +1756,6 @@ class google{
}
}
$url =
$this->fuckhtml
->getTextContent(
$url[0]["attributes"]["href"]
);
if(
preg_match(
'/^\/goto\?url/',
$url
)
){
// encrypted url detected
// get reference div
$refdivs =
$this->fuckhtml
->getElementsByAttributeName(
"id",
"div"
);
foreach($refdivs as $r){
if(
preg_match(
'/^atritem-/',
$r["attributes"]["id"]
) &&
isset($r["attributes"]["jsdata"])
){
$ref = explode(";", $r["attributes"]["jsdata"]);
$ref = $ref[count($ref) - 1];
if(isset($encoded_links[$ref][32][11][7])){
$url = $encoded_links[$ref][32][11][7];
}
break;
}
}
}
$out["video"][] = [
"title" =>
$this->titledots(
@@ -2112,12 +1774,14 @@ class google{
"duration" => $duration,
"views" => null,
"thumb" => $thumb,
"url" => $this->redirect_add_url($url)
"url" =>
$this->fuckhtml
->getTextContent(
$url[0]["attributes"]["href"]
)
];
}
$this->resolve_urls($out);
return $out;
}
@@ -2223,32 +1887,14 @@ class google{
$this->fuckhtml->load($html);
// parse styles
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
$this->detect_sorry();
// get images
$this->scrape_dimg($html);
// parse styles
$this->parsestyles();
$center_col =
$this->fuckhtml
->getElementById(
@@ -2482,8 +2128,6 @@ class google{
];
}
$this->resolve_urls($out);
return $out;
}
@@ -3009,7 +2653,6 @@ class google{
private function detect_sorry(){
// detect recaptcha
$captcha_form =
$this->fuckhtml
->getElementById(
@@ -3039,35 +2682,5 @@ class google{
throw new Exception("Google returned a JS challenge");
}
// bypass soft captcha
// "Verifying your request
// You'll be able to continue in a few seconds. Don't refresh this page."
$scripts =
$this->fuckhtml
->getElementsByTagName(
"script"
);
foreach($scripts as $sc){
if(
preg_match(
'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
$sc["innerHTML"],
$matches
)
){
sleep(10); // gotta sleep to not trigger captcha. blame google, not me
return [
"sei" => $matches[1],
"timeout" => (int)$matches[2] // set to 10000 usually
];
}
}
return false; // no need to bypass
}
}
+49 -191
View File
@@ -16,7 +16,18 @@ class mojeek{
case "web":
return [
"lang" => [ // lb
"focus" => [
"display" => "Focus",
"option" => [
"any" => "No focus",
"blogs" => "Blogs",
"Dictionary" => "Dictionary",
"Recipes" => "Recipes",
"Time" => "Time",
"Weather" => "Weather"
]
],
"lang" => [
"display" => "Language",
"option" => [
"any" => "Any language",
@@ -75,7 +86,7 @@ class mojeek{
"zu" => "Zulu"
]
],
"country" => [ // rb
"country" => [
"display" => "Country",
"option" => [
"any" => "No location bias",
@@ -329,21 +340,28 @@ class mojeek{
"zm" => "Zambia",
"zw" => "Zimbabwe"
]
],
"sort" => [
"display" => "Sort by",
],/*
"region" => [
"display" => "Region",
"option" => [
"relevance" => "Relevance",
"date" => "Date" // datewr=100
"any" => "Any region",
"eu" => "European Union",
"de" => "Germany",
"fr" => "France",
"uk" => "United Kingdom"
]
],*/
"domain" => [
"display" => "Results per domain",
"option" => [
"1" => "1 result",
"2" => "2 results",
"3" => "3 results",
"4" => "4 results",
"5" => "5 results",
"10" => "10 results",
"0" => "Unlimited",
]
],
"newer" => [
"display" => "Newer than",
"option" => "_DATE"
],
"older" => [
"display" => "Older than",
"option" => "_DATE"
]
];
break;
@@ -353,29 +371,21 @@ class mojeek{
}
}
private function get($proxy, $url, $get = [], $is_api = false){
private function get($proxy, $url, $get = []){
if($is_api){
$headers = [
"Accept: application/json"
];
}else{
$headers = [
"User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1"
];
}
$headers = [
"User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1"
];
$curlproc = curl_init();
@@ -410,154 +420,6 @@ class mojeek{
public function web($get){
return $this->web_api($get);
}
public function web_api($get){
$key = $this->backend->get_key();
$proxy = $this->backend->get_ip($key["increment"]);
if($get["npt"]){
[$params, $proxy] = $this->backend->get($get["npt"], "web");
$params = json_decode($params, true);
$params["api_key"] = $key["key"];
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
$params = [
"api_key" => $key["key"],
"q" => $get["s"],
"date" => 1,
"tlen" => 127, // title length
"dlen" => 511, // description length
"fmt" => "json"
];
if($get["lang"] != "any"){
$params["lb"] = $get["lang"];
}
if($get["country"] != "any"){
$params["rb"] = $get["country"];
}
if($get["sort"] == "date"){
$params["datewr"] = 100;
}
if($get["newer"] !== false){
$params["since"] = date("Ymd", $get["newer"]);
}
if($get["older"] !== false){
$params["before"] = date("Ymd", $get["before"]);
}
}
try{
$json =
$this->get(
$proxy,
"https://api.mojeek.com/search",
$params
);
}catch(Exception $error){
throw new Exception("Failed to get JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$out = [
"status" => "ok",
"spelling" => [
"type" => "no_correction",
"using" => null,
"correction" => null
],
"npt" => null,
"answer" => [],
"web" => [],
"image" => [],
"video" => [],
"news" => [],
"related" => []
];
if(!isset($json["response"]["status"])){
throw new Exception("API did not return a status field");
}
if($json["response"]["status"] != "OK"){
throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
}
if(!isset($json["response"]["results"])){
throw new Exception("Mojeek API did not return a results object");
}
foreach($json["response"]["results"] as $entry){
$out["web"][] = [
"title" => html_entity_decode($entry["title"]),
"description" => html_entity_decode($entry["desc"]),
"url" => $entry["url"],
"date" => $entry["timestamp"],
"type" => "web",
"thumb" => [
"url" => null,
"ratio" => null
],
"sublink" => [],
"table" => []
];
}
// get npt
if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
unset($params["api_key"]);
if(!isset($params["s"])){
$params["s"] = 11;
}else{
$params["s"] = $params["s"] + 10;
}
$out["npt"] =
$this->backend->store(
json_encode($params),
"web",
$proxy
);
}
return $out;
}
public function web_browser($get){
if($get["npt"]){
[$token, $proxy] = $this->backend->get($get["npt"], "web");
@@ -595,6 +457,7 @@ class mojeek{
"tn" => 7, // number of news results/page
"date" => 1, // show date
"tlen" => 128, // max length of title
//"dlen" => 511, // max length of description
//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
];
@@ -1137,12 +1000,7 @@ class mojeek{
return $out;
}
public function news(){
throw new Exception("Currently unsupported");
}
public function news_browser($get){
public function news($get){
$search = $get["s"];
+1102
View File
File diff suppressed because it is too large Load Diff
+153 -396
View File
@@ -9,10 +9,6 @@ class startpage{
include "lib/fuckhtml.php";
$this->fuckhtml = new fuckhtml();
include "lib/anubis.php";
$this->anubis = new anubis();
$this->anubis_error = "all gucci";
}
public function getfilters($page){
@@ -248,7 +244,7 @@ class startpage{
}
}
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
$curlproc = curl_init();
@@ -263,27 +259,6 @@ class startpage{
$url .= "?" . $get;
}
if($return_cookies){
$cookies = [];
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
$length = strlen($header);
$header = explode(":", $header, 2);
if(trim(strtolower($header[0])) == "set-cookie"){
$cookie_tmp = explode("=", trim($header[1]), 2);
$cookies[trim($cookie_tmp[0])] =
explode(";", $cookie_tmp[1], 2)[0];
}
return $length;
});
}
curl_setopt($curlproc, CURLOPT_URL, $url);
// http2 bypass
@@ -291,15 +266,6 @@ class startpage{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
foreach($passed_cookies as $name => $val){
$header_cookies[] = "$name=$val";
}
$header_cookies = rtrim(implode("; ", $header_cookies));
if($is_xhr === true){
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -313,7 +279,7 @@ class startpage{
"Origin: https://www.startpage.com/",
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
@@ -332,7 +298,7 @@ class startpage{
"Content-Length: " . strlen($get),
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
@@ -350,7 +316,7 @@ class startpage{
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
@@ -376,15 +342,6 @@ class startpage{
}
curl_close($curlproc);
if($return_cookies){
return [
"cookies" => $cookies,
"data" => $data
];
}
return $data;
}
@@ -392,22 +349,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "web");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -449,11 +398,7 @@ class startpage{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
$params
);
}catch(Exception $error){
@@ -461,41 +406,12 @@ class startpage{
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$cookies = [];
$html = $html["data"];
}else{
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
}
}
}
// detect captcha even if we passed anubis challenge
$this->detect_captcha($html);
//echo htmlspecialchars($html);
$this->detect_captcha($html);
if(
preg_match(
'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
@@ -504,14 +420,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to grep JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON");
}
//print_r($json);
@@ -533,7 +449,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "web", $proxy);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -742,22 +658,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "images");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -772,102 +680,76 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "images",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["size"] != "any"){
if(
$get["size"] == "Small" ||
$get["size"] == "Medium" ||
$get["size"] == "Large" ||
$get["size"] == "Wallpaper"
){
$params["flimgsize"] = $get["size"];
}else{
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
}
}
if($get["color"] != "any"){
if($get["color"] == "color"){
$params["flimgcolor"] = "ic:color";
}elseif($get["color"] == "bnw"){
$params["flimgcolor"] = "ic:gray";
}else{
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
}
}
if($get["type"] != "any"){
$params["flimgtype"] = $get["type"];
}
if($get["license"] != "any"){
$params["flimglicense"] = $get["license"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "images",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["size"] != "any"){
if(
$get["size"] == "Small" ||
$get["size"] == "Medium" ||
$get["size"] == "Large" ||
$get["size"] == "Wallpaper"
){
$params["flimgsize"] = $get["size"];
}else{
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
}
}
if($get["color"] != "any"){
if($get["color"] == "color"){
$params["flimgcolor"] = "ic:color";
}elseif($get["color"] == "bnw"){
$params["flimgcolor"] = "ic:gray";
}else{
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
}
}
if($get["type"] != "any"){
$params["flimgtype"] = $get["type"];
}
if($get["license"] != "any"){
$params["flimglicense"] = $get["license"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -887,18 +769,18 @@ class startpage{
) === 0
){
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to grep JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
// get npt
$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "images", $proxy);
// get images
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -937,22 +819,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "videos");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -967,72 +841,46 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "video",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["sort"] != "relevance"){
$params["sort_by"] = $get["sort"];
}
if($get["duration"] != "any"){
$params["with_duration"] = $get["duration"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "video",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["sort"] != "relevance"){
$params["sort_by"] = $get["sort"];
}
if($get["duration"] != "any"){
$params["with_duration"] = $get["duration"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -1046,14 +894,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to get JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
$out = [
@@ -1067,7 +915,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "video", $proxy);
// get results
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -1123,22 +971,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "news");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -1153,67 +993,41 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "news",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["time"] != "any"){
$params["with_date"] = $get["time"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "news",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["time"] != "any"){
$params["with_date"] = $get["time"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -1227,14 +1041,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to get JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
$out = [
@@ -1244,7 +1058,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "news", $proxy);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -1287,7 +1101,7 @@ class startpage{
return $out;
}
private function parse_npt($json, $pagetype, $proxy, $cookies){
private function parse_npt($json, $pagetype, $proxy){
foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
@@ -1295,27 +1109,22 @@ class startpage{
return
$this->backend->store(
json_encode([
"query" =>
http_build_query(
[
"lui" => "english",
"language" => "english",
"sc" => $json["render"]["search_sc"],
"t" => "device",
"cat" => $pagetype,
"segment" => "startpage.udog", // "original" sometimes
"abd" => "0",
"abe" => "0",
"query" => rawurldecode($json["render"]["query"]),
"page" => $page["number"],
"qsr" => "all",
"qadf" => "none", // @ todo fix (??)
...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
]
),
"cookies" => $cookies
]),
http_build_query(
[
"lui" => "english",
"language" => "english",
"sc" => $json["render"]["search_sc"],
"t" => "device",
"cat" => $pagetype,
"segment" => "organic",
"abd" => "0",
"abe" => "0",
"query" => rawurldecode($json["render"]["query"]),
"page" => $page["number"],
"qsr" => "all",
"qadf" => "none" // @ todo fix (??)
]
),
$pagetype,
$proxy
);
@@ -1461,56 +1270,4 @@ class startpage{
throw new Exception("Startpage redirected the scraper to an unhandled page");
}
}
private function solve_anubis($proxy, $html){
//
// solve anubis challenge, if theres one
//
$res = null;
$cookies = [];
try{
$res = $this->anubis->scrape($html["data"]);
}catch(Exception $error){
// do nothang
$this->anubis_error = $error->getMessage();
}
if($res !== null){
// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
// &nonce=3098
// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
// &elapsedTime=111
$anubis_response =
$this->get(
$proxy,
"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
[
"id" => $res["id"],
"response" => $res["response"],
"nonce" => $res["nonce"],
"redir" => "https://www.startpage.com/sp/search",
"elapsedTime" => $res["elapsedTime"]
],
false,
false,
true,
$html["cookies"]
);
// re-fetch search page AGAIN with the anubis bullshit
$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
return $cookies;
}
return false;
}
}
+941 -653
View File
File diff suppressed because it is too large Load Diff
+30 -2
View File
@@ -87,6 +87,10 @@ $settings = [
"value" => "kagi",
"text" => "Kagi"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "yep",
"text" => "Yep"
@@ -141,6 +145,14 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "ghostery",
"text" => "Ghostery"
],
[
"value" => "yep",
"text" => "Yep"
@@ -219,6 +231,10 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "naver",
"text" => "Naver"
@@ -235,6 +251,10 @@ $settings = [
"value" => "pinterest",
"text" => "Pinterest"
],
[
"value" => "cara",
"text" => "Cara"
],
[
"value" => "flickr",
"text" => "Flickr"
@@ -309,6 +329,10 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "naver",
"text" => "Naver"
@@ -355,10 +379,14 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
/*[
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "mojeek",
"text" => "Mojeek"
],*/
],
[
"value" => "baidu",
"text" => "Baidu"
+3 -3
View File
@@ -22,7 +22,7 @@ var list = [];
var pinged_list = [];
var reqs = 0;
var errors = 0;
var sort = 12;
var sort = 6; // highest version first
// check for instance redirect stuff
var redir = [];
@@ -68,8 +68,8 @@ table.innerHTML =
'<th class="extend">Server</th>' +
'<th>Address</th>' +
'<th>Bot protection</th>' +
'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
'<th>API</th>' +
'<th><div class="arrow up"></div>Version</th>' +
'</tr>' +
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+3 -12
View File
@@ -1,18 +1,9 @@
<a href="/" class="link">&lt; Go back</a>
<h1>Donate to the project</h1>
This project takes up most of my free time and costs (a lot of) money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
This project takes up most of my free time and costs money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
To run smoothly, 4get requires:
<ul>
<li>$350/year for server hosting at OVH</li>
<li>$60-80/month for proxies</li>
<li>$54/month for home internet</li>
</ul>
I also bought a dedicated machine for browser rendering hosted at home coming at around $380.<br><br>
According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> currently serves between <a href="https://4get.ca/ami4get">2000-2500 users every day</a>, and serves around <a href="https://4get.ca/api/v2/graph">20,000-22,000 searches</a>!
To run smoothly, 4get requires 20$/month for the server fees and 15-25$ for residential proxies. I also have plans to build my own index to provide better search results; funds raised here will go directly towards better hardware for these purposes. According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> serves between 800-1400 users every day, or around 7000-10000 searches!
<h2>Ko-fi</h2>
This is the most convenient way to donate. Supports PayPal.
@@ -26,4 +17,4 @@ Due to popular demand, I have added an XMR addy. I'm planning to reward users wh
<li><a href="monero:85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB?tx_description=4get+donation">85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB</a></li>
</ul>
I'm immensely grateful for every donation. Thank you all for your generous support!!
Thank you all for your generous support!!
+1 -1
View File
@@ -15,7 +15,7 @@
<div class="navigation">
<a href="/">Home</a>
<a href="/settings">Settings</a>
<a href="https://bsky.app/profile/4getisdown.bsky.social" target="_BLANK">News</a>
<a href="https://git.lolcat.ca/lolcat/4get_news" target="_BLANK">News</a>
</div>
<form method="GET" autocomplete="off">
<div class="searchbox">
+1 -1
View File
@@ -28,7 +28,7 @@
<div class="autocomplete"></div>
</div>
</form>
<a href="settings">Settings</a><a href="instances">Instances</a><a href="https://bsky.app/profile/4getisdown.bsky.social">News</a><a href="api.txt">API</a><a href="about">About</a><a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
<a href="settings">Settings</a><a href="instances">Instances</a><a href="https://git.lolcat.ca/lolcat/4get_news">News</a><a href="api.txt">API</a><a href="about">About</a><a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
<a href="/donate"><b>Donate to the project</b></a>
<div class="subtext">
<a href="https://4get.ca">Clearnet</a><a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">Tor</a><a href="https://lolcat.ca">Report a problem</a><br>
-3
View File
@@ -26,9 +26,6 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->video($get);
-3
View File
@@ -26,9 +26,6 @@ $payload = [
"left" => ""
];
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
$results = $scraper->web($get);