Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0efbb81d23 | ||
|
|
4b83c47637 |
@@ -26,3 +26,4 @@ data/captcha/minecraft/
|
||||
!banner/*default*
|
||||
scraper/curlie.html
|
||||
icons/*
|
||||
node_modules/
|
||||
|
||||
+1
-1
@@ -4,7 +4,7 @@ FROM alpine:3.21
|
||||
WORKDIR /var/www/html/4get
|
||||
|
||||
RUN apk update && apk upgrade
|
||||
RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates ghostscript-fonts
|
||||
RUN apk add php apache2-ssl php84-fileinfo php84-openssl php84-iconv php84-common php84-dom php84-sodium php84-curl curl php84-pecl-apcu php84-apache2 imagemagick php84-pecl-imagick php84-mbstring imagemagick-webp imagemagick-jpeg nss ca-certificates
|
||||
|
||||
COPY --from=curl-impersonate /usr/local/bin /usr/local/bin
|
||||
COPY --from=curl-impersonate /usr/local/lib /usr/local/lib
|
||||
|
||||
@@ -37,20 +37,21 @@ tl;dr 4get is the best way to browse for shit.
|
||||
| Brave | Yandex | Vimeo | Brave | Swisscows | DuckDuckGo |
|
||||
| Yandex | Brave | Sepia Search | Google | | Yandex |
|
||||
| Google | Google | DuckDuckGo | Yahoo! JAPAN | | Google |
|
||||
| Google API | Google API | Brave | Startpage | | Qwant |
|
||||
| Google CSE | Google CSE | Yandex | Baidu | | Marginalia |
|
||||
| Yahoo! JAPAN | Yahoo! JAPAN | Google | | | YouTube |
|
||||
| Startpage | Startpage | Yahoo! JAPAN | | | SoundCloud |
|
||||
| Yep | Naver | Startpage | | | Startpage |
|
||||
| Mwmbl | Baidu | Naver | | | Kagi |
|
||||
| Mojeek | Solofield | Baidu | | | |
|
||||
| Naver | Pinterest | Cốc Cốc | | | |
|
||||
| Baidu | Flickr | Purili | | | |
|
||||
| Cốc Cốc | Pexels | Solofield | | | |
|
||||
| Google API | Google API | Brave | Startpage | | Startpage |
|
||||
| Google CSE | Google CSE | Yandex | Qwant | | Kagi |
|
||||
| Yahoo! JAPAN | Yahoo! JAPAN | Google | Mojeek | | Qwant |
|
||||
| Startpage | Startpage | Yahoo! JAPAN | Baidu | | Ghostery |
|
||||
| Qwant | Qwant | Startpage | | | Yep |
|
||||
| Ghostery | Baidu | Qwant | | | Marginalia |
|
||||
| Yep | Solofield | Baidu | | | YouTube |
|
||||
| Mwmbl | Pinterest | Coc Coc | | | SoundCloud |
|
||||
| Mojeek | Cara | Solofield | | | |
|
||||
| Baidu | Flickr | | | | |
|
||||
| Coc Coc | Pexels | | | | |
|
||||
| Solofield | Pixabay | | | | |
|
||||
| Marginalia | Unsplash | | | | |
|
||||
| Purili | 500px | | | | |
|
||||
| wiby | VSCO | | | | |
|
||||
| wiby | 500px | | | | |
|
||||
| | VSCO | | | | |
|
||||
| | Imgur | | | | |
|
||||
| | FindThatMeme | | | | |
|
||||
|
||||
|
||||
+4
-16
@@ -5,20 +5,8 @@ header("Access-Control-Allow-Origin: *");
|
||||
|
||||
include "data/config.php";
|
||||
|
||||
// get request count in last 24 hours
|
||||
$bot_requests = 0;
|
||||
$real_requests = 0;
|
||||
$time = time();
|
||||
|
||||
for($i=0; $i<24; $i++){
|
||||
|
||||
$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
|
||||
}
|
||||
|
||||
for($i=0; $i<24; $i++){
|
||||
|
||||
$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
|
||||
}
|
||||
$real_requests = apcu_fetch("real_requests");
|
||||
$bot_requests = apcu_fetch("captcha_gen");
|
||||
|
||||
echo json_encode(
|
||||
[
|
||||
@@ -28,8 +16,8 @@ echo json_encode(
|
||||
"name" => config::SERVER_NAME,
|
||||
"description" => config::SERVER_LONG_DESCRIPTION,
|
||||
"bot_protection" => config::BOT_PROTECTION,
|
||||
"real_requests" => $real_requests,
|
||||
"bot_requests" => $bot_requests,
|
||||
"real_requests" => $real_requests === false ? 0 : $real_requests,
|
||||
"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
|
||||
"api_enabled" => config::API_ENABLED,
|
||||
"alt_addresses" => config::ALT_ADDRESSES,
|
||||
"version" => config::VERSION
|
||||
|
||||
+3
-1
@@ -15,11 +15,13 @@ class autocomplete{
|
||||
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
|
||||
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
|
||||
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
|
||||
"yep" => "https://api.yep.com/ac/?query={searchTerms}",
|
||||
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
|
||||
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
|
||||
"sc" => "",
|
||||
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
|
||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
|
||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
|
||||
"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
|
||||
];
|
||||
|
||||
/*
|
||||
|
||||
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "images", false);
|
||||
|
||||
$get = $frontend->parsegetfilters($_GET, $filters);
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
echo json_encode(
|
||||
$scraper->image($get),
|
||||
|
||||
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "music", false);
|
||||
|
||||
$get = $frontend->parsegetfilters($_GET, $filters);
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
echo json_encode(
|
||||
$scraper->music($get),
|
||||
|
||||
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "news", false);
|
||||
|
||||
$get = $frontend->parsegetfilters($_GET, $filters);
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
echo json_encode(
|
||||
$scraper->news($get),
|
||||
|
||||
@@ -27,9 +27,6 @@ new bot_protection($null, $null, $null, "videos", false);
|
||||
|
||||
$get = $frontend->parsegetfilters($_GET, $filters);
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
echo json_encode(
|
||||
$scraper->video($get),
|
||||
|
||||
@@ -38,9 +38,6 @@ if(
|
||||
$get["extendedsearch"] = "no";
|
||||
}
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
|
||||
echo
|
||||
|
||||
@@ -1,100 +0,0 @@
|
||||
<?php
|
||||
|
||||
header("Content-Type: image/png");
|
||||
date_default_timezone_set('America/Toronto');
|
||||
include "../../data/config.php";
|
||||
|
||||
function get_requests($slug){
|
||||
|
||||
$time = time();
|
||||
$requests = [];
|
||||
|
||||
for($i=0; $i<73; $i++){
|
||||
|
||||
$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
|
||||
|
||||
$requests[] = [$i, ($data === false ? 0 : $data)];
|
||||
}
|
||||
|
||||
// rtrim() out all entries that are 0
|
||||
$out = [];
|
||||
$found = false;
|
||||
for($i=count($requests) - 1; $i>=0; $i--){
|
||||
|
||||
if($requests[$i][1] !== 0){
|
||||
|
||||
$found = $i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if($found === false){
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
$requests = array_slice($requests, 0, $found + 1);
|
||||
|
||||
return $requests;
|
||||
}
|
||||
|
||||
$bot_requests = get_requests("bot_requests");
|
||||
$real_requests = get_requests("real_requests");
|
||||
|
||||
$real_reqs_8h =
|
||||
array_sum(
|
||||
array_column(
|
||||
array_slice(
|
||||
$real_requests,
|
||||
0,
|
||||
8
|
||||
),
|
||||
1
|
||||
)
|
||||
);
|
||||
|
||||
$real_reqs_24h =
|
||||
array_sum(
|
||||
array_column(
|
||||
array_slice(
|
||||
$real_requests,
|
||||
0,
|
||||
24
|
||||
),
|
||||
1
|
||||
)
|
||||
);
|
||||
|
||||
$real_reqs_72h =
|
||||
array_sum(
|
||||
array_column(
|
||||
$real_requests,
|
||||
1
|
||||
)
|
||||
);
|
||||
|
||||
include "../../lib/graph_gen.php";
|
||||
$graph = new graph();
|
||||
|
||||
$c = count($real_requests);
|
||||
|
||||
if($c > 25){
|
||||
|
||||
$c = 25;
|
||||
}elseif($c <= 2){
|
||||
|
||||
$c = 2;
|
||||
}
|
||||
|
||||
echo $graph->render([
|
||||
"title" => "Number of requests within the last 72 hours (non-cummulative)",
|
||||
"subtitle" => date("jS M y @ g:ia", time()) . " || 8h=" . number_format($real_reqs_8h) . " - 24h=" . number_format($real_reqs_24h) . " - 72h=" . number_format($real_reqs_72h) . " || " . config::SERVER_NAME,
|
||||
"x_label" => "<= Hours passed =>",
|
||||
"y_label" => "<= Request count/hour =>",
|
||||
"grad_x" => $c,
|
||||
"grad_y" => 20,
|
||||
"data" => [
|
||||
"Bot requests" => $bot_requests,
|
||||
"Real requests" => $real_requests,
|
||||
]
|
||||
]);
|
||||
+18
-17
@@ -5,7 +5,7 @@ class config{
|
||||
// any parameters.
|
||||
|
||||
// 4get version. Please keep this updated
|
||||
const VERSION = 9;
|
||||
const VERSION = 8;
|
||||
|
||||
// Will be shown pretty much everywhere.
|
||||
const SERVER_NAME = "4get";
|
||||
@@ -23,9 +23,6 @@ class config{
|
||||
// Enable the API?
|
||||
const API_ENABLED = true;
|
||||
|
||||
// Display errors. This should ONLY be enabled for debugging and constitutes a security risk.
|
||||
const DISPLAY_ERRORS = false;
|
||||
|
||||
//
|
||||
// 4play settings
|
||||
//
|
||||
@@ -109,31 +106,35 @@ class config{
|
||||
// eachother your serber should appear everywhere.
|
||||
const INSTANCES = [
|
||||
"https://4get.ca",
|
||||
"https://search.fischbytes.de",
|
||||
"https://4get.nadeko.net",
|
||||
"https://4get.zzls.xyz",
|
||||
"https://4getus.zzls.xyz",
|
||||
"https://4get.silly.computer",
|
||||
"https://4get.konakona.moe",
|
||||
"https://4get.lvkaszus.pl",
|
||||
"https://4g.ggtyler.dev",
|
||||
"https://4get.perennialte.ch",
|
||||
"https://4get.sijh.net",
|
||||
"https://4get.hbubli.cc",
|
||||
"https://4get.plunked.party",
|
||||
"https://4get.etenie.pl",
|
||||
"https://4get.lunar.icu",
|
||||
"https://4get.dcs0.hu",
|
||||
"https://4get.kizuki.lol",
|
||||
"https://4get.psily.garden",
|
||||
"https://search.milivojevic.in.rs",
|
||||
"https://4get.snine.nl",
|
||||
"https://4get.datura.network",
|
||||
"https://4get.neco.lol",
|
||||
"https://4get.lol",
|
||||
"https://4get.ch",
|
||||
"https://4get.edmateo.site",
|
||||
"https://4get.sudovanilla.org",
|
||||
"https://search.mint.lgbt",
|
||||
"https://4get.aishiteiru.moe",
|
||||
"https://4.nboeck.de",
|
||||
"https://search.yonderly.org",
|
||||
"https://4get.lurx.net",
|
||||
"https://4get.canine.tools",
|
||||
"https://4get.sny.sh",
|
||||
"https://4g.opnxng.com",
|
||||
"https://s.307200.xyz",
|
||||
"https://4get.privadency.com/"
|
||||
"https://search.mint.lgbt"
|
||||
];
|
||||
|
||||
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
|
||||
// Changing this might break things.
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0";
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
|
||||
|
||||
// User agent to use with 4get-friendly APIs
|
||||
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
|
||||
|
||||
+1
-1
@@ -104,7 +104,7 @@ Now, edit the following file: `/etc/apache2/sites-available/000-default.conf`, r
|
||||
</Directory>
|
||||
|
||||
# deny access to private resources
|
||||
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra|resolve)/">
|
||||
<DirectoryMatch "^/var/www/4get/(data|lib|scraper|template|oracles|docs|docker|extra)/">
|
||||
Require all denied
|
||||
</DirectoryMatch>
|
||||
|
||||
|
||||
@@ -152,8 +152,6 @@ This should expose a webservice located at `http://localhost:3000`. I highly rec
|
||||
|
||||
If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
|
||||
|
||||
**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
|
||||
|
||||
## Setting up 4get
|
||||
Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
|
||||
|
||||
@@ -165,7 +163,3 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
|
||||
**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
|
||||
|
||||
If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
|
||||
|
||||
In case it isn't clear, here is a visual representation of what you want to achieve:
|
||||
|
||||

|
||||
|
||||
@@ -5,32 +5,3 @@ Start the PHP webserver like this, assuming you have curl-impersonate installed
|
||||
```sh
|
||||
LD_PRELOAD=/usr/local/lib/libcurl-impersonate-ff.so CURL_IMPERSONATE=ff117 php -S localhost:8000
|
||||
```
|
||||
|
||||
Make sure to add `.php` to paths, the embedded PHP webserver doesnt support `mod_rewrite`-like directives.
|
||||
|
||||
If, for some reason, you're in an environment where accessing the debug logs is impossible (like those shitty shared webhosts), you can set `DISPLAY_ERRORS` to `true` in `data/config.php`.
|
||||
|
||||
# Debug guide for docker
|
||||
|
||||
Install:
|
||||
```sh
|
||||
git clone https://git.lolcat.ca/lolcat/4get
|
||||
# make changes to docker/apache/http/httpd.conf (comment out the lines that disable logging)
|
||||
docker build -t luuul/4get .
|
||||
```
|
||||
|
||||
Run:
|
||||
```sh
|
||||
docker run --name 4get -p 80:80 -p 443:443 luuul/4get
|
||||
```
|
||||
|
||||
Get debug logs:
|
||||
```sh
|
||||
docker exec -it `docker ps -qf ancestor=luuul/4get:latest` sh # browse the container
|
||||
docker exec -it `docker ps -qf ancestor=luuul/4get:latest` tail -f /var/log/apache2/error_log
|
||||
```
|
||||
|
||||
Restart the container:
|
||||
```sh
|
||||
docker restart 4get
|
||||
```
|
||||
|
||||
@@ -150,3 +150,7 @@ services:
|
||||
##### Tor
|
||||
|
||||
You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
|
||||
|
||||
##### 4play
|
||||
|
||||
You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
### Running 4get and 4play via docker guide
|
||||
|
||||
This setup will run both 4get and the 4play sidecar
|
||||
|
||||
# Docker compose
|
||||
|
||||
```
|
||||
# docker-compose.yaml
|
||||
|
||||
services:
|
||||
fourget:
|
||||
image: luuul/4get:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- FOURGET_PROTO=http
|
||||
- FOURGET_SERVER_NAME=4get.ca
|
||||
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
|
||||
- FOURGET_FPLAY_PASSWORD=cnc
|
||||
- FOURGET_FPLAY_TIMEOUT=30000
|
||||
- FOURGET_FPLAY_ENABLE_API=true
|
||||
ports:
|
||||
- "80:80"
|
||||
fourplay:
|
||||
image: luuul/4get-4play:latest
|
||||
environment:
|
||||
- FOURPLAY_PORT=3030
|
||||
- FOURPLAY_PASSWORD=cnc
|
||||
- FOURPLAY_COMMAND_TIMEOUT=5000
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "3030:3030"
|
||||
|
||||
```
|
||||
|
||||
There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
|
||||
|
||||
# Steps
|
||||
|
||||
1. start the services with
|
||||
|
||||
```
|
||||
docker compose up -d
|
||||
|
||||
```
|
||||
|
||||
2. configure firefox
|
||||
|
||||
folllow the steps in [configure](./configure.md) for firefox setup
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM alpine:3.21
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk update && apk upgrade
|
||||
RUN apk add nodejs npm
|
||||
|
||||
COPY package*.json ./
|
||||
|
||||
RUN npm ci --only=production && npm cache clean --force
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 3030
|
||||
|
||||
ENTRYPOINT ["node"]
|
||||
CMD ["page-render.js"]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# docker-compose.yaml
|
||||
|
||||
services:
|
||||
fourget:
|
||||
image: luuul/4get:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- FOURGET_PROTO=http
|
||||
- FOURGET_SERVER_NAME=4get.ca
|
||||
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
|
||||
- FOURGET_FPLAY_PASSWORD=cnc
|
||||
- FOURGET_FPLAY_TIMEOUT=30000
|
||||
- FOURGET_FPLAY_ENABLE_API=true
|
||||
ports:
|
||||
- "80:80"
|
||||
fourplay:
|
||||
image: luuul/4get-4play:latest
|
||||
environment:
|
||||
- FOURPLAY_PORT=3030
|
||||
- FOURPLAY_PASSWORD=cnc
|
||||
- FOURPLAY_COMMAND_TIMEOUT=5000
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "3030:3030"
|
||||
Generated
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"@lawlers/4play": "^1.2.5",
|
||||
"ws": "^8.21.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@lawlers/4play": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
|
||||
"integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"ws": "^8.21.1"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
|
||||
"integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://git.lolcat.ca/lolcat/4get/"
|
||||
},
|
||||
"license": "AGPL-3.0-only",
|
||||
"author": "lolcat",
|
||||
"type": "commonjs",
|
||||
"main": "page-render.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@lawlers/4play": "^1.2.5",
|
||||
"ws": "^8.21.2"
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
|
||||
const http = require("http");
|
||||
|
||||
// 4play settings
|
||||
var port = 3030;
|
||||
var timeout = 30000;
|
||||
var password = "cnc";
|
||||
var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
|
||||
var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
|
||||
var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
|
||||
|
||||
// ==================================================
|
||||
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
<?php
|
||||
|
||||
$scrapers = [
|
||||
"web" => [
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Google API",
|
||||
"Google CSE",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Yep",
|
||||
"Mwmbl",
|
||||
"Mojeek",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Cốc Cốc",
|
||||
"Solofield",
|
||||
"Marginalia",
|
||||
"Purili",
|
||||
"wiby"
|
||||
],
|
||||
"images" => [
|
||||
"DuckDuckGo",
|
||||
"Yandex",
|
||||
"Brave",
|
||||
"Google",
|
||||
"Google API",
|
||||
"Google CSE",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Solofield",
|
||||
"Pinterest",
|
||||
"Flickr",
|
||||
"Pexels",
|
||||
"Pixabay",
|
||||
"Unsplash",
|
||||
"500px",
|
||||
"VSCO",
|
||||
"Imgur",
|
||||
"FindThatMeme"
|
||||
],
|
||||
"videos" => [
|
||||
"YouTube",
|
||||
"Vimeo",
|
||||
"Sepia Search",
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Cốc Cốc",
|
||||
"Purili",
|
||||
"Solofield"
|
||||
],
|
||||
"news" => [
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Google",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Baidu"
|
||||
],
|
||||
"music" => [
|
||||
"SoundCloud",
|
||||
"Swisscows"
|
||||
],
|
||||
"autocomplete" => [
|
||||
"Brave",
|
||||
"DuckDuckGo",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Qwant",
|
||||
"Marginalia",
|
||||
"YouTube",
|
||||
"SoundCloud",
|
||||
"Startpage",
|
||||
"Kagi"
|
||||
]
|
||||
];
|
||||
|
||||
$lines = [
|
||||
0 => "",
|
||||
1 => ""
|
||||
];
|
||||
|
||||
// get longest scraper string
|
||||
// also get list of scrapers with most entries
|
||||
$biggest_category_len = 0;
|
||||
foreach($scrapers as $category_name => $category){
|
||||
|
||||
$count = count($category);
|
||||
|
||||
if($count > $biggest_category_len){
|
||||
|
||||
$biggest_category_len = $count;
|
||||
}
|
||||
}
|
||||
|
||||
foreach($scrapers as $category_name => $category){
|
||||
|
||||
// get longest string of the category
|
||||
$longest_len = mb_strlen($category_name);
|
||||
foreach($category as $scraper){
|
||||
|
||||
$strlen = mb_strlen($scraper);
|
||||
|
||||
if($strlen > $longest_len){
|
||||
|
||||
$longest_len = $strlen;
|
||||
}
|
||||
}
|
||||
|
||||
// add header
|
||||
$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
|
||||
$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
|
||||
|
||||
for($i=2; $i<count($category) + 2; $i++){
|
||||
|
||||
if(!isset($lines[$i])){
|
||||
|
||||
$lines[$i] = "";
|
||||
}
|
||||
|
||||
$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
|
||||
}
|
||||
|
||||
if($i - 2 < $biggest_category_len){
|
||||
|
||||
for($k=$i; $k<$biggest_category_len + 2; $k++){
|
||||
|
||||
if(!isset($lines[$k])){
|
||||
|
||||
$lines[$k] = "";
|
||||
}
|
||||
|
||||
$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach($lines as $line){
|
||||
|
||||
echo $line . "|\n";
|
||||
}
|
||||
@@ -24,9 +24,6 @@ $payload = [
|
||||
"nextpage" => ""
|
||||
];
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
$results = $scraper->image($get);
|
||||
|
||||
|
||||
+5
-8
@@ -45,15 +45,15 @@ class anubis{
|
||||
|
||||
if(
|
||||
!isset($script["challenge"]) ||
|
||||
!is_string($script["challenge"]["randomData"]) ||
|
||||
!isset($script["rules"]["difficulty"]) ||
|
||||
!is_int($script["rules"]["difficulty"])
|
||||
!is_int($script["rules"]["difficulty"]) ||
|
||||
!is_string($script["challenge"])
|
||||
){
|
||||
|
||||
throw new Exception("Found invalid challenge data");
|
||||
}
|
||||
|
||||
return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
|
||||
return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
|
||||
}
|
||||
|
||||
private function is_valid_hash($hash, $difficulty){
|
||||
@@ -74,10 +74,9 @@ class anubis{
|
||||
return true;
|
||||
}
|
||||
|
||||
public function rape($id, $data, $difficulty = 5){
|
||||
public function rape($data, $difficulty = 5){
|
||||
|
||||
$nonce = 0;
|
||||
$microtime = microtime(true);
|
||||
|
||||
while(true){
|
||||
|
||||
@@ -88,12 +87,10 @@ class anubis{
|
||||
$hash_hex = bin2hex($hash_binary);
|
||||
|
||||
return [
|
||||
"id" => $id,
|
||||
"response" => $hash_hex,
|
||||
//"data" => $data,
|
||||
//"difficulty" => $difficulty,
|
||||
"nonce" => $nonce,
|
||||
"elapsedTime" => round((microtime(true) - $microtime) * 1000)
|
||||
"nonce" => $nonce
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@ class bot_protection{
|
||||
// check if we want captcha
|
||||
if(config::BOT_PROTECTION !== 1){
|
||||
|
||||
apcu_inc("real_requests");
|
||||
if($output === true){
|
||||
$frontend->loadheader(
|
||||
$get,
|
||||
@@ -31,7 +32,7 @@ class bot_protection{
|
||||
// does key exist
|
||||
apcu_exists($_COOKIE["pass"])
|
||||
){
|
||||
|
||||
|
||||
// exists, increment counter
|
||||
$inc = apcu_inc($_COOKIE["pass"]);
|
||||
|
||||
@@ -45,6 +46,8 @@ class bot_protection{
|
||||
}else{
|
||||
|
||||
// the cookie is OK! dont die() and give results
|
||||
apcu_inc("real_requests");
|
||||
|
||||
if($output === true){
|
||||
$frontend->loadheader(
|
||||
$get,
|
||||
@@ -161,7 +164,8 @@ class bot_protection{
|
||||
$key = "k" . $inc . "." . $this->randomchars();
|
||||
|
||||
apcu_inc($key, 1, $stupid, 86400);
|
||||
apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
|
||||
|
||||
apcu_inc("real_requests");
|
||||
|
||||
setcookie(
|
||||
"pass",
|
||||
@@ -186,7 +190,6 @@ class bot_protection{
|
||||
}
|
||||
}
|
||||
|
||||
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200);
|
||||
$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
|
||||
|
||||
$payload = [
|
||||
@@ -249,8 +252,7 @@ class bot_protection{
|
||||
$frontend->loadheader(
|
||||
$get,
|
||||
$filters,
|
||||
$page,
|
||||
false
|
||||
$page
|
||||
);
|
||||
|
||||
echo $frontend->load("search.html", $payload);
|
||||
|
||||
+18
-510
@@ -118,7 +118,7 @@ class frontend{
|
||||
return trim($html);
|
||||
}
|
||||
|
||||
public function loadheader(array $get, array $filters, string $page, bool $increment_bot_counter = true){
|
||||
public function loadheader(array $get, array $filters, string $page){
|
||||
|
||||
echo
|
||||
$this->load("header.html", [
|
||||
@@ -177,15 +177,13 @@ class frontend{
|
||||
){
|
||||
|
||||
// bot detected !!
|
||||
if($increment_bot_counter){
|
||||
|
||||
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
|
||||
}
|
||||
apcu_inc("captcha_gen");
|
||||
|
||||
$this->drawerror(
|
||||
"Tshh, blocked!",
|
||||
'Your browser, IP or IP range has been blocked from this 4get instance. If this is an error, please <a href="/about">contact the administrator</a>.'
|
||||
);
|
||||
die();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -223,7 +221,7 @@ class frontend{
|
||||
'This scraper returned an error:' .
|
||||
'<div class="code">' . htmlspecialchars($error) . '</div>' .
|
||||
'Things you can try:' .
|
||||
'<ul>' .
|
||||
'<ul>' .
|
||||
'<li>Use a different scraper</li>' .
|
||||
'<li>Remove keywords that could cause errors</li>' .
|
||||
'<li><a href="/instances?target=' . $target . "&" . $this->buildquery($get, false) . '">Try your search on another 4get instance</a></li>' .
|
||||
@@ -535,490 +533,14 @@ class frontend{
|
||||
'</button>' .
|
||||
'<div class="favicon-dropdown">';
|
||||
|
||||
$archives = [];
|
||||
|
||||
$fansites = [
|
||||
[
|
||||
"url" => "https://cum.st/creators?cq=",
|
||||
"favicon" => "cum.st",
|
||||
"favicon_alt" => "cu",
|
||||
"title" => "OnlyHaven"
|
||||
],
|
||||
[
|
||||
"url" => "https://pawchive.pw/artists?q=",
|
||||
"favicon" => "pawchive.pw",
|
||||
"favicon_alt" => "pa",
|
||||
"title" => "Pawchive"
|
||||
],
|
||||
[
|
||||
"url" => "https://bakemono.app/creators?q=",
|
||||
"favicon" => "bakemono.app",
|
||||
"favicon_alt" => "ba",
|
||||
"title" => "Bakemono"
|
||||
],
|
||||
[
|
||||
"url" => "https://kemono.cr/artists?q=",
|
||||
"favicon" => "kemono.cr",
|
||||
"favicon_alt" => "ke",
|
||||
"title" => "Kemono"
|
||||
],
|
||||
[
|
||||
"url" => "https://coomer.st/artists?q=",
|
||||
"favicon" => "coomer.st",
|
||||
"favicon_alt" => "co",
|
||||
"title" => "Coomer"
|
||||
]
|
||||
];
|
||||
|
||||
// add website-specific archives
|
||||
switch($host["host"]){
|
||||
|
||||
case "www.youtube.com":
|
||||
case "music.youtube.com":
|
||||
case "youtube.com":
|
||||
case "m.youtube.com":
|
||||
case "youtu.be":
|
||||
|
||||
if(
|
||||
(
|
||||
$host["host"] == "youtu.be" &&
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9_-]+)/',
|
||||
$host["path"],
|
||||
$slug
|
||||
)
|
||||
) ||
|
||||
(
|
||||
isset($host["query"]) &&
|
||||
preg_match(
|
||||
'/v=([A-Za-z0-9_-]+)/',
|
||||
$host["query"],
|
||||
$slug
|
||||
)
|
||||
)
|
||||
){
|
||||
|
||||
// for watch?v=slug
|
||||
$archives[] = [
|
||||
"url" => "https://findyoutubevideo.thetechrobo.ca/?q=" . $slug[1],
|
||||
"favicon" => "findyoutubevideo.thetechrobo.ca",
|
||||
"favicon_alt" => "fi",
|
||||
"title" => "FindYouTubeVideo"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://filmot.com/video/" . $slug[1] . "/",
|
||||
"favicon" => "filmot.com",
|
||||
"favicon_alt" => "fi",
|
||||
"title" => "Filmot (metadata only)"
|
||||
];
|
||||
}elseif(
|
||||
preg_match(
|
||||
'/^\/(?:channel)\/@?([A-Za-z0-9_-]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://filmot.com/channel/" . $username[1] . "/",
|
||||
"favicon" => "filmot.com",
|
||||
"favicon_alt" => "fi",
|
||||
"title" => "Filmot (metadata only)"
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "m.twitch.tv":
|
||||
case "twitch.tv":
|
||||
case "player.twitch.tv":
|
||||
case "www.twitch.tv":
|
||||
if(
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
) &&
|
||||
$username[1] != "videos"
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://vodvod.top/channels/@" . $username[1],
|
||||
"favicon" => "vodvod.top",
|
||||
"favicon_alt" => "vo",
|
||||
"title" => "vodvod"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://twitchtracker.com/" . $username[1],
|
||||
"favicon" => "twitchtracker.com",
|
||||
"favicon_alt" => "tw",
|
||||
"title" => "TwitchTracker"
|
||||
];
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case "kick.com":
|
||||
case "player.kick.com":
|
||||
if(
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://lick.lolcat.ca/channel?name=" . $username[1],
|
||||
"favicon" => "lick.lolcat.ca",
|
||||
"favicon_alt" => "li",
|
||||
"title" => "lick"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://kicktracker.net/" . $username[1],
|
||||
"favicon" => "kicktracker.net",
|
||||
"favicon_alt" => "ki",
|
||||
"title" => "Kick tracker"
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "x.com":
|
||||
case "twitter.com":
|
||||
if(
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://web.archive.org/web/*/https://x.com/" . $username[1] . "/status*",
|
||||
"favicon" => "archive.org",
|
||||
"favicon_alt" => "ar",
|
||||
"title" => "Archive.org: Tweets >2023"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://web.archive.org/web/*/https://twitter.com/" . $username[1] . "/status*",
|
||||
"favicon" => "archive.org",
|
||||
"favicon_alt" => "ar",
|
||||
"title" => "Archive.org: Tweets <2023"
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "www.instagram.com":
|
||||
case "instagram.com":
|
||||
if(
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
) &&
|
||||
$username[1] != "p"
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://instarchiver.net/users?q=" . $username[1],
|
||||
"favicon" => "instarchiver.net",
|
||||
"favicon_alt" => "in",
|
||||
"title" => "Instarchiver"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://www.storiesdb.ch/" . $username[1],
|
||||
"favicon" => "www.storiesdb.ch",
|
||||
"favicon_alt" => "st",
|
||||
"title" => "StoriesDB"
|
||||
];
|
||||
}
|
||||
break;
|
||||
|
||||
case "reddit.com":
|
||||
case "old.reddit.com":
|
||||
case "www.reddit.com":
|
||||
if(isset($host["path"])){
|
||||
// direct thread lookup
|
||||
// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=pullpush
|
||||
// https://ihsoyct.github.io/r/selfhosted/comments/16emfv0/4get_a_proxy_search_engine_that_doesnt_suck/?backend=artic_shift
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/r\/[^\/]+\/comments\/[^?&]+/',
|
||||
$host["path"],
|
||||
$slug
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io{$slug[0]}?backend=artic_shift",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "Artic Shift"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io{$slug[0]}?backend=pullpush",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "PullPush"
|
||||
];
|
||||
}
|
||||
|
||||
// subreddit thread search
|
||||
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=submissions&sort=desc
|
||||
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=submissions&sort=desc
|
||||
// subreddit comment search
|
||||
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=artic_shift&mode=comments&sort=desc
|
||||
// https://ihsoyct.github.io/?subreddit=selfhosted&backend=pullpush&mode=comments&sort=desc
|
||||
elseif(
|
||||
preg_match(
|
||||
'/^\/r\/([^\/]+)\/(?:(?:search|wiki|about)\/?)?(?:$|\?|&)/',
|
||||
$host["path"],
|
||||
$slug
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=submissions&sort=desc",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "Artic Shift (Search threads)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=submissions&sort=desc",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "PullPush (Search threads)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=artic_shift&mode=comments&sort=desc",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "Artic Shift (Search comments)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/?subreddit={$slug[1]}&backend=pullpush&mode=comments&sort=desc",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "PullPush (Search comments)"
|
||||
];
|
||||
}
|
||||
|
||||
// user thread lookup
|
||||
// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=artic_shift
|
||||
// https://ihsoyct.github.io/index.html?author=google&mode=submissions&backend=pullpush
|
||||
// user comment lookup
|
||||
// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=artic_shift
|
||||
// https://ihsoyct.github.io/index.html?author=google&mode=comments&backend=pullpush
|
||||
elseif(
|
||||
preg_match(
|
||||
'/^\/(?:u|user)\/([^\/]+)\/(?:(?:comments|submitted)\/?)?(?:$|\?|&)/',
|
||||
$host["path"],
|
||||
$slug
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=artic_shift",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "Artic Shift (Search threads)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=submissions&backend=pullpush",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "PullPush (Search threads)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=artic_shift",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "Artic Shift (Search comments)"
|
||||
];
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://ihsoyct.github.io/index.html?author={$slug[1]}&mode=comments&backend=pullpush",
|
||||
"favicon" => "reddit.com",
|
||||
"favicon_alt" => "re",
|
||||
"title" => "PullPush (Search comments)"
|
||||
];
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
// https://cum.st/creators?cq= onlyfans fansly patreon
|
||||
// https://pawchive.pw/artists?q= patreon pixiv/fanbox discord
|
||||
// https://bakemono.app/creators?q= pixiv/fanbox fansly onlyfans patreon
|
||||
// https://kemono.cr/artists?q= patreon pixiv/fanbox fantia afdian boosty gumroad subscribestar dlsite
|
||||
// https://coomer.st/artists?q= onlyfans fansly candfans
|
||||
|
||||
// cant look up
|
||||
/*
|
||||
case "fantia.jp": // https://fantia.jp/fanclubs/ <integer>
|
||||
*/
|
||||
|
||||
// normal
|
||||
case "onlyfans.com": // https://onlyfans.com/ <username>
|
||||
case "fansly.com": // https://fansly.com/ <username>
|
||||
case "fanbox.cc":
|
||||
case "patreon.com": // https://www.patreon.com/ <username>
|
||||
case "www.patreon.com":
|
||||
case "boosty.to": // https://boosty.to/ <username>
|
||||
case "www.subscribestar.com": // https://www.subscribestar.com/ <username>
|
||||
case "subscribestar.com":
|
||||
case "www.dlsite.com": // https://www.dlsite.com/ <username> /
|
||||
case "dlsite.com":
|
||||
case "candfans.com": // https://candfans.com/ <username>
|
||||
case "afdian.com": // https://afdian.com/a/ <username>
|
||||
if(
|
||||
isset($host["path"]) &&
|
||||
(
|
||||
(
|
||||
$host["host"] == "afdian.com" &&
|
||||
preg_match(
|
||||
'/^\/a\/([A-Za-z0-9-_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
)
|
||||
) ||
|
||||
preg_match(
|
||||
'/^\/([A-Za-z0-9-_.]+)/',
|
||||
$host["path"],
|
||||
$username
|
||||
)
|
||||
)
|
||||
){
|
||||
|
||||
foreach($fansites as $fansite){
|
||||
|
||||
$archives[] = [
|
||||
"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
|
||||
"favicon" => $fansite["favicon"],
|
||||
"favicon_alt" => $fansite["favicon_alt"],
|
||||
"title" => $fansite["title"]
|
||||
];
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// special fansite cases
|
||||
if(
|
||||
preg_match(
|
||||
'/^([A-Za-z0-9_]+)\.(?:fanbox\.cc|gumroad\.com)/',
|
||||
$host["host"],
|
||||
$username
|
||||
)
|
||||
){
|
||||
|
||||
foreach($fansites as $fansite){
|
||||
|
||||
$archives[] = [
|
||||
"url" => $fansite["url"] . $this->sanitize_slug($username[1]),
|
||||
"favicon" => $fansite["favicon"],
|
||||
"favicon_alt" => $fansite["favicon_alt"],
|
||||
"title" => $fansite["title"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
// detect git
|
||||
if(
|
||||
(
|
||||
stripos(
|
||||
$host["host"],
|
||||
"git."
|
||||
) !== false ||
|
||||
$host["host"] == "codeberg.org" ||
|
||||
$host["host"] == "sourceforge.net" ||
|
||||
$host["host"] == "github.com"
|
||||
) &&
|
||||
isset($host["path"]) &&
|
||||
preg_match(
|
||||
'/^(\/[^\/]+\/[^\/]+\/?)/',
|
||||
$host["path"],
|
||||
$edge
|
||||
)
|
||||
){
|
||||
|
||||
$archives[] = [
|
||||
"url" => "https://archive.softwareheritage.org/browse/origin/directory/?origin_url=" . urlencode($host["scheme"] . "://" . $host["host"] . $edge[1]),
|
||||
"favicon" => "www.softwareheritage.org",
|
||||
"favicon_alt" => "so",
|
||||
"title" => "SoftwareHeritage"
|
||||
];
|
||||
}
|
||||
|
||||
$archives =
|
||||
array_merge(
|
||||
$archives,
|
||||
[
|
||||
[
|
||||
"url" => "https://web.archive.org/web/" . $urlencode,
|
||||
"favicon" => "archive.org",
|
||||
"favicon_alt" => "ar",
|
||||
"title" => "Archive.org"
|
||||
],
|
||||
[
|
||||
"url" => "https://archive.ph/newest/" . htmlspecialchars($link),
|
||||
"favicon" => "archive.ph",
|
||||
"favicon_alt" => "ar",
|
||||
"title" => "Archive.ph"
|
||||
],
|
||||
[
|
||||
"url" => "https://yandex.com/search/?text=" . urlencode("url:" . $link),
|
||||
"favicon" => "yandex.com",
|
||||
"favicon_alt" => "ya",
|
||||
"title" => "Yandex cache"
|
||||
],
|
||||
[
|
||||
"url" => "https://ghostarchive.org/search?term=" . $urlencode,
|
||||
"favicon" => "ghostarchive.org",
|
||||
"favicon_alt" => "gh",
|
||||
"title" => "Ghostarchive"
|
||||
],
|
||||
[
|
||||
"url" => "https://arquivo.pt/wayback/" . htmlspecialchars($link),
|
||||
"favicon" => "arquivo.pt",
|
||||
"favicon_alt" => "ar",
|
||||
"title" => "Arquivo.pt"
|
||||
],
|
||||
[
|
||||
"url" => "https://megalodon.jp/?url=" . $urlencode,
|
||||
"favicon" => "megalodon.jp",
|
||||
"favicon_alt" => "me",
|
||||
"title" => "Megalodon"
|
||||
],
|
||||
[
|
||||
"url" => "https://www.webcitation.org/query?url=" . $urlencode,
|
||||
"favicon" => "webcitation.org",
|
||||
"favicon_alt" => "we",
|
||||
"title" => "Webcitation"
|
||||
]
|
||||
]
|
||||
);
|
||||
|
||||
foreach($archives as $archive){
|
||||
|
||||
$payload .= '<a href="' . $archive["url"] . '" class="list" target="_BLANK"><img src="/favicon?s=https://' . $archive["favicon"] . '" alt="' . $archive["favicon_alt"] . '">' . $archive["title"] . '</a>';
|
||||
}
|
||||
|
||||
$payload .= '</div>';
|
||||
$payload .=
|
||||
'<a href="https://web.archive.org/web/' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.org" alt="ar">Archive.org</a>' .
|
||||
'<a href="https://archive.ph/newest/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://archive.is" alt="ar">Archive.is</a>' .
|
||||
'<a href="https://ghostarchive.org/search?term=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://ghostarchive.org" alt="gh">Ghostarchive</a>' .
|
||||
'<a href="https://arquivo.pt/wayback/' . htmlspecialchars($link) . '" class="list" target="_BLANK"><img src="/favicon?s=https://arquivo.pt" alt="ar">Arquivo.pt</a>' .
|
||||
'<a href="https://www.bing.com/search?q=url%3A' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://bing.com" alt="bi">Bing cache</a>' .
|
||||
'<a href="https://megalodon.jp/?url=' . $urlencode . '" class="list" target="_BLANK"><img src="/favicon?s=https://megalodon.jp" alt="me">Megalodon</a>' .
|
||||
'</div>';
|
||||
}
|
||||
|
||||
/*
|
||||
@@ -1071,13 +593,6 @@ class frontend{
|
||||
|
||||
public function getscraperfilters($page){
|
||||
|
||||
// hack: enable error reporting if configured
|
||||
if(config::DISPLAY_ERRORS === true){
|
||||
|
||||
ini_set('display_errors', 1);
|
||||
ini_set('display_startup_errors', 1);
|
||||
}
|
||||
|
||||
$get_scraper = isset($_COOKIE["scraper_$page"]) ? $_COOKIE["scraper_$page"] : null;
|
||||
|
||||
if(
|
||||
@@ -1129,6 +644,7 @@ class frontend{
|
||||
"google_cse" => "Google CSE",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"yep" => "Yep",
|
||||
"mwmbl" => "Mwmbl",
|
||||
"mojeek" => "Mojeek",
|
||||
@@ -1155,10 +671,12 @@ class frontend{
|
||||
"google_cse" => "Google CSE",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"naver" => "Naver",
|
||||
"baidu" => "Baidu",
|
||||
"solofield" => "Solofield",
|
||||
"pinterest" => "Pinterest",
|
||||
"cara" => "Cara",
|
||||
"flickr" => "Flickr",
|
||||
"pexels" => "Pexels",
|
||||
"pixabay" => "Pixabay",
|
||||
@@ -1177,7 +695,6 @@ class frontend{
|
||||
"option" => [
|
||||
"yt" => "YouTube",
|
||||
//"archiveorg" => "Archive.org",
|
||||
//"dailymotion" => "Dailymotion",
|
||||
"vimeo" => "Vimeo",
|
||||
//"odysee" => "Odysee",
|
||||
"sepiasearch" => "Sepia Search",
|
||||
@@ -1188,6 +705,7 @@ class frontend{
|
||||
"google" => "Google",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"naver" => "Naver",
|
||||
"baidu" => "Baidu",
|
||||
"coccoc" => "Cốc Cốc",
|
||||
@@ -1206,7 +724,8 @@ class frontend{
|
||||
"google" => "Google",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
//"mojeek" => "Mojeek",
|
||||
"qwant" => "Qwant",
|
||||
"mojeek" => "Mojeek",
|
||||
"baidu" => "Baidu"
|
||||
]
|
||||
];
|
||||
@@ -1538,17 +1057,6 @@ class frontend{
|
||||
return http_build_query($out);
|
||||
}
|
||||
|
||||
private function sanitize_slug($username){
|
||||
|
||||
return urlencode(preg_replace('/[-_]/', " ", $username));
|
||||
}
|
||||
|
||||
public function increment_real_reqs($scraper){
|
||||
|
||||
apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
|
||||
apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
|
||||
}
|
||||
|
||||
public function htmlimage($image, $format){
|
||||
|
||||
if(
|
||||
|
||||
@@ -1,388 +0,0 @@
|
||||
<?php
|
||||
|
||||
class graph{
|
||||
|
||||
public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
|
||||
|
||||
$graph = array_merge([
|
||||
"title" => "Title",
|
||||
"subtitle" => "description",
|
||||
"grad_x" => 25,
|
||||
"grad_y" => 20,
|
||||
"x_label" => "<= x =>",
|
||||
"y_label" => "<= y =>",
|
||||
"antialias" => false,
|
||||
"data" => []
|
||||
], $graph);
|
||||
|
||||
$size = array_merge([
|
||||
"image_width" => 750,
|
||||
"image_height" => 500,
|
||||
"x" => 90,
|
||||
"y" => 50,
|
||||
"width" => 650,
|
||||
"height" => 300,
|
||||
"square" => 11,
|
||||
"column" => 6
|
||||
], $size);
|
||||
|
||||
$colors = array_merge([
|
||||
"bg" => new ImagickPixel("#1d2021"),
|
||||
"lines_bg" => new ImagickPixel("#3c3836"),
|
||||
"outline" => new ImagickPixel("#a89984"),
|
||||
"text" => new ImagickPixel("#ebdbb2"),
|
||||
"transparent" => new ImagickPixel("transparent")
|
||||
], $colors);
|
||||
|
||||
$colors_line = array_merge([
|
||||
new ImagickPixel("#83a598"),
|
||||
new ImagickPixel("#fb4934"),
|
||||
new ImagickPixel("#b8bb26"),
|
||||
new ImagickPixel("#fabd2f"),
|
||||
new ImagickPixel("#d3869b"),
|
||||
new ImagickPixel("#8ec07c"),
|
||||
new ImagickPixel("#fe8019"),
|
||||
new ImagickPixel("#458588"),
|
||||
new ImagickPixel("#cc241d"),
|
||||
new ImagickPixel("#98971a"),
|
||||
new ImagickPixel("#d79921"),
|
||||
new ImagickPixel("#b16286"),
|
||||
new ImagickPixel("#689d6a"),
|
||||
new ImagickPixel("#d65d0e"),
|
||||
new ImagickPixel("#a89984") // other
|
||||
], $colors_line);
|
||||
|
||||
$not_enough_data = false;
|
||||
|
||||
// compute datapoint max ranges
|
||||
$max_range_x = 0;
|
||||
$max_range_y = 0;
|
||||
if(count($graph["data"]) === 0){
|
||||
|
||||
$not_enough_data = true;
|
||||
}else{
|
||||
|
||||
foreach($graph["data"] as $datapoint => $value){
|
||||
|
||||
if(count($value) < 2){
|
||||
|
||||
$not_enough_data = true;
|
||||
}else{
|
||||
|
||||
$nv1 = [];
|
||||
$nv2 = [];
|
||||
foreach($value as $k => $v){
|
||||
|
||||
$nv1[] = $v[0];
|
||||
$nv2[] = $v[1];
|
||||
}
|
||||
|
||||
$max1 = max($nv1);
|
||||
$max2 = max($nv2);
|
||||
if($max_range_x < $max1){ $max_range_x = $max1; }
|
||||
if($max_range_y < $max2){ $max_range_y = $max2; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$im = new Imagick();
|
||||
$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
|
||||
$im->setImageFormat("png");
|
||||
|
||||
$draw = new ImagickDraw();
|
||||
|
||||
//
|
||||
// Draw background grid
|
||||
//
|
||||
$draw->setStrokeColor($colors["lines_bg"]);
|
||||
$draw->setStrokeAntialias($graph["antialias"]);
|
||||
$draw->setStrokeWidth(1);
|
||||
$draw->setFillColor($colors["transparent"]);
|
||||
|
||||
$text = new ImagickDraw();
|
||||
$text->setFontSize(11);
|
||||
$text->setTextAntialias($graph["antialias"]);
|
||||
$text->setFillColor($colors["text"]);
|
||||
|
||||
|
||||
// vertical lines
|
||||
$offset = $size["width"] / ($graph["grad_x"] - 1);
|
||||
|
||||
for($i=0; $i<$graph["grad_x"]; $i++){
|
||||
|
||||
$x = round($size["x"] + ($offset * $i));
|
||||
|
||||
if(
|
||||
$i === 0 ||
|
||||
$i === $graph["grad_x"] - 1
|
||||
){
|
||||
|
||||
$draw->setStrokeColor($colors["outline"]);
|
||||
}else{
|
||||
|
||||
$draw->setStrokeColor($colors["lines_bg"]);
|
||||
}
|
||||
|
||||
$draw->line(
|
||||
$x,
|
||||
$size["y"],
|
||||
$x,
|
||||
$size["y"] + $size["height"],
|
||||
);
|
||||
|
||||
if($not_enough_data === false){
|
||||
// add numbas
|
||||
$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
|
||||
$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
|
||||
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
round($x - ($number_width / 2)),
|
||||
$size["y"] + $size["height"] + 14,
|
||||
0,
|
||||
$number
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
// horizontal lines
|
||||
$text->setTextAlignment(Imagick::ALIGN_RIGHT);
|
||||
$offset = $size["height"] / ($graph["grad_y"] - 1);
|
||||
|
||||
$vertical_text_size = 0;
|
||||
|
||||
for($i=0; $i<$graph["grad_y"]; $i++){
|
||||
|
||||
$y = round($size["y"] + ($offset * $i));
|
||||
|
||||
if(
|
||||
$i === 0 ||
|
||||
$i === $graph["grad_y"] - 1
|
||||
){
|
||||
|
||||
$draw->setStrokeColor($colors["outline"]);
|
||||
}else{
|
||||
|
||||
$draw->setStrokeColor($colors["lines_bg"]);
|
||||
}
|
||||
|
||||
$draw->line(
|
||||
$size["x"] + 1,
|
||||
$y,
|
||||
$size["x"] + $size["width"] - 1,
|
||||
$y,
|
||||
);
|
||||
|
||||
if($not_enough_data === false){
|
||||
$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
|
||||
$query = $im->queryFontMetrics($text, $val);
|
||||
|
||||
if($query["textWidth"] > $vertical_text_size){
|
||||
|
||||
$vertical_text_size = $query["textWidth"];
|
||||
}
|
||||
|
||||
// add numbas
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$size["x"] - 4,
|
||||
$y + 4,
|
||||
0,
|
||||
$val
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//
|
||||
// Draw legend
|
||||
//
|
||||
$text->setTextAlignment(Imagick::ALIGN_LEFT);
|
||||
$top_padding = 44;
|
||||
$c = count($colors_line);
|
||||
$padding = $size["width"] / $size["column"];
|
||||
|
||||
$line = new ImagickDraw();
|
||||
//$line->setStrokeWidth(1);
|
||||
$line->setStrokeAntialias($graph["antialias"]);
|
||||
|
||||
if($not_enough_data === false){
|
||||
$i = 0;
|
||||
foreach($graph["data"] as $datafield => $data){
|
||||
|
||||
$used_color = $colors_line[$i % $c];
|
||||
$draw->setStrokeColor($used_color);
|
||||
$row = floor($i / $size["column"]);
|
||||
|
||||
$x = round($size["x"] + (($i % $size["column"]) * $padding));
|
||||
$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
|
||||
|
||||
// draw legend
|
||||
$draw->rectangle(
|
||||
$x,
|
||||
$y,
|
||||
$x + $size["square"],
|
||||
$y + $size["square"]
|
||||
);
|
||||
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$x + $size["square"] + 4,
|
||||
$y + 10,
|
||||
0,
|
||||
$datafield
|
||||
);
|
||||
|
||||
//
|
||||
// Draw lines
|
||||
//
|
||||
$line->setStrokeColor($used_color);
|
||||
$old_x = null;
|
||||
$old_y = null;
|
||||
$continue = true;
|
||||
|
||||
foreach($data as $field){
|
||||
|
||||
// field 0 = hour (sorted)
|
||||
// field 1 = datapoint
|
||||
|
||||
$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
|
||||
$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
|
||||
|
||||
if($continue){
|
||||
|
||||
$continue = false;
|
||||
$old_x = $x;
|
||||
$old_y = $y;
|
||||
continue;
|
||||
}
|
||||
|
||||
$line->line($old_x, $old_y, $x, $y);
|
||||
|
||||
$old_x = $x;
|
||||
$old_y = $y;
|
||||
}
|
||||
|
||||
$i++;
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// add background labels
|
||||
//
|
||||
$text->setFontSize(13);
|
||||
$text->setFillColor($colors["bg"]);
|
||||
$draw->setFillColor($colors["text"]);
|
||||
$draw->setStrokeColor($colors["transparent"]);
|
||||
|
||||
// horizontal
|
||||
$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
|
||||
$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
|
||||
$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
|
||||
|
||||
$draw->rectangle(
|
||||
$x_x - 2,
|
||||
$x_y - $label_width_x["textHeight"] + 4,
|
||||
$x_x + $label_width_x["textWidth"] + 2,
|
||||
$x_y + 3,
|
||||
);
|
||||
|
||||
// vertical
|
||||
$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
|
||||
$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
|
||||
$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
|
||||
|
||||
$draw->rectangle(
|
||||
$y_x + 4,
|
||||
$y_y - 2,
|
||||
$y_x + $label_width_y["textHeight"] + 3,
|
||||
$y_y + $label_width_y["textWidth"] + 2,
|
||||
);
|
||||
|
||||
$im->drawImage($draw);
|
||||
$im->drawImage($line);
|
||||
|
||||
// horizontal text
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$x_x,
|
||||
$x_y,
|
||||
0,
|
||||
$graph["x_label"]
|
||||
);
|
||||
|
||||
// vertical text
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$y_x + $label_width_y["textHeight"],
|
||||
$y_y + $label_width_y["textWidth"],
|
||||
270,
|
||||
$graph["y_label"]
|
||||
);
|
||||
|
||||
//
|
||||
// draw title + subtext
|
||||
//
|
||||
$text->setFontSize(20);
|
||||
$text->setFontWeight(900);
|
||||
$text->setFillColor($colors["text"]);
|
||||
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$size["x"],
|
||||
$size["y"] - 25,
|
||||
0,
|
||||
$graph["title"]
|
||||
);
|
||||
|
||||
$text->setFontSize(13);
|
||||
$text->setFontWeight(100);
|
||||
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
$size["x"],
|
||||
$size["y"] - 10,
|
||||
0,
|
||||
$graph["subtitle"]
|
||||
);
|
||||
|
||||
if($not_enough_data){
|
||||
|
||||
// add not enough data notice
|
||||
$rect = new ImagickDraw();
|
||||
$rect->setFillColor($colors["bg"]);
|
||||
$rect->setStrokeColor($colors_line[1]);
|
||||
|
||||
$rect_w = 200;
|
||||
$rect_h = 24;
|
||||
|
||||
$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
|
||||
$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
|
||||
|
||||
$rect->rectangle(
|
||||
$rect_x,
|
||||
$rect_y,
|
||||
$rect_x + $rect_w,
|
||||
$rect_y + $rect_h
|
||||
);
|
||||
|
||||
$im->drawImage($rect);
|
||||
|
||||
$label = "Not enough data";
|
||||
|
||||
$text->setFontWeight(900);
|
||||
$query = $im->queryFontMetrics($text, $label);
|
||||
|
||||
$im->annotateImage(
|
||||
$text,
|
||||
round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
|
||||
round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
|
||||
0,
|
||||
$label
|
||||
);
|
||||
}
|
||||
|
||||
return $im->getImageBlob();
|
||||
}
|
||||
}
|
||||
@@ -26,9 +26,6 @@ $payload = [
|
||||
"left" => ""
|
||||
];
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
$results = $scraper->music($get);
|
||||
|
||||
|
||||
@@ -26,9 +26,6 @@ $payload = [
|
||||
"left" => ""
|
||||
];
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
$results = $scraper->news($get);
|
||||
|
||||
|
||||
@@ -1,107 +0,0 @@
|
||||
<?php
|
||||
|
||||
class google{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("google");
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
}
|
||||
|
||||
private function get(/*$proxy, */$url, $get = []){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
// http2 bypass
|
||||
curl_setopt($curlproc, CURLOPT_HTTP_VERSION, CURL_HTTP_VERSION_2_0);
|
||||
|
||||
$headers =
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"];
|
||||
|
||||
// i dont think proxies are necessary here...
|
||||
//$this->backend->assign_proxy($curlproc, $proxy);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER, $headers);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
function resolve($id){
|
||||
|
||||
$html =
|
||||
$this->get(
|
||||
//$this->backend->get_ip(),
|
||||
"https://www.google.com/goto",
|
||||
[
|
||||
"url" => $id
|
||||
]
|
||||
);
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$a =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"a"
|
||||
);
|
||||
|
||||
if(count($a) === 0){
|
||||
|
||||
throw new Exception("Failed to find link");
|
||||
}
|
||||
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
){
|
||||
|
||||
return $link;
|
||||
}
|
||||
|
||||
throw new Exception("Got malformed link: " . $link);
|
||||
}
|
||||
}
|
||||
+4
-9
@@ -5,8 +5,7 @@ $resolver = new resolver();
|
||||
class resolver{
|
||||
|
||||
public const resolvers = [
|
||||
"sc",
|
||||
"google"
|
||||
"sc"
|
||||
];
|
||||
|
||||
public function __construct(){
|
||||
@@ -41,13 +40,9 @@ class resolver{
|
||||
$resolver = new $scraper();
|
||||
$link = $resolver->resolve($target);
|
||||
|
||||
if(
|
||||
is_string($link) &&
|
||||
!empty($link)
|
||||
){
|
||||
if(is_string($link)){
|
||||
|
||||
header("Location: {$link}");
|
||||
echo 'If your ass is not getting redirected, <a href="' . htmlspecialchars($link) . '">click here</a>.';
|
||||
}
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -59,13 +54,13 @@ class resolver{
|
||||
|
||||
header("Content-Type: text/plain");
|
||||
http_response_code(400);
|
||||
echo htmlspecialchars($message);
|
||||
echo $message;
|
||||
}
|
||||
|
||||
public function do404($message){
|
||||
|
||||
header("Content-Type: text/plain");
|
||||
http_response_code(404);
|
||||
echo htmlspecialchars($message);
|
||||
echo $message;
|
||||
}
|
||||
}
|
||||
|
||||
+25
-13
@@ -1,16 +1,28 @@
|
||||
# 44
|
||||
# 4444444 44
|
||||
# 44444444 44444 444
|
||||
# 44444444 444444 444444444
|
||||
# 44444 44444444 444444444
|
||||
# 444444444 4444444
|
||||
# 4444444444 444444
|
||||
# 4444444444444
|
||||
# 444444444444444444
|
||||
# 444444444444444
|
||||
# 44444444
|
||||
# 4444
|
||||
# 44
|
||||
# When the robots.txt is sus
|
||||
|
||||
# ⠀⠀⠀⡯⡯⡾⠝⠘⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⠀⢊⠘⡮⣣⠪⠢⡑⡌
|
||||
# ⠀⠀⠀⠟⠝⠈⠀⠀⠀⠡⠀⠠⢈⠠⢐⢠⢂⢔⣐⢄⡂⢔⠀⡁⢉⠸⢨⢑⠕⡌
|
||||
# ⠀⠀⡀⠁⠀⠀⠀⡀⢂⠡⠈⡔⣕⢮⣳⢯⣿⣻⣟⣯⣯⢷⣫⣆⡂⠀⠀⢐⠑⡌
|
||||
# ⢀⠠⠐⠈⠀⢀⢂⠢⡂⠕⡁⣝⢮⣳⢽⡽⣾⣻⣿⣯⡯⣟⣞⢾⢜⢆⠀⡀⠀⠪
|
||||
# ⣬⠂⠀⠀⢀⢂⢪⠨⢂⠥⣺⡪⣗⢗⣽⢽⡯⣿⣽⣷⢿⡽⡾⡽⣝⢎⠀⠀⠀⢡
|
||||
# ⣿⠀⠀⠀⢂⠢⢂⢥⢱⡹⣪⢞⡵⣻⡪⡯⡯⣟⡾⣿⣻⡽⣯⡻⣪⠧⠑⠀⠁⢐
|
||||
# ⣿⠀⠀⠀⠢⢑⠠⠑⠕⡝⡎⡗⡝⡎⣞⢽⡹⣕⢯⢻⠹⡹⢚⠝⡷⡽⡨⠀⠀⢔
|
||||
# ⣿⡯⠀⢈⠈⢄⠂⠂⠐⠀⠌⠠⢑⠱⡱⡱⡑⢔⠁⠀⡀⠐⠐⠐⡡⡹⣪⠀⠀⢘
|
||||
# ⣿⣽⠀⡀⡊⠀⠐⠨⠈⡁⠂⢈⠠⡱⡽⣷⡑⠁⠠⠑⠀⢉⢇⣤⢘⣪⢽⠀⢌⢎
|
||||
# ⣿⢾⠀⢌⠌⠀⡁⠢⠂⠐⡀⠀⢀⢳⢽⣽⡺⣨⢄⣑⢉⢃⢭⡲⣕⡭⣹⠠⢐⢗
|
||||
# ⣿⡗⠀⠢⠡⡱⡸⣔⢵⢱⢸⠈⠀⡪⣳⣳⢹⢜⡵⣱⢱⡱⣳⡹⣵⣻⢔⢅⢬⡷
|
||||
# ⣷⡇⡂⠡⡑⢕⢕⠕⡑⠡⢂⢊⢐⢕⡝⡮⡧⡳⣝⢴⡐⣁⠃⡫⡒⣕⢏⡮⣷⡟
|
||||
# ⣷⣻⣅⠑⢌⠢⠁⢐⠠⠑⡐⠐⠌⡪⠮⡫⠪⡪⡪⣺⢸⠰⠡⠠⠐⢱⠨⡪⡪⡰
|
||||
# ⣯⢷⣟⣇⡂⡂⡌⡀⠀⠁⡂⠅⠂⠀⡑⡄⢇⠇⢝⡨⡠⡁⢐⠠⢀⢪⡐⡜⡪⡊
|
||||
# ⣿⢽⡾⢹⡄⠕⡅⢇⠂⠑⣴⡬⣬⣬⣆⢮⣦⣷⣵⣷⡗⢃⢮⠱⡸⢰⢱⢸⢨⢌
|
||||
# ⣯⢯⣟⠸⣳⡅⠜⠔⡌⡐⠈⠻⠟⣿⢿⣿⣿⠿⡻⣃⠢⣱⡳⡱⡩⢢⠣⡃⠢⠁
|
||||
# ⡯⣟⣞⡇⡿⣽⡪⡘⡰⠨⢐⢀⠢⢢⢄⢤⣰⠼⡾⢕⢕⡵⣝⠎⢌⢪⠪⡘⡌⠀
|
||||
# ⡯⣳⠯⠚⢊⠡⡂⢂⠨⠊⠔⡑⠬⡸⣘⢬⢪⣪⡺⡼⣕⢯⢞⢕⢝⠎⢻⢼⣀⠀
|
||||
# ⠁⡂⠔⡁⡢⠣⢀⠢⠀⠅⠱⡐⡱⡘⡔⡕⡕⣲⡹⣎⡮⡏⡑⢜⢼⡱⢩⣗⣯⣟
|
||||
# ⢀⢂⢑⠀⡂⡃⠅⠊⢄⢑⠠⠑⢕⢕⢝⢮⢺⢕⢟⢮⢊⢢⢱⢄⠃⣇⣞⢞⣞⢾
|
||||
# ⢀⠢⡑⡀⢂⢊⠠⠁⡂⡐⠀⠅⡈⠪⠪⠪⠣⠫⠑⡁⢔⠕⣜⣜⢦⡰⡎⡯⡾⡽
|
||||
|
||||
User-agent: *
|
||||
Disallow:
|
||||
Host: 4get.ca
|
||||
Sitemap: https://4get.ca/sitemap
|
||||
|
||||
+2
-2
@@ -919,8 +919,8 @@ class baidu{
|
||||
$description =
|
||||
$this->fuckhtml
|
||||
->getElementsByAttributeValue(
|
||||
"data-module",
|
||||
"abstract",
|
||||
"data-sanssr-cmpt",
|
||||
"card/www-summary-1",
|
||||
$div
|
||||
);
|
||||
|
||||
|
||||
@@ -383,8 +383,6 @@ class brave{
|
||||
// load html
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
/*
|
||||
Get next page "token"
|
||||
*/
|
||||
@@ -1177,8 +1175,6 @@ class brave{
|
||||
// load html
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
// get npt
|
||||
$out["npt"] =
|
||||
$this->generatenextpagetoken(
|
||||
@@ -1274,9 +1270,6 @@ class brave{
|
||||
fclose($handle);*/
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
$json = $this->get_js();
|
||||
|
||||
foreach(
|
||||
@@ -1381,8 +1374,6 @@ class brave{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
@@ -1862,34 +1853,4 @@ class brave{
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private function detect_captcha(){
|
||||
|
||||
$title =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"title"
|
||||
);
|
||||
|
||||
if(count($title) === 0){
|
||||
|
||||
if(strtolower($this->fuckhtml->getloadedhtml()) == "this service is not available in your region"){
|
||||
|
||||
throw new Exception("Brave rangebanned the IP range");
|
||||
}
|
||||
|
||||
throw new Exception("Brave returned a malformed page");
|
||||
}
|
||||
|
||||
$title =
|
||||
strtolower(
|
||||
$this->fuckhtml
|
||||
->getTextContent($title[0])
|
||||
);
|
||||
|
||||
if($title == "human verification"){
|
||||
|
||||
throw new Exception("Brave returned a Captcha");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,847 @@
|
||||
<?php
|
||||
|
||||
class cara{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("cara");
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
|
||||
return [
|
||||
"sort" => [
|
||||
"display" => "Sort by",
|
||||
"option" => [
|
||||
"Top" => "Top",
|
||||
"MostRecent" => "Most Recent"
|
||||
]
|
||||
],
|
||||
"type" => [
|
||||
"display" => "Post type",
|
||||
"option" => [
|
||||
"any" => "Any type",
|
||||
"portfolio" => "Portfolio", // {"posts":["portfolio"]}
|
||||
"timeline" => "Timeline" // {"posts":["timeline"]}
|
||||
]
|
||||
],
|
||||
"fields" => [
|
||||
"display" => "Field/Medium",
|
||||
"option" => [
|
||||
"any" => "Any field",
|
||||
"2D" => "2D Work",
|
||||
"3D" => "3D Work",
|
||||
"3DPrinting" => "3D Printing",
|
||||
"Acrylic" => "Acrylic",
|
||||
"AlcoholMarkers" => "Alcohol Markers",
|
||||
"Animation" => "Animation",
|
||||
"Chalk" => "Chalk",
|
||||
"Charcoal" => "Charcoal",
|
||||
"Colored pencil" => "Colored pencil",
|
||||
"Conte" => "Conte",
|
||||
"Crayon" => "Crayon",
|
||||
"Digital" => "Digital",
|
||||
"Gouache" => "Gouache",
|
||||
"Ink" => "Ink",
|
||||
"MixedMedia" => "Mixed-Media",
|
||||
"Oil" => "Oil",
|
||||
"Oil-based Markers" => "Oil-based Markers",
|
||||
"Other" => "Other",
|
||||
"Pastels" => "Pastels",
|
||||
"Photography" => "Photography",
|
||||
"Sculpture" => "Sculpture",
|
||||
"Sketches" => "Sketches",
|
||||
"Tattoos" => "Tattoos",
|
||||
"Traditional" => "Traditional",
|
||||
"VFX" => "VFX",
|
||||
"Watercolor" => "Watercolor"
|
||||
]
|
||||
],
|
||||
"category" => [
|
||||
"display" => "Category",
|
||||
"option" => [
|
||||
"any" => "Any category",
|
||||
"3DScanning" => "3D Scanning",
|
||||
"Abstract" => "Abstract",
|
||||
"Adoptable" => "Adoptable",
|
||||
"Anatomy" => "Anatomy",
|
||||
"Animals" => "Animals",
|
||||
"Anime" => "Anime",
|
||||
"App" => "App",
|
||||
"ArchitecturalConcepts" => "Architectural Concepts",
|
||||
"ArchitecturalVisualization" => "Architectural Visualization",
|
||||
"AugmentedReality" => "Augmented Reality",
|
||||
"Automotive" => "Automotive",
|
||||
"BoardGameArt" => "Board Game Art",
|
||||
"BookIllustration" => "Book Illustration",
|
||||
"CardGameArt" => "Card Game Art",
|
||||
"CeramicsPottery" => "Ceramics/Pottery",
|
||||
"CharacterAnimation" => "Character Animation",
|
||||
"CharacterDesign" => "Character Design",
|
||||
"CharacterModeling" => "Character Modeling",
|
||||
"ChildrensArt" => "Children's Illustration",
|
||||
"Collectibles" => "Collectibles",
|
||||
"ColoringPage" => "Coloring Page",
|
||||
"ComicArt" => "Comic Art",
|
||||
"ConceptArt" => "Concept Art",
|
||||
"Cosplay" => "Cosplay",
|
||||
"CostumeDesign" => "Costume Design",
|
||||
"CoverArt" => "Cover Art",
|
||||
"Creatures" => "Creatures",
|
||||
"Diorama" => "Diorama",
|
||||
"EditorialIllustration" => "Editorial Illustration",
|
||||
"EmbroiderySewing" => "Embroidery/Sewing",
|
||||
"EnvironmentalConceptArt" => "Environmental Concept Art",
|
||||
"EnvironmentalConceptDesign" => "Environmental Concept Design",
|
||||
"FanArt" => "Fan Art",
|
||||
"Fantasy" => "Fantasy",
|
||||
"Fashion" => "Fashion",
|
||||
"FashionStyling" => "Fashion Styling",
|
||||
"FiberArts" => "Fiber Arts",
|
||||
"Furry" => "Furry",
|
||||
"GameArt" => "Game Art",
|
||||
"GameplayDesign" => "Gameplay Design",
|
||||
"GamesEnvironmentArt" => "Games Environment Art",
|
||||
"Gem" => "Gem",
|
||||
"GraphicDesign" => "Graphic Design",
|
||||
"Handicraft" => "Handicraft",
|
||||
"HairStyling" => "Hair Styling",
|
||||
"HardSurface" => "Hard Surface",
|
||||
"Horror" => "Horror",
|
||||
"Illustration" => "Illustration",
|
||||
"IllustrationVisualization" => "Illustration Visualization",
|
||||
"IndustrialDesign" => "Industrial Design",
|
||||
"Jewelry" => "Jewelry",
|
||||
"KnittingCrochet" => "Knitting/Crochet",
|
||||
"Landscape" => "Landscape",
|
||||
"LevelDesign" => "Level Design",
|
||||
"Lighting" => "Lighting",
|
||||
"Makeup" => "Makeup",
|
||||
"Manga" => "Manga",
|
||||
"MapsCartography" => "Maps/Cartography",
|
||||
"MattePainting" => "Matte Painting",
|
||||
"Materials" => "Materials",
|
||||
"MechanicalDesign" => "Mechanical Design",
|
||||
"Medical" => "Medical",
|
||||
"Mecha" => "Mecha",
|
||||
"MiniatureArt" => "Miniature Art",
|
||||
"MotionGraphics" => "Motion Graphics",
|
||||
"FrescoMurals" => "Fresco/Murals",
|
||||
"Natural" => "Natural",
|
||||
"Original Character" => "Original Character",
|
||||
"Overlay" => "Overlay",
|
||||
"PleinAir" => "Plein Air",
|
||||
"Photogrammetry" => "Photogrammetry",
|
||||
"PixelArt" => "Pixel Art",
|
||||
"Portraits" => "Portraits",
|
||||
"Props" => "Props",
|
||||
"ProductDesign" => "Product Design",
|
||||
"PublicDomain" => "Public Domain or Royalty Free",
|
||||
"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
|
||||
"Realism" => "Realism",
|
||||
"ScienceFiction" => "Science Fiction",
|
||||
"ScientificVisualization" => "Scientific Visualization",
|
||||
"Scripts" => "Scripts",
|
||||
"StillLife" => "Still Life",
|
||||
"Storyboards" => "Storyboards",
|
||||
"Stylized" => "Stylized",
|
||||
"Surreal" => "Surreal",
|
||||
"TechnicalArt" => "Technical Art",
|
||||
"Textures" => "Textures",
|
||||
"Tools" => "Tools",
|
||||
"Toys" => "Toys",
|
||||
"ToyPackaging" => "Toy Packaging",
|
||||
"Tutorials" => "Tutorials",
|
||||
"UIArt" => "User Interface (UI) Art",
|
||||
"UrbanSketch" => "Urban Sketch",
|
||||
"VFXforAnimation" => "VFX for Animation",
|
||||
"VFXforFilm" => "VFX for Film",
|
||||
"VFXforGames" => "VFX for Games",
|
||||
"VFXforRealTime" => "VFX for Real-Time",
|
||||
"VFXforTV" => "VFX for TV",
|
||||
"Vehicles" => "Vehicles",
|
||||
"VirtualReality" => "Virtual Reality",
|
||||
"VisualDevelopment" => "Visual Development",
|
||||
"VoxelArt" => "Voxel Art",
|
||||
"Vtubers" => "Vtubers",
|
||||
"WIP" => "WIP (Work in Progress)",
|
||||
"Web" => "Web",
|
||||
"Weapons" => "Weapons",
|
||||
"Wildlife" => "Wildlife",
|
||||
"Woodcutting" => "Woodcutting"
|
||||
]
|
||||
],
|
||||
"software" => [
|
||||
"display" => "Software",
|
||||
"option" => [
|
||||
"any" => "Any software",
|
||||
"123D" => "123D",
|
||||
"123DCatch" => "123D Catch",
|
||||
"3DBee" => "3DBee",
|
||||
"3DCoat" => "3DCoat",
|
||||
"3DCoatPrint" => "3DCoatPrint",
|
||||
"3DCoatTextura" => "3DCoatTextura",
|
||||
"3DEqualizer" => "3DEqualizer",
|
||||
"3DFZephyr" => "3DF Zephyr",
|
||||
"3Delight" => "3Delight",
|
||||
"3dpeople" => "3dpeople",
|
||||
"3dsMax" => "3ds Max",
|
||||
"3DSPaint" => "3DS Paint",
|
||||
"ACDSeeCanvas" => "ACDSee Canvas",
|
||||
"AbletonLive" => "Ableton Live",
|
||||
"Acrobat" => "Acrobat",
|
||||
"AdobeDraw" => "Adobe Draw",
|
||||
"AdobeFlash" => "Adobe Flash",
|
||||
"AdobeFresco" => "Adobe Fresco",
|
||||
"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
|
||||
"AdobeXD" => "Adobe XD",
|
||||
"AffinityDesigner" => "Affinity Designer",
|
||||
"AffinityPhoto" => "Affinity Photo",
|
||||
"AfterEffects" => "After Effects",
|
||||
"Akeytsu" => "Akeytsu",
|
||||
"Alchemy" => "Alchemy",
|
||||
"AliasDesign" => "Alias Design",
|
||||
"AlightMotion" => "Alight Motion",
|
||||
"Amadine" => "Amadine",
|
||||
"Amberlight" => "Amberlight",
|
||||
"Animate" => "Animate",
|
||||
"AnimationMaster" => "Animation:Master",
|
||||
"AnimeStudio" => "Anime Studio",
|
||||
"Apophysis" => "Apophysis",
|
||||
"ArchiCAD" => "ArchiCAD",
|
||||
"Arion" => "Arion",
|
||||
"ArionFX" => "ArionFX",
|
||||
"Arnold" => "Arnold",
|
||||
"ArtEngine" => "ArtEngine",
|
||||
"ArtFlow" => "ArtFlow",
|
||||
"ArtRage" => "ArtRage",
|
||||
"ArtstudioPro" => "Artstudio Pro",
|
||||
"Artweaver" => "Artweaver",
|
||||
"Aseprite" => "Aseprite",
|
||||
"Audition" => "Audition",
|
||||
"AutoCAD" => "AutoCAD",
|
||||
"AutodeskSketchBook" => "Autodesk SketchBook",
|
||||
"AvidMediaComposer" => "Avid Media Composer",
|
||||
"AzPainter" => "AzPainter",
|
||||
"babylonjs" => "babylon.js",
|
||||
"BalsamiqMockup" => "Balsamiq Mockup",
|
||||
"Bforartists" => "Bforartists",
|
||||
"BlackInk" => "Black Ink",
|
||||
"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
|
||||
"Blender" => "Blender",
|
||||
"Blender DeepPaint" => "Blender DeepPaint",
|
||||
"BlenderGreasePencil" => "Blender Grease Pencil",
|
||||
"Blockbench" => "Blockbench",
|
||||
"BodyPaint" => "BodyPaint",
|
||||
"Boxcutter" => "Boxcutter",
|
||||
"BraidMaker" => "Braid Maker",
|
||||
"BrickLinkStudio" => "BrickLink Studio",
|
||||
"Bridge" => "Bridge",
|
||||
"Brushifyio" => "Brushify.io",
|
||||
"C" => "C",
|
||||
"C#" => "C#",
|
||||
"C++" => "C++",
|
||||
"CACANi" => "CACANi",
|
||||
"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
|
||||
"CLO" => "CLO",
|
||||
"CRYENGINE" => "CRYENGINE",
|
||||
"Callipeg" => "Callipeg",
|
||||
"Canva" => "Canva",
|
||||
"CaptureOne" => "Capture One",
|
||||
"CartoonAnimator" => "Cartoon Animator",
|
||||
"Carveco" => "Carveco",
|
||||
"Cavalry" => "Cavalry",
|
||||
"Chaotica" => "Chaotica",
|
||||
"CharacterAnimator" => "Character Animator",
|
||||
"CharacterCreator" => "Character Creator",
|
||||
"Cinema4D" => "Cinema 4D",
|
||||
"ClarisseiFX" => "Clarisse iFX",
|
||||
"Coiffure" => "Coiffure",
|
||||
"ColorsLive" => "Colors Live",
|
||||
"Combustion" => "Combustion",
|
||||
"Construct2" => "Construct 2",
|
||||
"Core" => "Core",
|
||||
"CorelPainter" => "Corel Painter",
|
||||
"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
|
||||
"CoronaRenderer" => "Corona Renderer",
|
||||
"ProMotionNG" => "Cosmigo Pro Motion NG",
|
||||
"CrazyBump" => "CrazyBump",
|
||||
"Crocotile3D" => "Crocotile 3D",
|
||||
"Curvy3D" => "Curvy 3D",
|
||||
"Cycles4D" => "Cycles 4D",
|
||||
"Darkroom" => "Darkroom",
|
||||
"DAZStudio" => "DAZ Studio",
|
||||
"DDO" => "DDO",
|
||||
"DECIMA" => "DECIMA",
|
||||
"Darktable" => "Darktable",
|
||||
"DaVinciResolve" => "DaVinci Resolve",
|
||||
"Dimension" => "Dimension",
|
||||
"DragonBones" => "DragonBones",
|
||||
"Dragonframe" => "Dragonframe",
|
||||
"Drawpile" => "Drawpile",
|
||||
"Dreams" => "Dreams",
|
||||
"Dreamweaver" => "Dreamweaver",
|
||||
"DxOPhotoLab" => "DxO PhotoLab",
|
||||
"ECycles" => "E-Cycles",
|
||||
"EmberGen" => "EmberGen",
|
||||
"Encore" => "Encore",
|
||||
"Expresii" => "Expresii",
|
||||
"FStorm" => "FStorm",
|
||||
"FadeIn" => "FadeIn",
|
||||
"Feather3D" => "Feather 3D",
|
||||
"FiberShop" => "FiberShop",
|
||||
"Figma" => "Figma",
|
||||
"FilmoraWondershare" => "Filmora Wondershare",
|
||||
"FilterForge" => "Filter Forge",
|
||||
"FinalCutPro" => "Final Cut Pro",
|
||||
"FinalDraft" => "Final Draft",
|
||||
"finalRender" => "finalRender",
|
||||
"FireAlpaca" => "FireAlpaca",
|
||||
"Fireworks" => "Fireworks",
|
||||
"FlamePainter" => "Flame Painter",
|
||||
"Flash" => "Flash",
|
||||
"FlipaClip" => "FlipaClip",
|
||||
"FlipnoteStudio" => "Flipnote Studio",
|
||||
"Fluent" => "Fluent",
|
||||
"ForestPack" => "Forest Pack",
|
||||
"FormZ" => "Form-Z",
|
||||
"Fractorium" => "Fractorium",
|
||||
"FreeCAD" => "FreeCAD",
|
||||
"FreeHand" => "FreeHand",
|
||||
"Forger" => "Forger",
|
||||
"FrostbiteEngine" => "Frostbite Engine",
|
||||
"fSpy" => "fSpy",
|
||||
"FumeFX" => "FumeFX",
|
||||
"Fusion360" => "Fusion 360",
|
||||
"GIMP" => "GIMP",
|
||||
"GSCurveTools" => "GS CurveTools",
|
||||
"GSToolbox" => "GS Toolbox",
|
||||
"Gaea" => "Gaea",
|
||||
"GameTextures" => "Game Textures",
|
||||
"GameMakerStudio" => "GameMaker: Studio",
|
||||
"GarageFarmNET" => "GarageFarm.NET",
|
||||
"GeoGlyph" => "GeoGlyph",
|
||||
"GigapixelAl" => "Gigapixel Al",
|
||||
"Glaxnimate" => "Glaxnimate",
|
||||
"GnomePaint" => "Gnome Paint",
|
||||
"Godot" => "Godot",
|
||||
"Goxel" => "Goxel",
|
||||
"Graphite" => "Graphite",
|
||||
"Graswald" => "Graswald",
|
||||
"GravitySketch" => "Gravity Sketch",
|
||||
"GuerillaRender" => "GuerillaRender",
|
||||
"HDRLightStudio" => "HDR Light Studio",
|
||||
"HairStrandDesigner" => "Hair Strand Designer",
|
||||
"HairTGHairFur" => "HairTG - Hair & Fur",
|
||||
"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
|
||||
"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
|
||||
"Handplane" => "Handplane",
|
||||
"Hansoft" => "Hansoft",
|
||||
"HardOps" => "Hard Ops",
|
||||
"HardMesh" => "HardMesh",
|
||||
"Harmony" => "Harmony",
|
||||
"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
|
||||
"HelloPaint" => "HelloPaint",
|
||||
"HeliconFocus" => "Helicon Focus",
|
||||
"Hexels" => "Hexels",
|
||||
"HiPaint" => "HiPaint",
|
||||
"Houdini" => "Houdini",
|
||||
"HydraRenderer" => "Hydra Renderer",
|
||||
"iArtbook" => "iArtbook",
|
||||
"IbisPaint" => "ibisPaint",
|
||||
"Ideas" => "Ideas",
|
||||
"IllustStudio" => "Illust Studio",
|
||||
"Illustrator" => "Illustrator",
|
||||
"IllustratorDraw" => "Illustrator Draw",
|
||||
"InDesign" => "InDesign",
|
||||
"Inochi2D" => "Inochi2D",
|
||||
"InVision" => "InVision",
|
||||
"InVisionCraft" => "InVision Craft",
|
||||
"InfinitePainter" => "Infinite Painter",
|
||||
"Inkscape" => "Inkscape",
|
||||
"Inspirit" => "Inspirit",
|
||||
"InstaLOD" => "InstaLOD",
|
||||
"InstaMAT" => "InstaMAT",
|
||||
"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
|
||||
"InstantMeshes" => "Instant Meshes",
|
||||
"InstantTerra" => "Instant Terra",
|
||||
"Inventor" => "Inventor",
|
||||
"Iray" => "Iray",
|
||||
"JWildfire" => "JWildfire",
|
||||
"Java" => "Java",
|
||||
"Jira" => "Jira",
|
||||
"JumpPaint" => "Jump Paint by MediBang",
|
||||
"JSPaint" => "JS Paint",
|
||||
"Katana" => "Katana",
|
||||
"Keyshot" => "Keyshot",
|
||||
"KidPix" => "Kid Pix",
|
||||
"KitBash3D" => "KitBash3D",
|
||||
"Knald" => "Knald",
|
||||
"Kodon" => "Kodon",
|
||||
"KolourPaint" => "KolourPaint",
|
||||
"Krakatoa" => "Krakatoa",
|
||||
"KRESKA" => "KRESKA",
|
||||
"Krita" => "Krita",
|
||||
"LensStudio" => "Lens Studio",
|
||||
"LibreSprite" => "LibreSprite",
|
||||
"LightWave3D" => "LightWave 3D",
|
||||
"Lightroom" => "Lightroom",
|
||||
"Linearity" => "Linearity",
|
||||
"LiquiGen" => "LiquiGen",
|
||||
"Live2DCubism" => "Live2D Cubism",
|
||||
"LookatmyHair" => "Look at my Hair",
|
||||
"Lotpixel" => "Lotpixel",
|
||||
"Lumion" => "Lumion",
|
||||
"LuxRender" => "LuxRender",
|
||||
"MacPaint" => "MacPaint",
|
||||
"MagicaCSG" => "MagicaCSG",
|
||||
"MagicaVoxel" => "MagicaVoxel",
|
||||
"Magma" => "Magma",
|
||||
"MakeHuman" => "MakeHuman",
|
||||
"Malmal" => "Malmal",
|
||||
"Mandelbulb3D" => "Mandelbulb 3D",
|
||||
"Mandelbulber" => "Mandelbulber",
|
||||
"MangaStudio" => "Manga Studio",
|
||||
"Mari" => "Mari",
|
||||
"MarmosetToolbag" => "Marmoset Toolbag",
|
||||
"MarvelousDesigner" => "Marvelous Designer",
|
||||
"MasterpieceStudioPro" => "Masterpiece Studio Pro",
|
||||
"MasterpieceVR" => "MasterpieceVR",
|
||||
"Maverick" => "Maverick",
|
||||
"MaxwellRender" => "Maxwell Render",
|
||||
"Maya" => "Maya",
|
||||
"MediBangPaint" => "MediBang Paint",
|
||||
"MediumbyAdobe" => "Medium by Adobe",
|
||||
"Megascans" => "Megascans",
|
||||
"mentalray" => "mental ray",
|
||||
"MeshLab" => "MeshLab",
|
||||
"Meshroom" => "Meshroom",
|
||||
"MetaHumanCreator" => "MetaHuman Creator",
|
||||
"Metashape" => "Metashape",
|
||||
"MightyBake" => "MightyBake",
|
||||
"MikuMikuDance" => "MikuMikuDance",
|
||||
"Minecraft" => "Minecraft",
|
||||
"Mischief" => "Mischief",
|
||||
"Mixamo" => "Mixamo",
|
||||
"Mixer" => "Mixer",
|
||||
"MoI3D" => "MoI3D",
|
||||
"Mocha" => "Mocha",
|
||||
"Modo" => "Modo",
|
||||
"Moho" => "Moho",
|
||||
"MotionBuilder" => "MotionBuilder",
|
||||
"Mudbox" => "Mudbox",
|
||||
"Muse" => "Muse",
|
||||
"MSPaint" => "MS Paint",
|
||||
"MyPaint" => "MyPaint",
|
||||
"NDO" => "NDO",
|
||||
"NX" => "NX",
|
||||
"NdotCAD" => "NdotCAD",
|
||||
"NintendoNotes" => "Nintendo Notes",
|
||||
"NomadSculpt" => "Nomad Sculpt",
|
||||
"Notability" => "Notability",
|
||||
"Nuke" => "Nuke",
|
||||
"Nvil" => "Nvil",
|
||||
"OctaneRender" => "Octane Render",
|
||||
"Omniverse" => "Omniverse",
|
||||
"OmniverseCreate" => "Omniverse Create",
|
||||
"ON1PhotoRAW" => "ON1 Photo RAW",
|
||||
"Open3DEngine" => "Open 3D Engine",
|
||||
"OpenCanvas" => "OpenCanvas",
|
||||
"OpenGL" => "OpenGL",
|
||||
"OpenToonz" => "OpenToonz",
|
||||
"Ornatrix" => "Ornatrix",
|
||||
"OsciRender" => "Osci-Render",
|
||||
"OurPaint" => "Our Paint",
|
||||
"PBRMAX" => "PBRMAX",
|
||||
"PFTrack" => "PFTrack",
|
||||
"PTGui" => "PTGui",
|
||||
"Paintbrush" => "Paintbrush",
|
||||
"PaintNET" => "Paint.NET",
|
||||
"PaintShopPro" => "PaintShop Pro",
|
||||
"PaintToolSAI" => "Paint Tool SAI",
|
||||
"PaintstormStudio" => "Paintstorm Studio",
|
||||
"Paper" => "Paper",
|
||||
"Pencil2D" => "Pencil2D",
|
||||
"Penpot" => "Penpot",
|
||||
"PhoenixFD" => "Phoenix FD",
|
||||
"Phonto" => "Phonto",
|
||||
"PhotoLab2" => "PhotoLab 2",
|
||||
"Photopea" => "Photopea",
|
||||
"Photoscan" => "Photoscan",
|
||||
"Photoshop" => "Photoshop",
|
||||
"PhotoshopElements" => "Photoshop Elements",
|
||||
"PicoCAD" => "picoCAD",
|
||||
"PicoCAD2" => "picoCAD 2",
|
||||
"Pinta" => "Pinta",
|
||||
"Piskel" => "Piskel",
|
||||
"Pixilart" => "Pixilart",
|
||||
"Pixelitor" => "Pixelitor",
|
||||
"Pixelmator" => "Pixelmator",
|
||||
"Pixelorama" => "Pixelorama",
|
||||
"PixivSketch" => "pixiv Sketch",
|
||||
"Pixquare" => "Pixquare",
|
||||
"PlantCatalog" => "PlantCatalog",
|
||||
"PlantFactory" => "PlantFactory",
|
||||
"Plasticity" => "Plasticity",
|
||||
"PNGtuberPlus" => "PNGtuber Plus",
|
||||
"Poliigon" => "Poliigon",
|
||||
"Polybrush" => "Polybrush",
|
||||
"PopcornFx" => "PopcornFx",
|
||||
"Poser" => "Poser",
|
||||
"Premiere" => "Premiere",
|
||||
"PremiereElements" => "Premiere Elements",
|
||||
"PresagisCreator" => "Presagis Creator",
|
||||
"ProTools" => "Pro Tools",
|
||||
"Procreate" => "Procreate",
|
||||
"ProcreateDreams" => "Procreate Dreams",
|
||||
"Producer" => "Producer",
|
||||
"PrometheanAI" => "Promethean AI",
|
||||
"PureRef" => "PureRef",
|
||||
"Python" => "Python",
|
||||
"PyxelEdit" => "PyxelEdit",
|
||||
"QuadRemesher" => "Quad Remesher",
|
||||
"QuarkXPress" => "QuarkXPress",
|
||||
"Qubicle" => "Qubicle",
|
||||
"Quill" => "Quill",
|
||||
"QuixelBridge" => "Quixel Bridge",
|
||||
"QuixelMegascans" => "Quixel Megascans",
|
||||
"QuixelMixer" => "Quixel Mixer",
|
||||
"QuixelSuite" => "Quixel Suite",
|
||||
"R3DSWrap" => "R3DS Wrap",
|
||||
"R3DSZWRAP" => "R3DS ZWRAP",
|
||||
"RDTextures" => "RD-Textures",
|
||||
"RailClone" => "RailClone",
|
||||
"RealFlow" => "RealFlow",
|
||||
"RealisticPaintStudio" => "Realistic Paint Studio",
|
||||
"RealityCapture" => "RealityCapture",
|
||||
"RealityScan" => "RealityScan",
|
||||
"RealtimeBoard" => "Realtime Board",
|
||||
"Rebelle" => "Rebelle",
|
||||
"Redshift" => "Redshift",
|
||||
"RenderMan" => "RenderMan",
|
||||
"RenderNetwork" => "Render Network",
|
||||
"Revit" => "Revit",
|
||||
"Rhino" => "Rhino",
|
||||
"Rhinoceros" => "Rhinoceros",
|
||||
"RizomUV" => "RizomUV",
|
||||
"RoughAnimator" => "Rough Animator",
|
||||
"SamsungNotes" => "Samsung Notes",
|
||||
"SamsungPENUP" => "Samsung PENUP",
|
||||
"ScansLibrary" => "ScansLibrary",
|
||||
"Scrivener" => "Scrivener",
|
||||
"Sculpt+" => "Sculpt+",
|
||||
"Sculptris" => "Sculptris",
|
||||
"ShaveandaHaircut" => "Shave and a Haircut",
|
||||
"ShiVa3D" => "ShiVa3D",
|
||||
"Shotgun" => "Shotgun",
|
||||
"Silo" => "Silo",
|
||||
"Silugen" => "Silugen",
|
||||
"Sketch" => "Sketch",
|
||||
"SketchApp" => "Sketch App",
|
||||
"SketchBookPro" => "SketchBook Pro",
|
||||
"SketchClub" => "SketchClub",
|
||||
"SketchUp" => "SketchUp",
|
||||
"Sketchable" => "Sketchable",
|
||||
"Sketchfab" => "Sketchfab",
|
||||
"Skyshop" => "Skyshop",
|
||||
"Snapseed" => "Snapseed",
|
||||
"Snowdrop" => "Snowdrop",
|
||||
"Softimage" => "Softimage",
|
||||
"SolidWorks" => "SolidWorks",
|
||||
"SonySketch" => "Sony Sketch",
|
||||
"Soundbooth" => "Soundbooth",
|
||||
"Source2" => "Source 2",
|
||||
"SourceControl" => "Source Control",
|
||||
"SourceFilmmaker" => "Source Filmmaker",
|
||||
"SpeedTree" => "SpeedTree",
|
||||
"Speedgrade" => "Speedgrade",
|
||||
"SpeedyPainter" => "SpeedyPainter",
|
||||
"Spine2D" => "Spine 2D",
|
||||
"Spriter" => "Spriter",
|
||||
"Stingray" => "Stingray",
|
||||
"Storyboarder" => "Storyboarder",
|
||||
"StoryboardPro" => "Storyboard Pro",
|
||||
"SublimeText" => "Sublime Text",
|
||||
"Substance3DDesigner" => "Substance 3D Designer",
|
||||
"Substance3DModeler" => "Substance 3D Modeler",
|
||||
"Substance3DPainter" => "Substance 3D Painter",
|
||||
"Substance3DSampler" => "Substance 3D Sampler",
|
||||
"Substance3DStager" => "Substance 3D Stager",
|
||||
"SubstanceB2M" => "Substance B2M",
|
||||
"SweetHome3D" => "Sweet Home 3D",
|
||||
"SynthEyes" => "SynthEyes",
|
||||
"TTools" => "TTools",
|
||||
"TVPaint" => "TVPaint",
|
||||
"TVPaintAnimation" => "TVPaint Animation",
|
||||
"TayasuiSketches" => "Tayasui Sketches",
|
||||
"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
|
||||
"TayasuiSketchesPro" => "Tayasui Sketches Pro",
|
||||
"Terragen" => "Terragen",
|
||||
"Texturescom" => "Textures.com",
|
||||
"Texturingxyz" => "Texturingxyz",
|
||||
"TeyaConceptor" => "Teya Conceptor",
|
||||
"TheGrove3D" => "The Grove 3D",
|
||||
"TheaRender" => "Thea Render",
|
||||
"Threejs" => "Three.js",
|
||||
"Tiled" => "Tiled",
|
||||
"TiltBrush" => "Tilt Brush",
|
||||
"Tooll3" => "Tooll3",
|
||||
"ToonBoomHarmony" => "Toon Boom Harmony",
|
||||
"ToonBoomStudio" => "Toon Boom Studio",
|
||||
"ToonSquid" => "ToonSquid",
|
||||
"TopoGun" => "TopoGun",
|
||||
"TuxPaint" => "Tux Paint",
|
||||
"Tvori" => "Tvori",
|
||||
"Twinmotion" => "Twinmotion",
|
||||
"UNIGINEEngine" => "UNIGINE Engine",
|
||||
"UVLayout" => "UVLayout",
|
||||
"UltraFractal" => "Ultra Fractal",
|
||||
"uMake" => "uMake",
|
||||
"Unfold3D" => "Unfold 3D",
|
||||
"Unity" => "Unity",
|
||||
"UnrealEngine" => "Unreal Engine",
|
||||
"Vengi" => "vengi",
|
||||
"VRay" => "V-Ray",
|
||||
"VRED" => "VRED",
|
||||
"VTubeStudio" => "VTube Studio",
|
||||
"Vectary" => "Vectary",
|
||||
"VectorayGen" => "VectorayGen",
|
||||
"Vectorworks" => "Vectorworks",
|
||||
"VegasPro" => "Vegas Pro",
|
||||
"VisualDesigner3D" => "Visual Designer 3D",
|
||||
"VisualStudio" => "Visual Studio",
|
||||
"VRoidStudio" => "VRoid Studio",
|
||||
"Vue" => "Vue",
|
||||
"Vuforia" => "Vuforia",
|
||||
"WebGL" => "WebGL",
|
||||
"WhiteboardFox" => "Whiteboard Fox",
|
||||
"WickEditor" => "Wick Editor",
|
||||
"Wings3D" => "Wings 3D",
|
||||
"Word" => "Word",
|
||||
"WorldCreator" => "World Creator",
|
||||
"WorldMachine" => "World Machine",
|
||||
"XParticles" => "X-Particles",
|
||||
"Xfrog" => "Xfrog",
|
||||
"Xgen" => "Xgen",
|
||||
"xNormal" => "xNormal",
|
||||
"xTex" => "xTex",
|
||||
"XoliulShader" => "Xoliul Shader",
|
||||
"Yafaray" => "Yafaray",
|
||||
"Yeti" => "Yeti",
|
||||
"ZBrush" => "ZBrush",
|
||||
"ZBrushCore" => "ZBrushCore",
|
||||
"ZenBrush" => "Zen Brush"
|
||||
]
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $search){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: application/json, text/plain, */*",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
|
||||
//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
|
||||
"Referer: https://cara.app/search?q=" . urlencode($search),
|
||||
//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
"TE: trailers"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$this->backend->assign_proxy($curlproc, $proxy);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function image($get){
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$npt, $proxy] =
|
||||
$this->backend->get(
|
||||
$get["npt"],
|
||||
"images"
|
||||
);
|
||||
|
||||
$npt = json_decode($npt, true);
|
||||
}else{
|
||||
|
||||
$search = $get["s"];
|
||||
if(strlen($search) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$npt = [
|
||||
"q" => $get["s"],
|
||||
"sortBy" => $get["sort"],
|
||||
"take" => 24,
|
||||
"skip" => 0,
|
||||
"filters" => []
|
||||
];
|
||||
|
||||
// parse filters
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$npt["filters"]["posts"] = [$get["type"]];
|
||||
}
|
||||
|
||||
if($get["fields"] != "any"){
|
||||
|
||||
$npt["filters"]["fields"] = [$get["fields"]];
|
||||
}
|
||||
|
||||
if($get["category"] != "any"){
|
||||
|
||||
$npt["filters"]["categories"] = [$get["category"]];
|
||||
}
|
||||
|
||||
if($get["software"] != "any"){
|
||||
|
||||
$npt["filters"]["softwares"] = [$get["software"]];
|
||||
}
|
||||
|
||||
if($npt["filters"] == []){
|
||||
|
||||
$npt["filters"] = "{}";
|
||||
}else{
|
||||
|
||||
$npt["filters"] = json_encode($npt["filters"]);
|
||||
}
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
"image" => []
|
||||
];
|
||||
|
||||
// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://cara.app/api/search/posts",
|
||||
$npt,
|
||||
$npt["q"]
|
||||
);
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch JSON");
|
||||
}
|
||||
|
||||
$json = json_decode($json, true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
$imagecount = 0;
|
||||
foreach($json as $image){
|
||||
|
||||
if(count($image["images"]) === 0){
|
||||
|
||||
// sometimes the api returns no images for an object
|
||||
$imagecount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$cover = null;
|
||||
$sources = [];
|
||||
|
||||
foreach($image["images"] as $source){
|
||||
|
||||
if($source["isCoverImg"]){
|
||||
|
||||
$cover = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => 500,
|
||||
"height" => 500
|
||||
];
|
||||
}else{
|
||||
|
||||
$sources[] = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => null,
|
||||
"height" => null
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if($cover !== null){
|
||||
|
||||
$sources[] = $cover;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" => str_replace("\n", " ", $image["content"]),
|
||||
"source" => $sources,
|
||||
"url" => "https://cara.app/post/" . $image["id"]
|
||||
];
|
||||
|
||||
$imagecount++;
|
||||
}
|
||||
|
||||
if($imagecount === 24){
|
||||
|
||||
$npt["skip"] += 24;
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode($npt),
|
||||
"images",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function fix_url($url){
|
||||
|
||||
return
|
||||
str_replace(
|
||||
[" "],
|
||||
["%20"],
|
||||
$url
|
||||
);
|
||||
}
|
||||
}
|
||||
+50
-115
File diff suppressed because one or more lines are too long
+35
-422
@@ -9,9 +9,6 @@ class google{
|
||||
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("google");
|
||||
|
||||
$this->proc = curl_multi_init();
|
||||
$this->handles = [];
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
@@ -526,7 +523,7 @@ class google{
|
||||
config::FPLAY_EXTERNAL_ENDPOINT .
|
||||
"?url=" . urlencode($url) .
|
||||
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
|
||||
"&container=" .// urlencode($container) .
|
||||
"&container=" . urlencode($container) .
|
||||
"&proxy=" . urlencode($proxy) .
|
||||
"&url_match=" . urlencode("&sei=") .
|
||||
"&fail_url_match=" . urlencode("\/sorry\/");
|
||||
@@ -617,145 +614,6 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
private function redirect_add_url($url){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/goto\?url=([^&]+)/',
|
||||
$url,
|
||||
$slug
|
||||
) === 0
|
||||
){
|
||||
|
||||
// not a google redirect
|
||||
return $url;
|
||||
}
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, "https://www.google.com/goto?url=" . urlencode($slug[1]));
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1",
|
||||
"Priority: u=0, i"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
//$this->backend->assign_proxy($curlproc, $proxy);
|
||||
|
||||
curl_multi_add_handle($this->proc, $curlproc);
|
||||
$this->handles[] = [
|
||||
"replacement" => $url,
|
||||
"handle" => $curlproc
|
||||
];
|
||||
|
||||
return $url;
|
||||
}
|
||||
|
||||
private function resolve_urls(&$scraped_data){
|
||||
|
||||
do{
|
||||
do{
|
||||
$status = curl_multi_exec($this->proc, $active);
|
||||
}while($status === CURLM_CALL_MULTI_PERFORM);
|
||||
|
||||
if($active){
|
||||
curl_multi_select($this->proc);
|
||||
}
|
||||
|
||||
}while($active && $status === CURLM_OK);
|
||||
|
||||
//
|
||||
// if we reach this, we're done downloading garbage
|
||||
//
|
||||
|
||||
// parse redirect values & create data shit
|
||||
$keys = [];
|
||||
$replacements = [];
|
||||
|
||||
for($i=0; $i<count($this->handles); $i++){
|
||||
|
||||
$this->fuckhtml->load(
|
||||
curl_multi_getcontent(
|
||||
$this->handles[$i]["handle"]
|
||||
)
|
||||
);
|
||||
|
||||
$a =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"a"
|
||||
);
|
||||
|
||||
if(count($a) === 0){
|
||||
|
||||
throw new Exception("Failed to find link");
|
||||
}
|
||||
|
||||
$link =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$a[0]["attributes"]["href"]
|
||||
);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^(https?:|)\/\//',
|
||||
$link
|
||||
)
|
||||
){
|
||||
|
||||
$keys[] = $this->handles[$i]["replacement"];
|
||||
$replacements[] = $link;
|
||||
}
|
||||
}
|
||||
|
||||
$this->recursive_replace($keys, $replacements, $scraped_data);
|
||||
|
||||
curl_multi_close($this->proc);
|
||||
}
|
||||
|
||||
private function recursive_replace(&$keys, &$replacements, &$pointer){
|
||||
|
||||
foreach($pointer as $key => &$value){
|
||||
|
||||
if($key == "url"){
|
||||
|
||||
$search =
|
||||
array_search(
|
||||
$value,
|
||||
$keys
|
||||
);
|
||||
|
||||
if($search !== false){
|
||||
|
||||
// replace url
|
||||
$value = $replacements[$search];
|
||||
}
|
||||
|
||||
}elseif(is_array($value)){
|
||||
|
||||
$this->recursive_replace($keys, $replacements, $value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public function web($get){
|
||||
|
||||
@@ -875,8 +733,7 @@ class google{
|
||||
$container = &$data["container"];
|
||||
|
||||
//file_put_contents("scraper/google-jp.html", $html);
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
//$container = "";
|
||||
//$html = file_get_contents("scraper/google-jp.html");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -897,27 +754,9 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
}
|
||||
|
||||
// init
|
||||
$this->detect_sorry();
|
||||
$this->parsestyles();
|
||||
|
||||
$this->scrape_dimg($html);
|
||||
$this->scrape_imagearr($html);
|
||||
|
||||
@@ -1048,8 +887,6 @@ class google{
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("a");
|
||||
|
||||
$illegal = false;
|
||||
|
||||
foreach($as as $a){
|
||||
|
||||
if(!isset($a["attributes"]["href"])){ continue; }
|
||||
@@ -1060,35 +897,14 @@ class google{
|
||||
$a["attributes"]["href"]
|
||||
);
|
||||
|
||||
$link_text =
|
||||
strtolower(
|
||||
$this->fuckhtml
|
||||
->getTextContent($a)
|
||||
);
|
||||
|
||||
if(
|
||||
str_contains(
|
||||
$link_text,
|
||||
"complaint"
|
||||
) ||
|
||||
str_contains(
|
||||
$link_text,
|
||||
"request"
|
||||
preg_match(
|
||||
'/^https?:\/\/lumendatabase\.org/',
|
||||
$link
|
||||
)
|
||||
){
|
||||
|
||||
if(
|
||||
str_contains(
|
||||
$link_text,
|
||||
"read more about the request"
|
||||
)
|
||||
){
|
||||
|
||||
$illegal = true;
|
||||
}
|
||||
|
||||
// sometimes the link is an encrypted redirect, handle redirection securely
|
||||
$notices[] = $this->redirect_add_url($link);
|
||||
$notices[] = $link;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1099,24 +915,13 @@ class google{
|
||||
|
||||
$i = 0;
|
||||
$c = count($notices);
|
||||
|
||||
if($illegal === false){
|
||||
|
||||
$notices_dom = [
|
||||
[
|
||||
"type" => "text",
|
||||
"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
|
||||
]
|
||||
];
|
||||
}else{
|
||||
|
||||
$notices_dom = [
|
||||
[
|
||||
"type" => "text",
|
||||
"value" => "Google removed results from this page because they were illegal. You can view the takedown requests here: "
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
$notices_dom = [
|
||||
[
|
||||
"type" => "text",
|
||||
"value" => "Google removed results from this page, but you can get them anyways by consulting these takedown notices: "
|
||||
]
|
||||
];
|
||||
|
||||
foreach($notices as $n){
|
||||
|
||||
@@ -1426,39 +1231,10 @@ class google{
|
||||
|
||||
$sublinks = [];
|
||||
|
||||
// get sublinks (rare)
|
||||
$as =
|
||||
$this->fuckhtml
|
||||
->getElementsByClassName(
|
||||
$this->getstyle([
|
||||
"padding-right" => "6px",
|
||||
"margin-top" => "4px",
|
||||
"display" => "inline-block"
|
||||
]),
|
||||
"a"
|
||||
);
|
||||
|
||||
foreach($as as $sublink){
|
||||
|
||||
$sublink_url =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$sublink["attributes"]["href"]
|
||||
);
|
||||
|
||||
$sublinks[] = [
|
||||
"title" =>
|
||||
$this->fuckhtml
|
||||
->getTextContent($sublink),
|
||||
"date" => null,
|
||||
"url" => $this->redirect_add_url($sublink_url)
|
||||
];
|
||||
}
|
||||
|
||||
$out["web"][] = [
|
||||
"title" => $title,
|
||||
"description" => $description,
|
||||
"url" => $this->redirect_add_url($url),
|
||||
"url" => $url,
|
||||
"date" => $date,
|
||||
"type" => "web",
|
||||
"thumb" => $thumb,
|
||||
@@ -1467,8 +1243,6 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -1607,28 +1381,7 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
$this->detect_sorry();
|
||||
|
||||
// get javascript images
|
||||
$this->scrape_imagearr($html);
|
||||
@@ -1679,11 +1432,9 @@ class google{
|
||||
),
|
||||
"source" => $source,
|
||||
"url" =>
|
||||
$this->redirect_add_url(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$div["attributes"]["data-lpage"]
|
||||
)
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$div["attributes"]["data-lpage"]
|
||||
)
|
||||
];
|
||||
}
|
||||
@@ -1708,8 +1459,6 @@ class google{
|
||||
);
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -1794,7 +1543,8 @@ class google{
|
||||
implode(",", $tbs);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
$data = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
@@ -1805,8 +1555,6 @@ class google{
|
||||
$html = &$data["data"];
|
||||
$container = &$data["container"];
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
if(!isset($params["start"])){
|
||||
|
||||
$params["start"] = 0;
|
||||
@@ -1815,32 +1563,13 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
|
||||
//
|
||||
// Parse web video page
|
||||
//
|
||||
$this->detect_sorry();
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// get javascript images
|
||||
$this->scrape_dimg($html);
|
||||
@@ -1849,29 +1578,6 @@ class google{
|
||||
|
||||
$params["c"] = $container;
|
||||
|
||||
// get encoded links
|
||||
$encoded_links =
|
||||
explode(
|
||||
"function(){var m={",
|
||||
$html,
|
||||
2
|
||||
);
|
||||
|
||||
if(count($encoded_links) === 2){
|
||||
|
||||
$encoded_links =
|
||||
json_decode(
|
||||
$this->fuckhtml
|
||||
->extract_json(
|
||||
"{" . $encoded_links[1]
|
||||
),
|
||||
true
|
||||
);
|
||||
}else{
|
||||
|
||||
$encoded_links = [];
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" =>
|
||||
@@ -2050,50 +1756,6 @@ class google{
|
||||
}
|
||||
}
|
||||
|
||||
$url =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$url[0]["attributes"]["href"]
|
||||
);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/goto\?url/',
|
||||
$url
|
||||
)
|
||||
){
|
||||
|
||||
// encrypted url detected
|
||||
// get reference div
|
||||
$refdivs =
|
||||
$this->fuckhtml
|
||||
->getElementsByAttributeName(
|
||||
"id",
|
||||
"div"
|
||||
);
|
||||
|
||||
foreach($refdivs as $r){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^atritem-/',
|
||||
$r["attributes"]["id"]
|
||||
) &&
|
||||
isset($r["attributes"]["jsdata"])
|
||||
){
|
||||
|
||||
$ref = explode(";", $r["attributes"]["jsdata"]);
|
||||
$ref = $ref[count($ref) - 1];
|
||||
|
||||
if(isset($encoded_links[$ref][32][11][7])){
|
||||
|
||||
$url = $encoded_links[$ref][32][11][7];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$out["video"][] = [
|
||||
"title" =>
|
||||
$this->titledots(
|
||||
@@ -2112,12 +1774,14 @@ class google{
|
||||
"duration" => $duration,
|
||||
"views" => null,
|
||||
"thumb" => $thumb,
|
||||
"url" => $this->redirect_add_url($url)
|
||||
"url" =>
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$url[0]["attributes"]["href"]
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -2223,32 +1887,14 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse styles
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
$this->detect_sorry();
|
||||
|
||||
// get images
|
||||
$this->scrape_dimg($html);
|
||||
|
||||
// parse styles
|
||||
$this->parsestyles();
|
||||
|
||||
$center_col =
|
||||
$this->fuckhtml
|
||||
->getElementById(
|
||||
@@ -2482,8 +2128,6 @@ class google{
|
||||
];
|
||||
}
|
||||
|
||||
$this->resolve_urls($out);
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
@@ -3009,7 +2653,6 @@ class google{
|
||||
|
||||
private function detect_sorry(){
|
||||
|
||||
// detect recaptcha
|
||||
$captcha_form =
|
||||
$this->fuckhtml
|
||||
->getElementById(
|
||||
@@ -3039,35 +2682,5 @@ class google{
|
||||
|
||||
throw new Exception("Google returned a JS challenge");
|
||||
}
|
||||
|
||||
// bypass soft captcha
|
||||
// "Verifying your request
|
||||
// You'll be able to continue in a few seconds. Don't refresh this page."
|
||||
$scripts =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"script"
|
||||
);
|
||||
|
||||
foreach($scripts as $sc){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
|
||||
$sc["innerHTML"],
|
||||
$matches
|
||||
)
|
||||
){
|
||||
|
||||
sleep(10); // gotta sleep to not trigger captcha. blame google, not me
|
||||
|
||||
return [
|
||||
"sei" => $matches[1],
|
||||
"timeout" => (int)$matches[2] // set to 10000 usually
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return false; // no need to bypass
|
||||
}
|
||||
}
|
||||
|
||||
+49
-191
@@ -16,7 +16,18 @@ class mojeek{
|
||||
|
||||
case "web":
|
||||
return [
|
||||
"lang" => [ // lb
|
||||
"focus" => [
|
||||
"display" => "Focus",
|
||||
"option" => [
|
||||
"any" => "No focus",
|
||||
"blogs" => "Blogs",
|
||||
"Dictionary" => "Dictionary",
|
||||
"Recipes" => "Recipes",
|
||||
"Time" => "Time",
|
||||
"Weather" => "Weather"
|
||||
]
|
||||
],
|
||||
"lang" => [
|
||||
"display" => "Language",
|
||||
"option" => [
|
||||
"any" => "Any language",
|
||||
@@ -75,7 +86,7 @@ class mojeek{
|
||||
"zu" => "Zulu"
|
||||
]
|
||||
],
|
||||
"country" => [ // rb
|
||||
"country" => [
|
||||
"display" => "Country",
|
||||
"option" => [
|
||||
"any" => "No location bias",
|
||||
@@ -329,21 +340,28 @@ class mojeek{
|
||||
"zm" => "Zambia",
|
||||
"zw" => "Zimbabwe"
|
||||
]
|
||||
],
|
||||
"sort" => [
|
||||
"display" => "Sort by",
|
||||
],/*
|
||||
"region" => [
|
||||
"display" => "Region",
|
||||
"option" => [
|
||||
"relevance" => "Relevance",
|
||||
"date" => "Date" // datewr=100
|
||||
"any" => "Any region",
|
||||
"eu" => "European Union",
|
||||
"de" => "Germany",
|
||||
"fr" => "France",
|
||||
"uk" => "United Kingdom"
|
||||
]
|
||||
],*/
|
||||
"domain" => [
|
||||
"display" => "Results per domain",
|
||||
"option" => [
|
||||
"1" => "1 result",
|
||||
"2" => "2 results",
|
||||
"3" => "3 results",
|
||||
"4" => "4 results",
|
||||
"5" => "5 results",
|
||||
"10" => "10 results",
|
||||
"0" => "Unlimited",
|
||||
]
|
||||
],
|
||||
"newer" => [
|
||||
"display" => "Newer than",
|
||||
"option" => "_DATE"
|
||||
],
|
||||
"older" => [
|
||||
"display" => "Older than",
|
||||
"option" => "_DATE"
|
||||
]
|
||||
];
|
||||
break;
|
||||
@@ -353,29 +371,21 @@ class mojeek{
|
||||
}
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $is_api = false){
|
||||
private function get($proxy, $url, $get = []){
|
||||
|
||||
if($is_api){
|
||||
|
||||
$headers = [
|
||||
"Accept: application/json"
|
||||
];
|
||||
}else{
|
||||
|
||||
$headers = [
|
||||
"User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
];
|
||||
}
|
||||
$headers = [
|
||||
"User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
];
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
@@ -410,154 +420,6 @@ class mojeek{
|
||||
|
||||
public function web($get){
|
||||
|
||||
return $this->web_api($get);
|
||||
}
|
||||
|
||||
public function web_api($get){
|
||||
|
||||
$key = $this->backend->get_key();
|
||||
$proxy = $this->backend->get_ip($key["increment"]);
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$params, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
$params = json_decode($params, true);
|
||||
$params["api_key"] = $key["key"];
|
||||
|
||||
}else{
|
||||
|
||||
if(strlen($get["s"]) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$params = [
|
||||
"api_key" => $key["key"],
|
||||
"q" => $get["s"],
|
||||
"date" => 1,
|
||||
"tlen" => 127, // title length
|
||||
"dlen" => 511, // description length
|
||||
"fmt" => "json"
|
||||
];
|
||||
|
||||
if($get["lang"] != "any"){
|
||||
|
||||
$params["lb"] = $get["lang"];
|
||||
}
|
||||
|
||||
if($get["country"] != "any"){
|
||||
|
||||
$params["rb"] = $get["country"];
|
||||
}
|
||||
|
||||
if($get["sort"] == "date"){
|
||||
|
||||
$params["datewr"] = 100;
|
||||
}
|
||||
|
||||
if($get["newer"] !== false){
|
||||
|
||||
$params["since"] = date("Ymd", $get["newer"]);
|
||||
}
|
||||
|
||||
if($get["older"] !== false){
|
||||
|
||||
$params["before"] = date("Ymd", $get["before"]);
|
||||
}
|
||||
}
|
||||
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://api.mojeek.com/search",
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to get JSON");
|
||||
}
|
||||
|
||||
$json = json_decode($json, true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"spelling" => [
|
||||
"type" => "no_correction",
|
||||
"using" => null,
|
||||
"correction" => null
|
||||
],
|
||||
"npt" => null,
|
||||
"answer" => [],
|
||||
"web" => [],
|
||||
"image" => [],
|
||||
"video" => [],
|
||||
"news" => [],
|
||||
"related" => []
|
||||
];
|
||||
|
||||
if(!isset($json["response"]["status"])){
|
||||
|
||||
throw new Exception("API did not return a status field");
|
||||
}
|
||||
|
||||
if($json["response"]["status"] != "OK"){
|
||||
|
||||
throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
|
||||
}
|
||||
|
||||
if(!isset($json["response"]["results"])){
|
||||
|
||||
throw new Exception("Mojeek API did not return a results object");
|
||||
}
|
||||
|
||||
foreach($json["response"]["results"] as $entry){
|
||||
|
||||
$out["web"][] = [
|
||||
"title" => html_entity_decode($entry["title"]),
|
||||
"description" => html_entity_decode($entry["desc"]),
|
||||
"url" => $entry["url"],
|
||||
"date" => $entry["timestamp"],
|
||||
"type" => "web",
|
||||
"thumb" => [
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"sublink" => [],
|
||||
"table" => []
|
||||
];
|
||||
}
|
||||
|
||||
// get npt
|
||||
if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
|
||||
|
||||
unset($params["api_key"]);
|
||||
if(!isset($params["s"])){
|
||||
|
||||
$params["s"] = 11;
|
||||
}else{
|
||||
|
||||
$params["s"] = $params["s"] + 10;
|
||||
}
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode($params),
|
||||
"web",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function web_browser($get){
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$token, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
@@ -595,6 +457,7 @@ class mojeek{
|
||||
"tn" => 7, // number of news results/page
|
||||
"date" => 1, // show date
|
||||
"tlen" => 128, // max length of title
|
||||
//"dlen" => 511, // max length of description
|
||||
//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
|
||||
];
|
||||
|
||||
@@ -1137,12 +1000,7 @@ class mojeek{
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function news(){
|
||||
|
||||
throw new Exception("Currently unsupported");
|
||||
}
|
||||
|
||||
public function news_browser($get){
|
||||
public function news($get){
|
||||
|
||||
$search = $get["s"];
|
||||
|
||||
|
||||
+1102
File diff suppressed because it is too large
Load Diff
+153
-396
@@ -9,10 +9,6 @@ class startpage{
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
|
||||
include "lib/anubis.php";
|
||||
$this->anubis = new anubis();
|
||||
$this->anubis_error = "all gucci";
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
@@ -248,7 +244,7 @@ class startpage{
|
||||
}
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
|
||||
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
@@ -263,27 +259,6 @@ class startpage{
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
if($return_cookies){
|
||||
|
||||
$cookies = [];
|
||||
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
|
||||
|
||||
$length = strlen($header);
|
||||
|
||||
$header = explode(":", $header, 2);
|
||||
|
||||
if(trim(strtolower($header[0])) == "set-cookie"){
|
||||
|
||||
$cookie_tmp = explode("=", trim($header[1]), 2);
|
||||
|
||||
$cookies[trim($cookie_tmp[0])] =
|
||||
explode(";", $cookie_tmp[1], 2)[0];
|
||||
}
|
||||
|
||||
return $length;
|
||||
});
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
// http2 bypass
|
||||
@@ -291,15 +266,6 @@ class startpage{
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
|
||||
$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
|
||||
|
||||
foreach($passed_cookies as $name => $val){
|
||||
|
||||
$header_cookies[] = "$name=$val";
|
||||
}
|
||||
|
||||
$header_cookies = rtrim(implode("; ", $header_cookies));
|
||||
|
||||
if($is_xhr === true){
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
@@ -313,7 +279,7 @@ class startpage{
|
||||
"Origin: https://www.startpage.com/",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
@@ -332,7 +298,7 @@ class startpage{
|
||||
"Content-Length: " . strlen($get),
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
@@ -350,7 +316,7 @@ class startpage{
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
@@ -376,15 +342,6 @@ class startpage{
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
|
||||
if($return_cookies){
|
||||
|
||||
return [
|
||||
"cookies" => $cookies,
|
||||
"data" => $data
|
||||
];
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
@@ -392,22 +349,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -449,11 +398,7 @@ class startpage{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -461,41 +406,12 @@ class startpage{
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detect captcha even if we passed anubis challenge
|
||||
$this->detect_captcha($html);
|
||||
|
||||
//echo htmlspecialchars($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
|
||||
@@ -504,14 +420,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to grep JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
//print_r($json);
|
||||
@@ -533,7 +449,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "web", $proxy);
|
||||
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
|
||||
@@ -742,22 +658,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "images");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -772,102 +680,76 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "images",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["size"] != "any"){
|
||||
|
||||
if(
|
||||
$get["size"] == "Small" ||
|
||||
$get["size"] == "Medium" ||
|
||||
$get["size"] == "Large" ||
|
||||
$get["size"] == "Wallpaper"
|
||||
){
|
||||
|
||||
$params["flimgsize"] = $get["size"];
|
||||
}else{
|
||||
|
||||
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["color"] != "any"){
|
||||
|
||||
if($get["color"] == "color"){
|
||||
|
||||
$params["flimgcolor"] = "ic:color";
|
||||
}elseif($get["color"] == "bnw"){
|
||||
|
||||
$params["flimgcolor"] = "ic:gray";
|
||||
}else{
|
||||
|
||||
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$params["flimgtype"] = $get["type"];
|
||||
}
|
||||
|
||||
if($get["license"] != "any"){
|
||||
|
||||
$params["flimglicense"] = $get["license"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "images",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["size"] != "any"){
|
||||
|
||||
if(
|
||||
$get["size"] == "Small" ||
|
||||
$get["size"] == "Medium" ||
|
||||
$get["size"] == "Large" ||
|
||||
$get["size"] == "Wallpaper"
|
||||
){
|
||||
|
||||
$params["flimgsize"] = $get["size"];
|
||||
}else{
|
||||
|
||||
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["color"] != "any"){
|
||||
|
||||
if($get["color"] == "color"){
|
||||
|
||||
$params["flimgcolor"] = "ic:color";
|
||||
}elseif($get["color"] == "bnw"){
|
||||
|
||||
$params["flimgcolor"] = "ic:gray";
|
||||
}else{
|
||||
|
||||
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$params["flimgtype"] = $get["type"];
|
||||
}
|
||||
|
||||
if($get["license"] != "any"){
|
||||
|
||||
$params["flimglicense"] = $get["license"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -887,18 +769,18 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to grep JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "images", $proxy);
|
||||
|
||||
// get images
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
@@ -937,22 +819,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "videos");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -967,72 +841,46 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "video",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["sort"] != "relevance"){
|
||||
|
||||
$params["sort_by"] = $get["sort"];
|
||||
}
|
||||
|
||||
if($get["duration"] != "any"){
|
||||
|
||||
$params["with_duration"] = $get["duration"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "video",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["sort"] != "relevance"){
|
||||
|
||||
$params["sort_by"] = $get["sort"];
|
||||
}
|
||||
|
||||
if($get["duration"] != "any"){
|
||||
|
||||
$params["with_duration"] = $get["duration"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1046,14 +894,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to get JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -1067,7 +915,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "video", $proxy);
|
||||
|
||||
// get results
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
@@ -1123,22 +971,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "news");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -1153,67 +993,41 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "news",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["time"] != "any"){
|
||||
|
||||
$params["with_date"] = $get["time"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "news",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["time"] != "any"){
|
||||
|
||||
$params["with_date"] = $get["time"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1227,14 +1041,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to get JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -1244,7 +1058,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "news", $proxy);
|
||||
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
|
||||
@@ -1287,7 +1101,7 @@ class startpage{
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function parse_npt($json, $pagetype, $proxy, $cookies){
|
||||
private function parse_npt($json, $pagetype, $proxy){
|
||||
|
||||
foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
|
||||
|
||||
@@ -1295,27 +1109,22 @@ class startpage{
|
||||
|
||||
return
|
||||
$this->backend->store(
|
||||
json_encode([
|
||||
"query" =>
|
||||
http_build_query(
|
||||
[
|
||||
"lui" => "english",
|
||||
"language" => "english",
|
||||
"sc" => $json["render"]["search_sc"],
|
||||
"t" => "device",
|
||||
"cat" => $pagetype,
|
||||
"segment" => "startpage.udog", // "original" sometimes
|
||||
"abd" => "0",
|
||||
"abe" => "0",
|
||||
"query" => rawurldecode($json["render"]["query"]),
|
||||
"page" => $page["number"],
|
||||
"qsr" => "all",
|
||||
"qadf" => "none", // @ todo fix (??)
|
||||
...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
|
||||
]
|
||||
),
|
||||
"cookies" => $cookies
|
||||
]),
|
||||
http_build_query(
|
||||
[
|
||||
"lui" => "english",
|
||||
"language" => "english",
|
||||
"sc" => $json["render"]["search_sc"],
|
||||
"t" => "device",
|
||||
"cat" => $pagetype,
|
||||
"segment" => "organic",
|
||||
"abd" => "0",
|
||||
"abe" => "0",
|
||||
"query" => rawurldecode($json["render"]["query"]),
|
||||
"page" => $page["number"],
|
||||
"qsr" => "all",
|
||||
"qadf" => "none" // @ todo fix (??)
|
||||
]
|
||||
),
|
||||
$pagetype,
|
||||
$proxy
|
||||
);
|
||||
@@ -1461,56 +1270,4 @@ class startpage{
|
||||
throw new Exception("Startpage redirected the scraper to an unhandled page");
|
||||
}
|
||||
}
|
||||
|
||||
private function solve_anubis($proxy, $html){
|
||||
|
||||
//
|
||||
// solve anubis challenge, if theres one
|
||||
//
|
||||
$res = null;
|
||||
$cookies = [];
|
||||
|
||||
try{
|
||||
$res = $this->anubis->scrape($html["data"]);
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
// do nothang
|
||||
$this->anubis_error = $error->getMessage();
|
||||
}
|
||||
|
||||
if($res !== null){
|
||||
|
||||
// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
|
||||
// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
|
||||
// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
|
||||
// &nonce=3098
|
||||
// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
|
||||
// &elapsedTime=111
|
||||
|
||||
$anubis_response =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
|
||||
[
|
||||
"id" => $res["id"],
|
||||
"response" => $res["response"],
|
||||
"nonce" => $res["nonce"],
|
||||
"redir" => "https://www.startpage.com/sp/search",
|
||||
"elapsedTime" => $res["elapsedTime"]
|
||||
],
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
$html["cookies"]
|
||||
);
|
||||
|
||||
// re-fetch search page AGAIN with the anubis bullshit
|
||||
$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
|
||||
|
||||
return $cookies;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+941
-653
File diff suppressed because it is too large
Load Diff
+30
-2
@@ -87,6 +87,10 @@ $settings = [
|
||||
"value" => "kagi",
|
||||
"text" => "Kagi"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
@@ -141,6 +145,14 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "ghostery",
|
||||
"text" => "Ghostery"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
@@ -219,6 +231,10 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "naver",
|
||||
"text" => "Naver"
|
||||
@@ -235,6 +251,10 @@ $settings = [
|
||||
"value" => "pinterest",
|
||||
"text" => "Pinterest"
|
||||
],
|
||||
[
|
||||
"value" => "cara",
|
||||
"text" => "Cara"
|
||||
],
|
||||
[
|
||||
"value" => "flickr",
|
||||
"text" => "Flickr"
|
||||
@@ -309,6 +329,10 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "naver",
|
||||
"text" => "Naver"
|
||||
@@ -355,10 +379,14 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
/*[
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "mojeek",
|
||||
"text" => "Mojeek"
|
||||
],*/
|
||||
],
|
||||
[
|
||||
"value" => "baidu",
|
||||
"text" => "Baidu"
|
||||
|
||||
@@ -22,7 +22,7 @@ var list = [];
|
||||
var pinged_list = [];
|
||||
var reqs = 0;
|
||||
var errors = 0;
|
||||
var sort = 12;
|
||||
var sort = 6; // highest version first
|
||||
|
||||
// check for instance redirect stuff
|
||||
var redir = [];
|
||||
@@ -68,8 +68,8 @@ table.innerHTML =
|
||||
'<th class="extend">Server</th>' +
|
||||
'<th>Address</th>' +
|
||||
'<th>Bot protection</th>' +
|
||||
'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
|
||||
'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
|
||||
'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
|
||||
'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
|
||||
'<th>API</th>' +
|
||||
'<th><div class="arrow up"></div>Version</th>' +
|
||||
'</tr>' +
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
+3
-12
@@ -1,18 +1,9 @@
|
||||
<a href="/" class="link">< Go back</a>
|
||||
|
||||
<h1>Donate to the project</h1>
|
||||
This project takes up most of my free time and costs (a lot of) money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
|
||||
This project takes up most of my free time and costs money to run. If you feel like this project is worthy of a donation, please do so using the resources on this page!<br><br>
|
||||
|
||||
To run smoothly, 4get requires:
|
||||
<ul>
|
||||
<li>$350/year for server hosting at OVH</li>
|
||||
<li>$60-80/month for proxies</li>
|
||||
<li>$54/month for home internet</li>
|
||||
</ul>
|
||||
|
||||
I also bought a dedicated machine for browser rendering hosted at home coming at around $380.<br><br>
|
||||
|
||||
According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> currently serves between <a href="https://4get.ca/ami4get">2000-2500 users every day</a>, and serves around <a href="https://4get.ca/api/v2/graph">20,000-22,000 searches</a>!
|
||||
To run smoothly, 4get requires 20$/month for the server fees and 15-25$ for residential proxies. I also have plans to build my own index to provide better search results; funds raised here will go directly towards better hardware for these purposes. According to the number of captchas solved, <a href="https://4get.ca">4get.ca</a> serves between 800-1400 users every day, or around 7000-10000 searches!
|
||||
|
||||
<h2>Ko-fi</h2>
|
||||
This is the most convenient way to donate. Supports PayPal.
|
||||
@@ -26,4 +17,4 @@ Due to popular demand, I have added an XMR addy. I'm planning to reward users wh
|
||||
<li><a href="monero:85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB?tx_description=4get+donation">85G95fXXfuh72oQd9LmdMHNqDTqsUPKkLCBEYE7XnnMLPvBfkdnan2FVTmVAwT4JEuXsRL7xPgqzuV2YdbFkWmxK2FgGBmB</a></li>
|
||||
</ul>
|
||||
|
||||
I'm immensely grateful for every donation. Thank you all for your generous support!!
|
||||
Thank you all for your generous support!!
|
||||
|
||||
@@ -15,7 +15,7 @@
|
||||
<div class="navigation">
|
||||
<a href="/">Home</a>
|
||||
<a href="/settings">Settings</a>
|
||||
<a href="https://bsky.app/profile/4getisdown.bsky.social" target="_BLANK">News</a>
|
||||
<a href="https://git.lolcat.ca/lolcat/4get_news" target="_BLANK">News</a>
|
||||
</div>
|
||||
<form method="GET" autocomplete="off">
|
||||
<div class="searchbox">
|
||||
|
||||
+1
-1
@@ -28,7 +28,7 @@
|
||||
<div class="autocomplete"></div>
|
||||
</div>
|
||||
</form>
|
||||
<a href="settings">Settings</a> • <a href="instances">Instances</a> • <a href="https://bsky.app/profile/4getisdown.bsky.social">News</a> • <a href="api.txt">API</a> • <a href="about">About</a> • <a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
|
||||
<a href="settings">Settings</a> • <a href="instances">Instances</a> • <a href="https://git.lolcat.ca/lolcat/4get_news">News</a> • <a href="api.txt">API</a> • <a href="about">About</a> • <a href="https://git.lolcat.ca/lolcat/4get">Source</a><br>
|
||||
<a href="/donate"><b>Donate to the project</b></a>
|
||||
<div class="subtext">
|
||||
<a href="https://4get.ca">Clearnet</a> • <a href="http://4getwebfrq5zr4sxugk6htxvawqehxtdgjrbcn2oslllcol2vepa23yd.onion">Tor</a> • <a href="https://lolcat.ca">Report a problem</a><br>
|
||||
|
||||
@@ -26,9 +26,6 @@ $payload = [
|
||||
"left" => ""
|
||||
];
|
||||
|
||||
// increment request count
|
||||
$frontend->increment_real_reqs($get["scraper"]);
|
||||
|
||||
try{
|
||||
$results = $scraper->video($get);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user