Compare commits
2
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
0efbb81d23 | ||
|
|
4b83c47637 |
@@ -26,3 +26,4 @@ data/captcha/minecraft/
|
||||
!banner/*default*
|
||||
scraper/curlie.html
|
||||
icons/*
|
||||
node_modules/
|
||||
|
||||
@@ -37,20 +37,21 @@ tl;dr 4get is the best way to browse for shit.
|
||||
| Brave | Yandex | Vimeo | Brave | Swisscows | DuckDuckGo |
|
||||
| Yandex | Brave | Sepia Search | Google | | Yandex |
|
||||
| Google | Google | DuckDuckGo | Yahoo! JAPAN | | Google |
|
||||
| Google API | Google API | Brave | Startpage | | Qwant |
|
||||
| Google CSE | Google CSE | Yandex | Baidu | | Marginalia |
|
||||
| Yahoo! JAPAN | Yahoo! JAPAN | Google | | | YouTube |
|
||||
| Startpage | Startpage | Yahoo! JAPAN | | | SoundCloud |
|
||||
| Yep | Naver | Startpage | | | Startpage |
|
||||
| Mwmbl | Baidu | Naver | | | Kagi |
|
||||
| Mojeek | Solofield | Baidu | | | |
|
||||
| Naver | Pinterest | Cốc Cốc | | | |
|
||||
| Baidu | Flickr | Purili | | | |
|
||||
| Cốc Cốc | Pexels | Solofield | | | |
|
||||
| Google API | Google API | Brave | Startpage | | Startpage |
|
||||
| Google CSE | Google CSE | Yandex | Qwant | | Kagi |
|
||||
| Yahoo! JAPAN | Yahoo! JAPAN | Google | Mojeek | | Qwant |
|
||||
| Startpage | Startpage | Yahoo! JAPAN | Baidu | | Ghostery |
|
||||
| Qwant | Qwant | Startpage | | | Yep |
|
||||
| Ghostery | Baidu | Qwant | | | Marginalia |
|
||||
| Yep | Solofield | Baidu | | | YouTube |
|
||||
| Mwmbl | Pinterest | Coc Coc | | | SoundCloud |
|
||||
| Mojeek | Cara | Solofield | | | |
|
||||
| Baidu | Flickr | | | | |
|
||||
| Coc Coc | Pexels | | | | |
|
||||
| Solofield | Pixabay | | | | |
|
||||
| Marginalia | Unsplash | | | | |
|
||||
| Purili | 500px | | | | |
|
||||
| wiby | VSCO | | | | |
|
||||
| wiby | 500px | | | | |
|
||||
| | VSCO | | | | |
|
||||
| | Imgur | | | | |
|
||||
| | FindThatMeme | | | | |
|
||||
|
||||
|
||||
+3
-1
@@ -15,11 +15,13 @@ class autocomplete{
|
||||
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
|
||||
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
|
||||
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
|
||||
"yep" => "https://api.yep.com/ac/?query={searchTerms}",
|
||||
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
|
||||
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
|
||||
"sc" => "",
|
||||
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
|
||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
|
||||
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
|
||||
"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
|
||||
];
|
||||
|
||||
/*
|
||||
|
||||
@@ -152,8 +152,6 @@ This should expose a webservice located at `http://localhost:3000`. I highly rec
|
||||
|
||||
If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
|
||||
|
||||
**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
|
||||
|
||||
## Setting up 4get
|
||||
Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
|
||||
|
||||
@@ -165,7 +163,3 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
|
||||
**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
|
||||
|
||||
If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
|
||||
|
||||
In case it isn't clear, here is a visual representation of what you want to achieve:
|
||||
|
||||

|
||||
|
||||
@@ -150,3 +150,7 @@ services:
|
||||
##### Tor
|
||||
|
||||
You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
|
||||
|
||||
##### 4play
|
||||
|
||||
You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)
|
||||
|
||||
@@ -0,0 +1,48 @@
|
||||
### Running 4get and 4play via docker guide
|
||||
|
||||
This setup will run both 4get and the 4play sidecar
|
||||
|
||||
# Docker compose
|
||||
|
||||
```
|
||||
# docker-compose.yaml
|
||||
|
||||
services:
|
||||
fourget:
|
||||
image: luuul/4get:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- FOURGET_PROTO=http
|
||||
- FOURGET_SERVER_NAME=4get.ca
|
||||
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
|
||||
- FOURGET_FPLAY_PASSWORD=cnc
|
||||
- FOURGET_FPLAY_TIMEOUT=30000
|
||||
- FOURGET_FPLAY_ENABLE_API=true
|
||||
ports:
|
||||
- "80:80"
|
||||
fourplay:
|
||||
image: luuul/4get-4play:latest
|
||||
environment:
|
||||
- FOURPLAY_PORT=3030
|
||||
- FOURPLAY_PASSWORD=cnc
|
||||
- FOURPLAY_COMMAND_TIMEOUT=5000
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "3030:3030"
|
||||
|
||||
```
|
||||
|
||||
There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
|
||||
|
||||
# Steps
|
||||
|
||||
1. start the services with
|
||||
|
||||
```
|
||||
docker compose up -d
|
||||
|
||||
```
|
||||
|
||||
2. configure firefox
|
||||
|
||||
folllow the steps in [configure](./configure.md) for firefox setup
|
||||
@@ -0,0 +1,19 @@
|
||||
FROM alpine:3.21
|
||||
WORKDIR /app
|
||||
|
||||
RUN apk update && apk upgrade
|
||||
RUN apk add nodejs npm
|
||||
|
||||
COPY package*.json ./
|
||||
|
||||
RUN npm ci --only=production && npm cache clean --force
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 3030
|
||||
|
||||
ENTRYPOINT ["node"]
|
||||
CMD ["page-render.js"]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# docker-compose.yaml
|
||||
|
||||
services:
|
||||
fourget:
|
||||
image: luuul/4get:latest
|
||||
restart: unless-stopped
|
||||
environment:
|
||||
- FOURGET_PROTO=http
|
||||
- FOURGET_SERVER_NAME=4get.ca
|
||||
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
|
||||
- FOURGET_FPLAY_PASSWORD=cnc
|
||||
- FOURGET_FPLAY_TIMEOUT=30000
|
||||
- FOURGET_FPLAY_ENABLE_API=true
|
||||
ports:
|
||||
- "80:80"
|
||||
fourplay:
|
||||
image: luuul/4get-4play:latest
|
||||
environment:
|
||||
- FOURPLAY_PORT=3030
|
||||
- FOURPLAY_PASSWORD=cnc
|
||||
- FOURPLAY_COMMAND_TIMEOUT=5000
|
||||
ports:
|
||||
- "3000:3000"
|
||||
- "3030:3030"
|
||||
Generated
+47
@@ -0,0 +1,47 @@
|
||||
{
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"@lawlers/4play": "^1.2.5",
|
||||
"ws": "^8.21.2"
|
||||
}
|
||||
},
|
||||
"node_modules/@lawlers/4play": {
|
||||
"version": "1.2.5",
|
||||
"resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
|
||||
"integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"ws": "^8.21.1"
|
||||
}
|
||||
},
|
||||
"node_modules/ws": {
|
||||
"version": "8.21.2",
|
||||
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
|
||||
"integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
|
||||
"license": "MIT",
|
||||
"engines": {
|
||||
"node": ">=10.0.0"
|
||||
},
|
||||
"peerDependencies": {
|
||||
"bufferutil": "^4.0.1",
|
||||
"utf-8-validate": ">=5.0.2"
|
||||
},
|
||||
"peerDependenciesMeta": {
|
||||
"bufferutil": {
|
||||
"optional": true
|
||||
},
|
||||
"utf-8-validate": {
|
||||
"optional": true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"name": "4play",
|
||||
"version": "1.0.0",
|
||||
"description": "",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "https://git.lolcat.ca/lolcat/4get/"
|
||||
},
|
||||
"license": "AGPL-3.0-only",
|
||||
"author": "lolcat",
|
||||
"type": "commonjs",
|
||||
"main": "page-render.js",
|
||||
"scripts": {
|
||||
"test": "echo \"Error: no test specified\" && exit 1"
|
||||
},
|
||||
"dependencies": {
|
||||
"@lawlers/4play": "^1.2.5",
|
||||
"ws": "^8.21.2"
|
||||
}
|
||||
}
|
||||
@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
|
||||
const http = require("http");
|
||||
|
||||
// 4play settings
|
||||
var port = 3030;
|
||||
var timeout = 30000;
|
||||
var password = "cnc";
|
||||
var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
|
||||
var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
|
||||
var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
|
||||
|
||||
// ==================================================
|
||||
|
||||
|
||||
@@ -1,151 +0,0 @@
|
||||
<?php
|
||||
|
||||
$scrapers = [
|
||||
"web" => [
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Google API",
|
||||
"Google CSE",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Yep",
|
||||
"Mwmbl",
|
||||
"Mojeek",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Cốc Cốc",
|
||||
"Solofield",
|
||||
"Marginalia",
|
||||
"Purili",
|
||||
"wiby"
|
||||
],
|
||||
"images" => [
|
||||
"DuckDuckGo",
|
||||
"Yandex",
|
||||
"Brave",
|
||||
"Google",
|
||||
"Google API",
|
||||
"Google CSE",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Solofield",
|
||||
"Pinterest",
|
||||
"Flickr",
|
||||
"Pexels",
|
||||
"Pixabay",
|
||||
"Unsplash",
|
||||
"500px",
|
||||
"VSCO",
|
||||
"Imgur",
|
||||
"FindThatMeme"
|
||||
],
|
||||
"videos" => [
|
||||
"YouTube",
|
||||
"Vimeo",
|
||||
"Sepia Search",
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Naver",
|
||||
"Baidu",
|
||||
"Cốc Cốc",
|
||||
"Purili",
|
||||
"Solofield"
|
||||
],
|
||||
"news" => [
|
||||
"DuckDuckGo",
|
||||
"Brave",
|
||||
"Google",
|
||||
"Yahoo! JAPAN",
|
||||
"Startpage",
|
||||
"Baidu"
|
||||
],
|
||||
"music" => [
|
||||
"SoundCloud",
|
||||
"Swisscows"
|
||||
],
|
||||
"autocomplete" => [
|
||||
"Brave",
|
||||
"DuckDuckGo",
|
||||
"Yandex",
|
||||
"Google",
|
||||
"Qwant",
|
||||
"Marginalia",
|
||||
"YouTube",
|
||||
"SoundCloud",
|
||||
"Startpage",
|
||||
"Kagi"
|
||||
]
|
||||
];
|
||||
|
||||
$lines = [
|
||||
0 => "",
|
||||
1 => ""
|
||||
];
|
||||
|
||||
// get longest scraper string
|
||||
// also get list of scrapers with most entries
|
||||
$biggest_category_len = 0;
|
||||
foreach($scrapers as $category_name => $category){
|
||||
|
||||
$count = count($category);
|
||||
|
||||
if($count > $biggest_category_len){
|
||||
|
||||
$biggest_category_len = $count;
|
||||
}
|
||||
}
|
||||
|
||||
foreach($scrapers as $category_name => $category){
|
||||
|
||||
// get longest string of the category
|
||||
$longest_len = mb_strlen($category_name);
|
||||
foreach($category as $scraper){
|
||||
|
||||
$strlen = mb_strlen($scraper);
|
||||
|
||||
if($strlen > $longest_len){
|
||||
|
||||
$longest_len = $strlen;
|
||||
}
|
||||
}
|
||||
|
||||
// add header
|
||||
$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
|
||||
$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
|
||||
|
||||
for($i=2; $i<count($category) + 2; $i++){
|
||||
|
||||
if(!isset($lines[$i])){
|
||||
|
||||
$lines[$i] = "";
|
||||
}
|
||||
|
||||
$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
|
||||
}
|
||||
|
||||
if($i - 2 < $biggest_category_len){
|
||||
|
||||
for($k=$i; $k<$biggest_category_len + 2; $k++){
|
||||
|
||||
if(!isset($lines[$k])){
|
||||
|
||||
$lines[$k] = "";
|
||||
}
|
||||
|
||||
$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach($lines as $line){
|
||||
|
||||
echo $line . "|\n";
|
||||
}
|
||||
+5
-8
@@ -45,15 +45,15 @@ class anubis{
|
||||
|
||||
if(
|
||||
!isset($script["challenge"]) ||
|
||||
!is_string($script["challenge"]["randomData"]) ||
|
||||
!isset($script["rules"]["difficulty"]) ||
|
||||
!is_int($script["rules"]["difficulty"])
|
||||
!is_int($script["rules"]["difficulty"]) ||
|
||||
!is_string($script["challenge"])
|
||||
){
|
||||
|
||||
throw new Exception("Found invalid challenge data");
|
||||
}
|
||||
|
||||
return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
|
||||
return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
|
||||
}
|
||||
|
||||
private function is_valid_hash($hash, $difficulty){
|
||||
@@ -74,10 +74,9 @@ class anubis{
|
||||
return true;
|
||||
}
|
||||
|
||||
public function rape($id, $data, $difficulty = 5){
|
||||
public function rape($data, $difficulty = 5){
|
||||
|
||||
$nonce = 0;
|
||||
$microtime = microtime(true);
|
||||
|
||||
while(true){
|
||||
|
||||
@@ -88,12 +87,10 @@ class anubis{
|
||||
$hash_hex = bin2hex($hash_binary);
|
||||
|
||||
return [
|
||||
"id" => $id,
|
||||
"response" => $hash_hex,
|
||||
//"data" => $data,
|
||||
//"difficulty" => $difficulty,
|
||||
"nonce" => $nonce,
|
||||
"elapsedTime" => round((microtime(true) - $microtime) * 1000)
|
||||
"nonce" => $nonce
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
+6
-1
@@ -644,6 +644,7 @@ class frontend{
|
||||
"google_cse" => "Google CSE",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"yep" => "Yep",
|
||||
"mwmbl" => "Mwmbl",
|
||||
"mojeek" => "Mojeek",
|
||||
@@ -670,10 +671,12 @@ class frontend{
|
||||
"google_cse" => "Google CSE",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"naver" => "Naver",
|
||||
"baidu" => "Baidu",
|
||||
"solofield" => "Solofield",
|
||||
"pinterest" => "Pinterest",
|
||||
"cara" => "Cara",
|
||||
"flickr" => "Flickr",
|
||||
"pexels" => "Pexels",
|
||||
"pixabay" => "Pixabay",
|
||||
@@ -702,6 +705,7 @@ class frontend{
|
||||
"google" => "Google",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
"qwant" => "Qwant",
|
||||
"naver" => "Naver",
|
||||
"baidu" => "Baidu",
|
||||
"coccoc" => "Cốc Cốc",
|
||||
@@ -720,7 +724,8 @@ class frontend{
|
||||
"google" => "Google",
|
||||
"yahoo_japan" => "Yahoo! JAPAN",
|
||||
"startpage" => "Startpage",
|
||||
//"mojeek" => "Mojeek",
|
||||
"qwant" => "Qwant",
|
||||
"mojeek" => "Mojeek",
|
||||
"baidu" => "Baidu"
|
||||
]
|
||||
];
|
||||
|
||||
+2
-2
@@ -919,8 +919,8 @@ class baidu{
|
||||
$description =
|
||||
$this->fuckhtml
|
||||
->getElementsByAttributeValue(
|
||||
"data-module",
|
||||
"abstract",
|
||||
"data-sanssr-cmpt",
|
||||
"card/www-summary-1",
|
||||
$div
|
||||
);
|
||||
|
||||
|
||||
@@ -383,8 +383,6 @@ class brave{
|
||||
// load html
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
/*
|
||||
Get next page "token"
|
||||
*/
|
||||
@@ -1177,8 +1175,6 @@ class brave{
|
||||
// load html
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
// get npt
|
||||
$out["npt"] =
|
||||
$this->generatenextpagetoken(
|
||||
@@ -1274,9 +1270,6 @@ class brave{
|
||||
fclose($handle);*/
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
$json = $this->get_js();
|
||||
|
||||
foreach(
|
||||
@@ -1381,8 +1374,6 @@ class brave{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
@@ -1862,34 +1853,4 @@ class brave{
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
private function detect_captcha(){
|
||||
|
||||
$title =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"title"
|
||||
);
|
||||
|
||||
if(count($title) === 0){
|
||||
|
||||
if(strtolower($this->fuckhtml->getloadedhtml()) == "this service is not available in your region"){
|
||||
|
||||
throw new Exception("Brave rangebanned the IP range");
|
||||
}
|
||||
|
||||
throw new Exception("Brave returned a malformed page");
|
||||
}
|
||||
|
||||
$title =
|
||||
strtolower(
|
||||
$this->fuckhtml
|
||||
->getTextContent($title[0])
|
||||
);
|
||||
|
||||
if($title == "human verification"){
|
||||
|
||||
throw new Exception("Brave returned a Captcha");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,847 @@
|
||||
<?php
|
||||
|
||||
class cara{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
include "lib/backend.php";
|
||||
$this->backend = new backend("cara");
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
|
||||
return [
|
||||
"sort" => [
|
||||
"display" => "Sort by",
|
||||
"option" => [
|
||||
"Top" => "Top",
|
||||
"MostRecent" => "Most Recent"
|
||||
]
|
||||
],
|
||||
"type" => [
|
||||
"display" => "Post type",
|
||||
"option" => [
|
||||
"any" => "Any type",
|
||||
"portfolio" => "Portfolio", // {"posts":["portfolio"]}
|
||||
"timeline" => "Timeline" // {"posts":["timeline"]}
|
||||
]
|
||||
],
|
||||
"fields" => [
|
||||
"display" => "Field/Medium",
|
||||
"option" => [
|
||||
"any" => "Any field",
|
||||
"2D" => "2D Work",
|
||||
"3D" => "3D Work",
|
||||
"3DPrinting" => "3D Printing",
|
||||
"Acrylic" => "Acrylic",
|
||||
"AlcoholMarkers" => "Alcohol Markers",
|
||||
"Animation" => "Animation",
|
||||
"Chalk" => "Chalk",
|
||||
"Charcoal" => "Charcoal",
|
||||
"Colored pencil" => "Colored pencil",
|
||||
"Conte" => "Conte",
|
||||
"Crayon" => "Crayon",
|
||||
"Digital" => "Digital",
|
||||
"Gouache" => "Gouache",
|
||||
"Ink" => "Ink",
|
||||
"MixedMedia" => "Mixed-Media",
|
||||
"Oil" => "Oil",
|
||||
"Oil-based Markers" => "Oil-based Markers",
|
||||
"Other" => "Other",
|
||||
"Pastels" => "Pastels",
|
||||
"Photography" => "Photography",
|
||||
"Sculpture" => "Sculpture",
|
||||
"Sketches" => "Sketches",
|
||||
"Tattoos" => "Tattoos",
|
||||
"Traditional" => "Traditional",
|
||||
"VFX" => "VFX",
|
||||
"Watercolor" => "Watercolor"
|
||||
]
|
||||
],
|
||||
"category" => [
|
||||
"display" => "Category",
|
||||
"option" => [
|
||||
"any" => "Any category",
|
||||
"3DScanning" => "3D Scanning",
|
||||
"Abstract" => "Abstract",
|
||||
"Adoptable" => "Adoptable",
|
||||
"Anatomy" => "Anatomy",
|
||||
"Animals" => "Animals",
|
||||
"Anime" => "Anime",
|
||||
"App" => "App",
|
||||
"ArchitecturalConcepts" => "Architectural Concepts",
|
||||
"ArchitecturalVisualization" => "Architectural Visualization",
|
||||
"AugmentedReality" => "Augmented Reality",
|
||||
"Automotive" => "Automotive",
|
||||
"BoardGameArt" => "Board Game Art",
|
||||
"BookIllustration" => "Book Illustration",
|
||||
"CardGameArt" => "Card Game Art",
|
||||
"CeramicsPottery" => "Ceramics/Pottery",
|
||||
"CharacterAnimation" => "Character Animation",
|
||||
"CharacterDesign" => "Character Design",
|
||||
"CharacterModeling" => "Character Modeling",
|
||||
"ChildrensArt" => "Children's Illustration",
|
||||
"Collectibles" => "Collectibles",
|
||||
"ColoringPage" => "Coloring Page",
|
||||
"ComicArt" => "Comic Art",
|
||||
"ConceptArt" => "Concept Art",
|
||||
"Cosplay" => "Cosplay",
|
||||
"CostumeDesign" => "Costume Design",
|
||||
"CoverArt" => "Cover Art",
|
||||
"Creatures" => "Creatures",
|
||||
"Diorama" => "Diorama",
|
||||
"EditorialIllustration" => "Editorial Illustration",
|
||||
"EmbroiderySewing" => "Embroidery/Sewing",
|
||||
"EnvironmentalConceptArt" => "Environmental Concept Art",
|
||||
"EnvironmentalConceptDesign" => "Environmental Concept Design",
|
||||
"FanArt" => "Fan Art",
|
||||
"Fantasy" => "Fantasy",
|
||||
"Fashion" => "Fashion",
|
||||
"FashionStyling" => "Fashion Styling",
|
||||
"FiberArts" => "Fiber Arts",
|
||||
"Furry" => "Furry",
|
||||
"GameArt" => "Game Art",
|
||||
"GameplayDesign" => "Gameplay Design",
|
||||
"GamesEnvironmentArt" => "Games Environment Art",
|
||||
"Gem" => "Gem",
|
||||
"GraphicDesign" => "Graphic Design",
|
||||
"Handicraft" => "Handicraft",
|
||||
"HairStyling" => "Hair Styling",
|
||||
"HardSurface" => "Hard Surface",
|
||||
"Horror" => "Horror",
|
||||
"Illustration" => "Illustration",
|
||||
"IllustrationVisualization" => "Illustration Visualization",
|
||||
"IndustrialDesign" => "Industrial Design",
|
||||
"Jewelry" => "Jewelry",
|
||||
"KnittingCrochet" => "Knitting/Crochet",
|
||||
"Landscape" => "Landscape",
|
||||
"LevelDesign" => "Level Design",
|
||||
"Lighting" => "Lighting",
|
||||
"Makeup" => "Makeup",
|
||||
"Manga" => "Manga",
|
||||
"MapsCartography" => "Maps/Cartography",
|
||||
"MattePainting" => "Matte Painting",
|
||||
"Materials" => "Materials",
|
||||
"MechanicalDesign" => "Mechanical Design",
|
||||
"Medical" => "Medical",
|
||||
"Mecha" => "Mecha",
|
||||
"MiniatureArt" => "Miniature Art",
|
||||
"MotionGraphics" => "Motion Graphics",
|
||||
"FrescoMurals" => "Fresco/Murals",
|
||||
"Natural" => "Natural",
|
||||
"Original Character" => "Original Character",
|
||||
"Overlay" => "Overlay",
|
||||
"PleinAir" => "Plein Air",
|
||||
"Photogrammetry" => "Photogrammetry",
|
||||
"PixelArt" => "Pixel Art",
|
||||
"Portraits" => "Portraits",
|
||||
"Props" => "Props",
|
||||
"ProductDesign" => "Product Design",
|
||||
"PublicDomain" => "Public Domain or Royalty Free",
|
||||
"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
|
||||
"Realism" => "Realism",
|
||||
"ScienceFiction" => "Science Fiction",
|
||||
"ScientificVisualization" => "Scientific Visualization",
|
||||
"Scripts" => "Scripts",
|
||||
"StillLife" => "Still Life",
|
||||
"Storyboards" => "Storyboards",
|
||||
"Stylized" => "Stylized",
|
||||
"Surreal" => "Surreal",
|
||||
"TechnicalArt" => "Technical Art",
|
||||
"Textures" => "Textures",
|
||||
"Tools" => "Tools",
|
||||
"Toys" => "Toys",
|
||||
"ToyPackaging" => "Toy Packaging",
|
||||
"Tutorials" => "Tutorials",
|
||||
"UIArt" => "User Interface (UI) Art",
|
||||
"UrbanSketch" => "Urban Sketch",
|
||||
"VFXforAnimation" => "VFX for Animation",
|
||||
"VFXforFilm" => "VFX for Film",
|
||||
"VFXforGames" => "VFX for Games",
|
||||
"VFXforRealTime" => "VFX for Real-Time",
|
||||
"VFXforTV" => "VFX for TV",
|
||||
"Vehicles" => "Vehicles",
|
||||
"VirtualReality" => "Virtual Reality",
|
||||
"VisualDevelopment" => "Visual Development",
|
||||
"VoxelArt" => "Voxel Art",
|
||||
"Vtubers" => "Vtubers",
|
||||
"WIP" => "WIP (Work in Progress)",
|
||||
"Web" => "Web",
|
||||
"Weapons" => "Weapons",
|
||||
"Wildlife" => "Wildlife",
|
||||
"Woodcutting" => "Woodcutting"
|
||||
]
|
||||
],
|
||||
"software" => [
|
||||
"display" => "Software",
|
||||
"option" => [
|
||||
"any" => "Any software",
|
||||
"123D" => "123D",
|
||||
"123DCatch" => "123D Catch",
|
||||
"3DBee" => "3DBee",
|
||||
"3DCoat" => "3DCoat",
|
||||
"3DCoatPrint" => "3DCoatPrint",
|
||||
"3DCoatTextura" => "3DCoatTextura",
|
||||
"3DEqualizer" => "3DEqualizer",
|
||||
"3DFZephyr" => "3DF Zephyr",
|
||||
"3Delight" => "3Delight",
|
||||
"3dpeople" => "3dpeople",
|
||||
"3dsMax" => "3ds Max",
|
||||
"3DSPaint" => "3DS Paint",
|
||||
"ACDSeeCanvas" => "ACDSee Canvas",
|
||||
"AbletonLive" => "Ableton Live",
|
||||
"Acrobat" => "Acrobat",
|
||||
"AdobeDraw" => "Adobe Draw",
|
||||
"AdobeFlash" => "Adobe Flash",
|
||||
"AdobeFresco" => "Adobe Fresco",
|
||||
"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
|
||||
"AdobeXD" => "Adobe XD",
|
||||
"AffinityDesigner" => "Affinity Designer",
|
||||
"AffinityPhoto" => "Affinity Photo",
|
||||
"AfterEffects" => "After Effects",
|
||||
"Akeytsu" => "Akeytsu",
|
||||
"Alchemy" => "Alchemy",
|
||||
"AliasDesign" => "Alias Design",
|
||||
"AlightMotion" => "Alight Motion",
|
||||
"Amadine" => "Amadine",
|
||||
"Amberlight" => "Amberlight",
|
||||
"Animate" => "Animate",
|
||||
"AnimationMaster" => "Animation:Master",
|
||||
"AnimeStudio" => "Anime Studio",
|
||||
"Apophysis" => "Apophysis",
|
||||
"ArchiCAD" => "ArchiCAD",
|
||||
"Arion" => "Arion",
|
||||
"ArionFX" => "ArionFX",
|
||||
"Arnold" => "Arnold",
|
||||
"ArtEngine" => "ArtEngine",
|
||||
"ArtFlow" => "ArtFlow",
|
||||
"ArtRage" => "ArtRage",
|
||||
"ArtstudioPro" => "Artstudio Pro",
|
||||
"Artweaver" => "Artweaver",
|
||||
"Aseprite" => "Aseprite",
|
||||
"Audition" => "Audition",
|
||||
"AutoCAD" => "AutoCAD",
|
||||
"AutodeskSketchBook" => "Autodesk SketchBook",
|
||||
"AvidMediaComposer" => "Avid Media Composer",
|
||||
"AzPainter" => "AzPainter",
|
||||
"babylonjs" => "babylon.js",
|
||||
"BalsamiqMockup" => "Balsamiq Mockup",
|
||||
"Bforartists" => "Bforartists",
|
||||
"BlackInk" => "Black Ink",
|
||||
"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
|
||||
"Blender" => "Blender",
|
||||
"Blender DeepPaint" => "Blender DeepPaint",
|
||||
"BlenderGreasePencil" => "Blender Grease Pencil",
|
||||
"Blockbench" => "Blockbench",
|
||||
"BodyPaint" => "BodyPaint",
|
||||
"Boxcutter" => "Boxcutter",
|
||||
"BraidMaker" => "Braid Maker",
|
||||
"BrickLinkStudio" => "BrickLink Studio",
|
||||
"Bridge" => "Bridge",
|
||||
"Brushifyio" => "Brushify.io",
|
||||
"C" => "C",
|
||||
"C#" => "C#",
|
||||
"C++" => "C++",
|
||||
"CACANi" => "CACANi",
|
||||
"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
|
||||
"CLO" => "CLO",
|
||||
"CRYENGINE" => "CRYENGINE",
|
||||
"Callipeg" => "Callipeg",
|
||||
"Canva" => "Canva",
|
||||
"CaptureOne" => "Capture One",
|
||||
"CartoonAnimator" => "Cartoon Animator",
|
||||
"Carveco" => "Carveco",
|
||||
"Cavalry" => "Cavalry",
|
||||
"Chaotica" => "Chaotica",
|
||||
"CharacterAnimator" => "Character Animator",
|
||||
"CharacterCreator" => "Character Creator",
|
||||
"Cinema4D" => "Cinema 4D",
|
||||
"ClarisseiFX" => "Clarisse iFX",
|
||||
"Coiffure" => "Coiffure",
|
||||
"ColorsLive" => "Colors Live",
|
||||
"Combustion" => "Combustion",
|
||||
"Construct2" => "Construct 2",
|
||||
"Core" => "Core",
|
||||
"CorelPainter" => "Corel Painter",
|
||||
"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
|
||||
"CoronaRenderer" => "Corona Renderer",
|
||||
"ProMotionNG" => "Cosmigo Pro Motion NG",
|
||||
"CrazyBump" => "CrazyBump",
|
||||
"Crocotile3D" => "Crocotile 3D",
|
||||
"Curvy3D" => "Curvy 3D",
|
||||
"Cycles4D" => "Cycles 4D",
|
||||
"Darkroom" => "Darkroom",
|
||||
"DAZStudio" => "DAZ Studio",
|
||||
"DDO" => "DDO",
|
||||
"DECIMA" => "DECIMA",
|
||||
"Darktable" => "Darktable",
|
||||
"DaVinciResolve" => "DaVinci Resolve",
|
||||
"Dimension" => "Dimension",
|
||||
"DragonBones" => "DragonBones",
|
||||
"Dragonframe" => "Dragonframe",
|
||||
"Drawpile" => "Drawpile",
|
||||
"Dreams" => "Dreams",
|
||||
"Dreamweaver" => "Dreamweaver",
|
||||
"DxOPhotoLab" => "DxO PhotoLab",
|
||||
"ECycles" => "E-Cycles",
|
||||
"EmberGen" => "EmberGen",
|
||||
"Encore" => "Encore",
|
||||
"Expresii" => "Expresii",
|
||||
"FStorm" => "FStorm",
|
||||
"FadeIn" => "FadeIn",
|
||||
"Feather3D" => "Feather 3D",
|
||||
"FiberShop" => "FiberShop",
|
||||
"Figma" => "Figma",
|
||||
"FilmoraWondershare" => "Filmora Wondershare",
|
||||
"FilterForge" => "Filter Forge",
|
||||
"FinalCutPro" => "Final Cut Pro",
|
||||
"FinalDraft" => "Final Draft",
|
||||
"finalRender" => "finalRender",
|
||||
"FireAlpaca" => "FireAlpaca",
|
||||
"Fireworks" => "Fireworks",
|
||||
"FlamePainter" => "Flame Painter",
|
||||
"Flash" => "Flash",
|
||||
"FlipaClip" => "FlipaClip",
|
||||
"FlipnoteStudio" => "Flipnote Studio",
|
||||
"Fluent" => "Fluent",
|
||||
"ForestPack" => "Forest Pack",
|
||||
"FormZ" => "Form-Z",
|
||||
"Fractorium" => "Fractorium",
|
||||
"FreeCAD" => "FreeCAD",
|
||||
"FreeHand" => "FreeHand",
|
||||
"Forger" => "Forger",
|
||||
"FrostbiteEngine" => "Frostbite Engine",
|
||||
"fSpy" => "fSpy",
|
||||
"FumeFX" => "FumeFX",
|
||||
"Fusion360" => "Fusion 360",
|
||||
"GIMP" => "GIMP",
|
||||
"GSCurveTools" => "GS CurveTools",
|
||||
"GSToolbox" => "GS Toolbox",
|
||||
"Gaea" => "Gaea",
|
||||
"GameTextures" => "Game Textures",
|
||||
"GameMakerStudio" => "GameMaker: Studio",
|
||||
"GarageFarmNET" => "GarageFarm.NET",
|
||||
"GeoGlyph" => "GeoGlyph",
|
||||
"GigapixelAl" => "Gigapixel Al",
|
||||
"Glaxnimate" => "Glaxnimate",
|
||||
"GnomePaint" => "Gnome Paint",
|
||||
"Godot" => "Godot",
|
||||
"Goxel" => "Goxel",
|
||||
"Graphite" => "Graphite",
|
||||
"Graswald" => "Graswald",
|
||||
"GravitySketch" => "Gravity Sketch",
|
||||
"GuerillaRender" => "GuerillaRender",
|
||||
"HDRLightStudio" => "HDR Light Studio",
|
||||
"HairStrandDesigner" => "Hair Strand Designer",
|
||||
"HairTGHairFur" => "HairTG - Hair & Fur",
|
||||
"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
|
||||
"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
|
||||
"Handplane" => "Handplane",
|
||||
"Hansoft" => "Hansoft",
|
||||
"HardOps" => "Hard Ops",
|
||||
"HardMesh" => "HardMesh",
|
||||
"Harmony" => "Harmony",
|
||||
"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
|
||||
"HelloPaint" => "HelloPaint",
|
||||
"HeliconFocus" => "Helicon Focus",
|
||||
"Hexels" => "Hexels",
|
||||
"HiPaint" => "HiPaint",
|
||||
"Houdini" => "Houdini",
|
||||
"HydraRenderer" => "Hydra Renderer",
|
||||
"iArtbook" => "iArtbook",
|
||||
"IbisPaint" => "ibisPaint",
|
||||
"Ideas" => "Ideas",
|
||||
"IllustStudio" => "Illust Studio",
|
||||
"Illustrator" => "Illustrator",
|
||||
"IllustratorDraw" => "Illustrator Draw",
|
||||
"InDesign" => "InDesign",
|
||||
"Inochi2D" => "Inochi2D",
|
||||
"InVision" => "InVision",
|
||||
"InVisionCraft" => "InVision Craft",
|
||||
"InfinitePainter" => "Infinite Painter",
|
||||
"Inkscape" => "Inkscape",
|
||||
"Inspirit" => "Inspirit",
|
||||
"InstaLOD" => "InstaLOD",
|
||||
"InstaMAT" => "InstaMAT",
|
||||
"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
|
||||
"InstantMeshes" => "Instant Meshes",
|
||||
"InstantTerra" => "Instant Terra",
|
||||
"Inventor" => "Inventor",
|
||||
"Iray" => "Iray",
|
||||
"JWildfire" => "JWildfire",
|
||||
"Java" => "Java",
|
||||
"Jira" => "Jira",
|
||||
"JumpPaint" => "Jump Paint by MediBang",
|
||||
"JSPaint" => "JS Paint",
|
||||
"Katana" => "Katana",
|
||||
"Keyshot" => "Keyshot",
|
||||
"KidPix" => "Kid Pix",
|
||||
"KitBash3D" => "KitBash3D",
|
||||
"Knald" => "Knald",
|
||||
"Kodon" => "Kodon",
|
||||
"KolourPaint" => "KolourPaint",
|
||||
"Krakatoa" => "Krakatoa",
|
||||
"KRESKA" => "KRESKA",
|
||||
"Krita" => "Krita",
|
||||
"LensStudio" => "Lens Studio",
|
||||
"LibreSprite" => "LibreSprite",
|
||||
"LightWave3D" => "LightWave 3D",
|
||||
"Lightroom" => "Lightroom",
|
||||
"Linearity" => "Linearity",
|
||||
"LiquiGen" => "LiquiGen",
|
||||
"Live2DCubism" => "Live2D Cubism",
|
||||
"LookatmyHair" => "Look at my Hair",
|
||||
"Lotpixel" => "Lotpixel",
|
||||
"Lumion" => "Lumion",
|
||||
"LuxRender" => "LuxRender",
|
||||
"MacPaint" => "MacPaint",
|
||||
"MagicaCSG" => "MagicaCSG",
|
||||
"MagicaVoxel" => "MagicaVoxel",
|
||||
"Magma" => "Magma",
|
||||
"MakeHuman" => "MakeHuman",
|
||||
"Malmal" => "Malmal",
|
||||
"Mandelbulb3D" => "Mandelbulb 3D",
|
||||
"Mandelbulber" => "Mandelbulber",
|
||||
"MangaStudio" => "Manga Studio",
|
||||
"Mari" => "Mari",
|
||||
"MarmosetToolbag" => "Marmoset Toolbag",
|
||||
"MarvelousDesigner" => "Marvelous Designer",
|
||||
"MasterpieceStudioPro" => "Masterpiece Studio Pro",
|
||||
"MasterpieceVR" => "MasterpieceVR",
|
||||
"Maverick" => "Maverick",
|
||||
"MaxwellRender" => "Maxwell Render",
|
||||
"Maya" => "Maya",
|
||||
"MediBangPaint" => "MediBang Paint",
|
||||
"MediumbyAdobe" => "Medium by Adobe",
|
||||
"Megascans" => "Megascans",
|
||||
"mentalray" => "mental ray",
|
||||
"MeshLab" => "MeshLab",
|
||||
"Meshroom" => "Meshroom",
|
||||
"MetaHumanCreator" => "MetaHuman Creator",
|
||||
"Metashape" => "Metashape",
|
||||
"MightyBake" => "MightyBake",
|
||||
"MikuMikuDance" => "MikuMikuDance",
|
||||
"Minecraft" => "Minecraft",
|
||||
"Mischief" => "Mischief",
|
||||
"Mixamo" => "Mixamo",
|
||||
"Mixer" => "Mixer",
|
||||
"MoI3D" => "MoI3D",
|
||||
"Mocha" => "Mocha",
|
||||
"Modo" => "Modo",
|
||||
"Moho" => "Moho",
|
||||
"MotionBuilder" => "MotionBuilder",
|
||||
"Mudbox" => "Mudbox",
|
||||
"Muse" => "Muse",
|
||||
"MSPaint" => "MS Paint",
|
||||
"MyPaint" => "MyPaint",
|
||||
"NDO" => "NDO",
|
||||
"NX" => "NX",
|
||||
"NdotCAD" => "NdotCAD",
|
||||
"NintendoNotes" => "Nintendo Notes",
|
||||
"NomadSculpt" => "Nomad Sculpt",
|
||||
"Notability" => "Notability",
|
||||
"Nuke" => "Nuke",
|
||||
"Nvil" => "Nvil",
|
||||
"OctaneRender" => "Octane Render",
|
||||
"Omniverse" => "Omniverse",
|
||||
"OmniverseCreate" => "Omniverse Create",
|
||||
"ON1PhotoRAW" => "ON1 Photo RAW",
|
||||
"Open3DEngine" => "Open 3D Engine",
|
||||
"OpenCanvas" => "OpenCanvas",
|
||||
"OpenGL" => "OpenGL",
|
||||
"OpenToonz" => "OpenToonz",
|
||||
"Ornatrix" => "Ornatrix",
|
||||
"OsciRender" => "Osci-Render",
|
||||
"OurPaint" => "Our Paint",
|
||||
"PBRMAX" => "PBRMAX",
|
||||
"PFTrack" => "PFTrack",
|
||||
"PTGui" => "PTGui",
|
||||
"Paintbrush" => "Paintbrush",
|
||||
"PaintNET" => "Paint.NET",
|
||||
"PaintShopPro" => "PaintShop Pro",
|
||||
"PaintToolSAI" => "Paint Tool SAI",
|
||||
"PaintstormStudio" => "Paintstorm Studio",
|
||||
"Paper" => "Paper",
|
||||
"Pencil2D" => "Pencil2D",
|
||||
"Penpot" => "Penpot",
|
||||
"PhoenixFD" => "Phoenix FD",
|
||||
"Phonto" => "Phonto",
|
||||
"PhotoLab2" => "PhotoLab 2",
|
||||
"Photopea" => "Photopea",
|
||||
"Photoscan" => "Photoscan",
|
||||
"Photoshop" => "Photoshop",
|
||||
"PhotoshopElements" => "Photoshop Elements",
|
||||
"PicoCAD" => "picoCAD",
|
||||
"PicoCAD2" => "picoCAD 2",
|
||||
"Pinta" => "Pinta",
|
||||
"Piskel" => "Piskel",
|
||||
"Pixilart" => "Pixilart",
|
||||
"Pixelitor" => "Pixelitor",
|
||||
"Pixelmator" => "Pixelmator",
|
||||
"Pixelorama" => "Pixelorama",
|
||||
"PixivSketch" => "pixiv Sketch",
|
||||
"Pixquare" => "Pixquare",
|
||||
"PlantCatalog" => "PlantCatalog",
|
||||
"PlantFactory" => "PlantFactory",
|
||||
"Plasticity" => "Plasticity",
|
||||
"PNGtuberPlus" => "PNGtuber Plus",
|
||||
"Poliigon" => "Poliigon",
|
||||
"Polybrush" => "Polybrush",
|
||||
"PopcornFx" => "PopcornFx",
|
||||
"Poser" => "Poser",
|
||||
"Premiere" => "Premiere",
|
||||
"PremiereElements" => "Premiere Elements",
|
||||
"PresagisCreator" => "Presagis Creator",
|
||||
"ProTools" => "Pro Tools",
|
||||
"Procreate" => "Procreate",
|
||||
"ProcreateDreams" => "Procreate Dreams",
|
||||
"Producer" => "Producer",
|
||||
"PrometheanAI" => "Promethean AI",
|
||||
"PureRef" => "PureRef",
|
||||
"Python" => "Python",
|
||||
"PyxelEdit" => "PyxelEdit",
|
||||
"QuadRemesher" => "Quad Remesher",
|
||||
"QuarkXPress" => "QuarkXPress",
|
||||
"Qubicle" => "Qubicle",
|
||||
"Quill" => "Quill",
|
||||
"QuixelBridge" => "Quixel Bridge",
|
||||
"QuixelMegascans" => "Quixel Megascans",
|
||||
"QuixelMixer" => "Quixel Mixer",
|
||||
"QuixelSuite" => "Quixel Suite",
|
||||
"R3DSWrap" => "R3DS Wrap",
|
||||
"R3DSZWRAP" => "R3DS ZWRAP",
|
||||
"RDTextures" => "RD-Textures",
|
||||
"RailClone" => "RailClone",
|
||||
"RealFlow" => "RealFlow",
|
||||
"RealisticPaintStudio" => "Realistic Paint Studio",
|
||||
"RealityCapture" => "RealityCapture",
|
||||
"RealityScan" => "RealityScan",
|
||||
"RealtimeBoard" => "Realtime Board",
|
||||
"Rebelle" => "Rebelle",
|
||||
"Redshift" => "Redshift",
|
||||
"RenderMan" => "RenderMan",
|
||||
"RenderNetwork" => "Render Network",
|
||||
"Revit" => "Revit",
|
||||
"Rhino" => "Rhino",
|
||||
"Rhinoceros" => "Rhinoceros",
|
||||
"RizomUV" => "RizomUV",
|
||||
"RoughAnimator" => "Rough Animator",
|
||||
"SamsungNotes" => "Samsung Notes",
|
||||
"SamsungPENUP" => "Samsung PENUP",
|
||||
"ScansLibrary" => "ScansLibrary",
|
||||
"Scrivener" => "Scrivener",
|
||||
"Sculpt+" => "Sculpt+",
|
||||
"Sculptris" => "Sculptris",
|
||||
"ShaveandaHaircut" => "Shave and a Haircut",
|
||||
"ShiVa3D" => "ShiVa3D",
|
||||
"Shotgun" => "Shotgun",
|
||||
"Silo" => "Silo",
|
||||
"Silugen" => "Silugen",
|
||||
"Sketch" => "Sketch",
|
||||
"SketchApp" => "Sketch App",
|
||||
"SketchBookPro" => "SketchBook Pro",
|
||||
"SketchClub" => "SketchClub",
|
||||
"SketchUp" => "SketchUp",
|
||||
"Sketchable" => "Sketchable",
|
||||
"Sketchfab" => "Sketchfab",
|
||||
"Skyshop" => "Skyshop",
|
||||
"Snapseed" => "Snapseed",
|
||||
"Snowdrop" => "Snowdrop",
|
||||
"Softimage" => "Softimage",
|
||||
"SolidWorks" => "SolidWorks",
|
||||
"SonySketch" => "Sony Sketch",
|
||||
"Soundbooth" => "Soundbooth",
|
||||
"Source2" => "Source 2",
|
||||
"SourceControl" => "Source Control",
|
||||
"SourceFilmmaker" => "Source Filmmaker",
|
||||
"SpeedTree" => "SpeedTree",
|
||||
"Speedgrade" => "Speedgrade",
|
||||
"SpeedyPainter" => "SpeedyPainter",
|
||||
"Spine2D" => "Spine 2D",
|
||||
"Spriter" => "Spriter",
|
||||
"Stingray" => "Stingray",
|
||||
"Storyboarder" => "Storyboarder",
|
||||
"StoryboardPro" => "Storyboard Pro",
|
||||
"SublimeText" => "Sublime Text",
|
||||
"Substance3DDesigner" => "Substance 3D Designer",
|
||||
"Substance3DModeler" => "Substance 3D Modeler",
|
||||
"Substance3DPainter" => "Substance 3D Painter",
|
||||
"Substance3DSampler" => "Substance 3D Sampler",
|
||||
"Substance3DStager" => "Substance 3D Stager",
|
||||
"SubstanceB2M" => "Substance B2M",
|
||||
"SweetHome3D" => "Sweet Home 3D",
|
||||
"SynthEyes" => "SynthEyes",
|
||||
"TTools" => "TTools",
|
||||
"TVPaint" => "TVPaint",
|
||||
"TVPaintAnimation" => "TVPaint Animation",
|
||||
"TayasuiSketches" => "Tayasui Sketches",
|
||||
"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
|
||||
"TayasuiSketchesPro" => "Tayasui Sketches Pro",
|
||||
"Terragen" => "Terragen",
|
||||
"Texturescom" => "Textures.com",
|
||||
"Texturingxyz" => "Texturingxyz",
|
||||
"TeyaConceptor" => "Teya Conceptor",
|
||||
"TheGrove3D" => "The Grove 3D",
|
||||
"TheaRender" => "Thea Render",
|
||||
"Threejs" => "Three.js",
|
||||
"Tiled" => "Tiled",
|
||||
"TiltBrush" => "Tilt Brush",
|
||||
"Tooll3" => "Tooll3",
|
||||
"ToonBoomHarmony" => "Toon Boom Harmony",
|
||||
"ToonBoomStudio" => "Toon Boom Studio",
|
||||
"ToonSquid" => "ToonSquid",
|
||||
"TopoGun" => "TopoGun",
|
||||
"TuxPaint" => "Tux Paint",
|
||||
"Tvori" => "Tvori",
|
||||
"Twinmotion" => "Twinmotion",
|
||||
"UNIGINEEngine" => "UNIGINE Engine",
|
||||
"UVLayout" => "UVLayout",
|
||||
"UltraFractal" => "Ultra Fractal",
|
||||
"uMake" => "uMake",
|
||||
"Unfold3D" => "Unfold 3D",
|
||||
"Unity" => "Unity",
|
||||
"UnrealEngine" => "Unreal Engine",
|
||||
"Vengi" => "vengi",
|
||||
"VRay" => "V-Ray",
|
||||
"VRED" => "VRED",
|
||||
"VTubeStudio" => "VTube Studio",
|
||||
"Vectary" => "Vectary",
|
||||
"VectorayGen" => "VectorayGen",
|
||||
"Vectorworks" => "Vectorworks",
|
||||
"VegasPro" => "Vegas Pro",
|
||||
"VisualDesigner3D" => "Visual Designer 3D",
|
||||
"VisualStudio" => "Visual Studio",
|
||||
"VRoidStudio" => "VRoid Studio",
|
||||
"Vue" => "Vue",
|
||||
"Vuforia" => "Vuforia",
|
||||
"WebGL" => "WebGL",
|
||||
"WhiteboardFox" => "Whiteboard Fox",
|
||||
"WickEditor" => "Wick Editor",
|
||||
"Wings3D" => "Wings 3D",
|
||||
"Word" => "Word",
|
||||
"WorldCreator" => "World Creator",
|
||||
"WorldMachine" => "World Machine",
|
||||
"XParticles" => "X-Particles",
|
||||
"Xfrog" => "Xfrog",
|
||||
"Xgen" => "Xgen",
|
||||
"xNormal" => "xNormal",
|
||||
"xTex" => "xTex",
|
||||
"XoliulShader" => "Xoliul Shader",
|
||||
"Yafaray" => "Yafaray",
|
||||
"Yeti" => "Yeti",
|
||||
"ZBrush" => "ZBrush",
|
||||
"ZBrushCore" => "ZBrushCore",
|
||||
"ZenBrush" => "Zen Brush"
|
||||
]
|
||||
]
|
||||
];
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $search){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
if($get !== []){
|
||||
$get = http_build_query($get);
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
["User-Agent: " . config::USER_AGENT,
|
||||
"Accept: application/json, text/plain, */*",
|
||||
"Accept-Language: en-US,en;q=0.5",
|
||||
"Accept-Encoding: gzip, deflate, br, zstd",
|
||||
//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
|
||||
//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
|
||||
"DNT: 1",
|
||||
"Sec-GPC: 1",
|
||||
"Connection: keep-alive",
|
||||
//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
|
||||
"Referer: https://cara.app/search?q=" . urlencode($search),
|
||||
//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
"TE: trailers"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$this->backend->assign_proxy($curlproc, $proxy);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function image($get){
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$npt, $proxy] =
|
||||
$this->backend->get(
|
||||
$get["npt"],
|
||||
"images"
|
||||
);
|
||||
|
||||
$npt = json_decode($npt, true);
|
||||
}else{
|
||||
|
||||
$search = $get["s"];
|
||||
if(strlen($search) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$npt = [
|
||||
"q" => $get["s"],
|
||||
"sortBy" => $get["sort"],
|
||||
"take" => 24,
|
||||
"skip" => 0,
|
||||
"filters" => []
|
||||
];
|
||||
|
||||
// parse filters
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$npt["filters"]["posts"] = [$get["type"]];
|
||||
}
|
||||
|
||||
if($get["fields"] != "any"){
|
||||
|
||||
$npt["filters"]["fields"] = [$get["fields"]];
|
||||
}
|
||||
|
||||
if($get["category"] != "any"){
|
||||
|
||||
$npt["filters"]["categories"] = [$get["category"]];
|
||||
}
|
||||
|
||||
if($get["software"] != "any"){
|
||||
|
||||
$npt["filters"]["softwares"] = [$get["software"]];
|
||||
}
|
||||
|
||||
if($npt["filters"] == []){
|
||||
|
||||
$npt["filters"] = "{}";
|
||||
}else{
|
||||
|
||||
$npt["filters"] = json_encode($npt["filters"]);
|
||||
}
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
"image" => []
|
||||
];
|
||||
|
||||
// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://cara.app/api/search/posts",
|
||||
$npt,
|
||||
$npt["q"]
|
||||
);
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch JSON");
|
||||
}
|
||||
|
||||
$json = json_decode($json, true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
$imagecount = 0;
|
||||
foreach($json as $image){
|
||||
|
||||
if(count($image["images"]) === 0){
|
||||
|
||||
// sometimes the api returns no images for an object
|
||||
$imagecount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
$cover = null;
|
||||
$sources = [];
|
||||
|
||||
foreach($image["images"] as $source){
|
||||
|
||||
if($source["isCoverImg"]){
|
||||
|
||||
$cover = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => 500,
|
||||
"height" => 500
|
||||
];
|
||||
}else{
|
||||
|
||||
$sources[] = [
|
||||
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
|
||||
"width" => null,
|
||||
"height" => null
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if($cover !== null){
|
||||
|
||||
$sources[] = $cover;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" => str_replace("\n", " ", $image["content"]),
|
||||
"source" => $sources,
|
||||
"url" => "https://cara.app/post/" . $image["id"]
|
||||
];
|
||||
|
||||
$imagecount++;
|
||||
}
|
||||
|
||||
if($imagecount === 24){
|
||||
|
||||
$npt["skip"] += 24;
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode($npt),
|
||||
"images",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function fix_url($url){
|
||||
|
||||
return
|
||||
str_replace(
|
||||
[" "],
|
||||
["%20"],
|
||||
$url
|
||||
);
|
||||
}
|
||||
}
|
||||
+21
-191
@@ -523,7 +523,7 @@ class google{
|
||||
config::FPLAY_EXTERNAL_ENDPOINT .
|
||||
"?url=" . urlencode($url) .
|
||||
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
|
||||
"&container=" .// urlencode($container) .
|
||||
"&container=" . urlencode($container) .
|
||||
"&proxy=" . urlencode($proxy) .
|
||||
"&url_match=" . urlencode("&sei=") .
|
||||
"&fail_url_match=" . urlencode("\/sorry\/");
|
||||
@@ -733,7 +733,7 @@ class google{
|
||||
$container = &$data["container"];
|
||||
|
||||
//file_put_contents("scraper/google-jp.html", $html);
|
||||
//$html = file_get_contents("scraper/google-softcaptcha.html");
|
||||
//$html = file_get_contents("scraper/google-jp.html");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -754,26 +754,9 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
|
||||
// init
|
||||
$this->detect_sorry();
|
||||
$this->parsestyles();
|
||||
$this->scrape_dimg($html);
|
||||
$this->scrape_imagearr($html);
|
||||
|
||||
@@ -1398,28 +1381,7 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
$this->detect_sorry();
|
||||
|
||||
// get javascript images
|
||||
$this->scrape_imagearr($html);
|
||||
@@ -1581,7 +1543,8 @@ class google{
|
||||
implode(",", $tbs);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
$data = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
@@ -1592,8 +1555,6 @@ class google{
|
||||
$html = &$data["data"];
|
||||
$container = &$data["container"];
|
||||
|
||||
//$html = file_get_contents("scraper/google.html");
|
||||
|
||||
if(!isset($params["start"])){
|
||||
|
||||
$params["start"] = 0;
|
||||
@@ -1602,32 +1563,13 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
|
||||
//
|
||||
// Parse web video page
|
||||
//
|
||||
$this->detect_sorry();
|
||||
|
||||
// parse all <style> tags
|
||||
$this->parsestyles();
|
||||
|
||||
// get javascript images
|
||||
$this->scrape_dimg($html);
|
||||
@@ -1636,29 +1578,6 @@ class google{
|
||||
|
||||
$params["c"] = $container;
|
||||
|
||||
// get encoded links
|
||||
$encoded_links =
|
||||
explode(
|
||||
"function(){var m={",
|
||||
$html,
|
||||
2
|
||||
);
|
||||
|
||||
if(count($encoded_links) === 2){
|
||||
|
||||
$encoded_links =
|
||||
json_decode(
|
||||
$this->fuckhtml
|
||||
->extract_json(
|
||||
"{" . $encoded_links[1]
|
||||
),
|
||||
true
|
||||
);
|
||||
}else{
|
||||
|
||||
$encoded_links = [];
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" =>
|
||||
@@ -1837,50 +1756,6 @@ class google{
|
||||
}
|
||||
}
|
||||
|
||||
$url =
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$url[0]["attributes"]["href"]
|
||||
);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^\/goto\?url/',
|
||||
$url
|
||||
)
|
||||
){
|
||||
|
||||
// encrypted url detected
|
||||
// get reference div
|
||||
$refdivs =
|
||||
$this->fuckhtml
|
||||
->getElementsByAttributeName(
|
||||
"id",
|
||||
"div"
|
||||
);
|
||||
|
||||
foreach($refdivs as $r){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/^atritem-/',
|
||||
$r["attributes"]["id"]
|
||||
) &&
|
||||
isset($r["attributes"]["jsdata"])
|
||||
){
|
||||
|
||||
$ref = explode(";", $r["attributes"]["jsdata"]);
|
||||
$ref = $ref[count($ref) - 1];
|
||||
|
||||
if(isset($encoded_links[$ref][32][11][7])){
|
||||
|
||||
$url = $encoded_links[$ref][32][11][7];
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
$out["video"][] = [
|
||||
"title" =>
|
||||
$this->titledots(
|
||||
@@ -1899,7 +1774,11 @@ class google{
|
||||
"duration" => $duration,
|
||||
"views" => null,
|
||||
"thumb" => $thumb,
|
||||
"url" => $url
|
||||
"url" =>
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$url[0]["attributes"]["href"]
|
||||
)
|
||||
];
|
||||
}
|
||||
|
||||
@@ -2008,32 +1887,14 @@ class google{
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
|
||||
// parse styles
|
||||
$this->parsestyles();
|
||||
|
||||
// solve softcaptcha
|
||||
$softcaptcha = $this->detect_sorry();
|
||||
|
||||
if($softcaptcha !== false){
|
||||
|
||||
$params["sei"] = $softcaptcha["sei"];
|
||||
|
||||
$data2 = $this->get(
|
||||
$proxy,
|
||||
"https://www.google.com/search",
|
||||
$params,
|
||||
$container
|
||||
);
|
||||
|
||||
$html = &$data2["data"];
|
||||
|
||||
$this->fuckhtml->load($html);
|
||||
$this->parsestyles();
|
||||
}
|
||||
$this->detect_sorry();
|
||||
|
||||
// get images
|
||||
$this->scrape_dimg($html);
|
||||
|
||||
// parse styles
|
||||
$this->parsestyles();
|
||||
|
||||
$center_col =
|
||||
$this->fuckhtml
|
||||
->getElementById(
|
||||
@@ -2792,7 +2653,6 @@ class google{
|
||||
|
||||
private function detect_sorry(){
|
||||
|
||||
// detect recaptcha
|
||||
$captcha_form =
|
||||
$this->fuckhtml
|
||||
->getElementById(
|
||||
@@ -2822,35 +2682,5 @@ class google{
|
||||
|
||||
throw new Exception("Google returned a JS challenge");
|
||||
}
|
||||
|
||||
// bypass soft captcha
|
||||
// "Verifying your request
|
||||
// You'll be able to continue in a few seconds. Don't refresh this page."
|
||||
$scripts =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName(
|
||||
"script"
|
||||
);
|
||||
|
||||
foreach($scripts as $sc){
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
|
||||
$sc["innerHTML"],
|
||||
$matches
|
||||
)
|
||||
){
|
||||
|
||||
sleep(10); // gotta sleep to not trigger captcha. blame google, not me
|
||||
|
||||
return [
|
||||
"sei" => $matches[1],
|
||||
"timeout" => (int)$matches[2] // set to 10000 usually
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return false; // no need to bypass
|
||||
}
|
||||
}
|
||||
|
||||
+49
-191
@@ -16,7 +16,18 @@ class mojeek{
|
||||
|
||||
case "web":
|
||||
return [
|
||||
"lang" => [ // lb
|
||||
"focus" => [
|
||||
"display" => "Focus",
|
||||
"option" => [
|
||||
"any" => "No focus",
|
||||
"blogs" => "Blogs",
|
||||
"Dictionary" => "Dictionary",
|
||||
"Recipes" => "Recipes",
|
||||
"Time" => "Time",
|
||||
"Weather" => "Weather"
|
||||
]
|
||||
],
|
||||
"lang" => [
|
||||
"display" => "Language",
|
||||
"option" => [
|
||||
"any" => "Any language",
|
||||
@@ -75,7 +86,7 @@ class mojeek{
|
||||
"zu" => "Zulu"
|
||||
]
|
||||
],
|
||||
"country" => [ // rb
|
||||
"country" => [
|
||||
"display" => "Country",
|
||||
"option" => [
|
||||
"any" => "No location bias",
|
||||
@@ -329,21 +340,28 @@ class mojeek{
|
||||
"zm" => "Zambia",
|
||||
"zw" => "Zimbabwe"
|
||||
]
|
||||
],
|
||||
"sort" => [
|
||||
"display" => "Sort by",
|
||||
],/*
|
||||
"region" => [
|
||||
"display" => "Region",
|
||||
"option" => [
|
||||
"relevance" => "Relevance",
|
||||
"date" => "Date" // datewr=100
|
||||
"any" => "Any region",
|
||||
"eu" => "European Union",
|
||||
"de" => "Germany",
|
||||
"fr" => "France",
|
||||
"uk" => "United Kingdom"
|
||||
]
|
||||
],*/
|
||||
"domain" => [
|
||||
"display" => "Results per domain",
|
||||
"option" => [
|
||||
"1" => "1 result",
|
||||
"2" => "2 results",
|
||||
"3" => "3 results",
|
||||
"4" => "4 results",
|
||||
"5" => "5 results",
|
||||
"10" => "10 results",
|
||||
"0" => "Unlimited",
|
||||
]
|
||||
],
|
||||
"newer" => [
|
||||
"display" => "Newer than",
|
||||
"option" => "_DATE"
|
||||
],
|
||||
"older" => [
|
||||
"display" => "Older than",
|
||||
"option" => "_DATE"
|
||||
]
|
||||
];
|
||||
break;
|
||||
@@ -353,29 +371,21 @@ class mojeek{
|
||||
}
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $is_api = false){
|
||||
private function get($proxy, $url, $get = []){
|
||||
|
||||
if($is_api){
|
||||
|
||||
$headers = [
|
||||
"Accept: application/json"
|
||||
];
|
||||
}else{
|
||||
|
||||
$headers = [
|
||||
"User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
];
|
||||
}
|
||||
$headers = [
|
||||
"User-Agent: " . config::USER_AGENT,
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
];
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
@@ -410,154 +420,6 @@ class mojeek{
|
||||
|
||||
public function web($get){
|
||||
|
||||
return $this->web_api($get);
|
||||
}
|
||||
|
||||
public function web_api($get){
|
||||
|
||||
$key = $this->backend->get_key();
|
||||
$proxy = $this->backend->get_ip($key["increment"]);
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$params, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
$params = json_decode($params, true);
|
||||
$params["api_key"] = $key["key"];
|
||||
|
||||
}else{
|
||||
|
||||
if(strlen($get["s"]) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$params = [
|
||||
"api_key" => $key["key"],
|
||||
"q" => $get["s"],
|
||||
"date" => 1,
|
||||
"tlen" => 127, // title length
|
||||
"dlen" => 511, // description length
|
||||
"fmt" => "json"
|
||||
];
|
||||
|
||||
if($get["lang"] != "any"){
|
||||
|
||||
$params["lb"] = $get["lang"];
|
||||
}
|
||||
|
||||
if($get["country"] != "any"){
|
||||
|
||||
$params["rb"] = $get["country"];
|
||||
}
|
||||
|
||||
if($get["sort"] == "date"){
|
||||
|
||||
$params["datewr"] = 100;
|
||||
}
|
||||
|
||||
if($get["newer"] !== false){
|
||||
|
||||
$params["since"] = date("Ymd", $get["newer"]);
|
||||
}
|
||||
|
||||
if($get["older"] !== false){
|
||||
|
||||
$params["before"] = date("Ymd", $get["before"]);
|
||||
}
|
||||
}
|
||||
|
||||
try{
|
||||
$json =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://api.mojeek.com/search",
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to get JSON");
|
||||
}
|
||||
|
||||
$json = json_decode($json, true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"spelling" => [
|
||||
"type" => "no_correction",
|
||||
"using" => null,
|
||||
"correction" => null
|
||||
],
|
||||
"npt" => null,
|
||||
"answer" => [],
|
||||
"web" => [],
|
||||
"image" => [],
|
||||
"video" => [],
|
||||
"news" => [],
|
||||
"related" => []
|
||||
];
|
||||
|
||||
if(!isset($json["response"]["status"])){
|
||||
|
||||
throw new Exception("API did not return a status field");
|
||||
}
|
||||
|
||||
if($json["response"]["status"] != "OK"){
|
||||
|
||||
throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
|
||||
}
|
||||
|
||||
if(!isset($json["response"]["results"])){
|
||||
|
||||
throw new Exception("Mojeek API did not return a results object");
|
||||
}
|
||||
|
||||
foreach($json["response"]["results"] as $entry){
|
||||
|
||||
$out["web"][] = [
|
||||
"title" => html_entity_decode($entry["title"]),
|
||||
"description" => html_entity_decode($entry["desc"]),
|
||||
"url" => $entry["url"],
|
||||
"date" => $entry["timestamp"],
|
||||
"type" => "web",
|
||||
"thumb" => [
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"sublink" => [],
|
||||
"table" => []
|
||||
];
|
||||
}
|
||||
|
||||
// get npt
|
||||
if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
|
||||
|
||||
unset($params["api_key"]);
|
||||
if(!isset($params["s"])){
|
||||
|
||||
$params["s"] = 11;
|
||||
}else{
|
||||
|
||||
$params["s"] = $params["s"] + 10;
|
||||
}
|
||||
|
||||
$out["npt"] =
|
||||
$this->backend->store(
|
||||
json_encode($params),
|
||||
"web",
|
||||
$proxy
|
||||
);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function web_browser($get){
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
[$token, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
@@ -595,6 +457,7 @@ class mojeek{
|
||||
"tn" => 7, // number of news results/page
|
||||
"date" => 1, // show date
|
||||
"tlen" => 128, // max length of title
|
||||
//"dlen" => 511, // max length of description
|
||||
//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
|
||||
];
|
||||
|
||||
@@ -1137,12 +1000,7 @@ class mojeek{
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function news(){
|
||||
|
||||
throw new Exception("Currently unsupported");
|
||||
}
|
||||
|
||||
public function news_browser($get){
|
||||
public function news($get){
|
||||
|
||||
$search = $get["s"];
|
||||
|
||||
|
||||
+1102
File diff suppressed because it is too large
Load Diff
+153
-396
@@ -9,10 +9,6 @@ class startpage{
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
|
||||
include "lib/anubis.php";
|
||||
$this->anubis = new anubis();
|
||||
$this->anubis_error = "all gucci";
|
||||
}
|
||||
|
||||
public function getfilters($page){
|
||||
@@ -248,7 +244,7 @@ class startpage{
|
||||
}
|
||||
}
|
||||
|
||||
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
|
||||
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
@@ -263,27 +259,6 @@ class startpage{
|
||||
$url .= "?" . $get;
|
||||
}
|
||||
|
||||
if($return_cookies){
|
||||
|
||||
$cookies = [];
|
||||
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
|
||||
|
||||
$length = strlen($header);
|
||||
|
||||
$header = explode(":", $header, 2);
|
||||
|
||||
if(trim(strtolower($header[0])) == "set-cookie"){
|
||||
|
||||
$cookie_tmp = explode("=", trim($header[1]), 2);
|
||||
|
||||
$cookies[trim($cookie_tmp[0])] =
|
||||
explode(";", $cookie_tmp[1], 2)[0];
|
||||
}
|
||||
|
||||
return $length;
|
||||
});
|
||||
}
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
// http2 bypass
|
||||
@@ -291,15 +266,6 @@ class startpage{
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
|
||||
$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
|
||||
|
||||
foreach($passed_cookies as $name => $val){
|
||||
|
||||
$header_cookies[] = "$name=$val";
|
||||
}
|
||||
|
||||
$header_cookies = rtrim(implode("; ", $header_cookies));
|
||||
|
||||
if($is_xhr === true){
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
|
||||
@@ -313,7 +279,7 @@ class startpage{
|
||||
"Origin: https://www.startpage.com/",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-origin",
|
||||
@@ -332,7 +298,7 @@ class startpage{
|
||||
"Content-Length: " . strlen($get),
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
@@ -350,7 +316,7 @@ class startpage{
|
||||
"Accept-Encoding: gzip",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Cookie: $header_cookies",
|
||||
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
@@ -376,15 +342,6 @@ class startpage{
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
|
||||
if($return_cookies){
|
||||
|
||||
return [
|
||||
"cookies" => $cookies,
|
||||
"data" => $data
|
||||
];
|
||||
}
|
||||
|
||||
return $data;
|
||||
}
|
||||
|
||||
@@ -392,22 +349,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "web");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -449,11 +398,7 @@ class startpage{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -461,41 +406,12 @@ class startpage{
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// detect captcha even if we passed anubis challenge
|
||||
$this->detect_captcha($html);
|
||||
|
||||
//echo htmlspecialchars($html);
|
||||
|
||||
$this->detect_captcha($html);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
|
||||
@@ -504,14 +420,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to grep JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
//print_r($json);
|
||||
@@ -533,7 +449,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "web", $proxy);
|
||||
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
|
||||
@@ -742,22 +658,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "images");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -772,102 +680,76 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "images",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["size"] != "any"){
|
||||
|
||||
if(
|
||||
$get["size"] == "Small" ||
|
||||
$get["size"] == "Medium" ||
|
||||
$get["size"] == "Large" ||
|
||||
$get["size"] == "Wallpaper"
|
||||
){
|
||||
|
||||
$params["flimgsize"] = $get["size"];
|
||||
}else{
|
||||
|
||||
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["color"] != "any"){
|
||||
|
||||
if($get["color"] == "color"){
|
||||
|
||||
$params["flimgcolor"] = "ic:color";
|
||||
}elseif($get["color"] == "bnw"){
|
||||
|
||||
$params["flimgcolor"] = "ic:gray";
|
||||
}else{
|
||||
|
||||
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$params["flimgtype"] = $get["type"];
|
||||
}
|
||||
|
||||
if($get["license"] != "any"){
|
||||
|
||||
$params["flimglicense"] = $get["license"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "images",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["size"] != "any"){
|
||||
|
||||
if(
|
||||
$get["size"] == "Small" ||
|
||||
$get["size"] == "Medium" ||
|
||||
$get["size"] == "Large" ||
|
||||
$get["size"] == "Wallpaper"
|
||||
){
|
||||
|
||||
$params["flimgsize"] = $get["size"];
|
||||
}else{
|
||||
|
||||
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["color"] != "any"){
|
||||
|
||||
if($get["color"] == "color"){
|
||||
|
||||
$params["flimgcolor"] = "ic:color";
|
||||
}elseif($get["color"] == "bnw"){
|
||||
|
||||
$params["flimgcolor"] = "ic:gray";
|
||||
}else{
|
||||
|
||||
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
|
||||
}
|
||||
}
|
||||
|
||||
if($get["type"] != "any"){
|
||||
|
||||
$params["flimgtype"] = $get["type"];
|
||||
}
|
||||
|
||||
if($get["license"] != "any"){
|
||||
|
||||
$params["flimglicense"] = $get["license"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -887,18 +769,18 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to grep JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "images", $proxy);
|
||||
|
||||
// get images
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
@@ -937,22 +819,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "videos");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -967,72 +841,46 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "video",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["sort"] != "relevance"){
|
||||
|
||||
$params["sort_by"] = $get["sort"];
|
||||
}
|
||||
|
||||
if($get["duration"] != "any"){
|
||||
|
||||
$params["with_duration"] = $get["duration"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "video",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["sort"] != "relevance"){
|
||||
|
||||
$params["sort_by"] = $get["sort"];
|
||||
}
|
||||
|
||||
if($get["duration"] != "any"){
|
||||
|
||||
$params["with_duration"] = $get["duration"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1046,14 +894,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to get JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -1067,7 +915,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "video", $proxy);
|
||||
|
||||
// get results
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
@@ -1123,22 +971,14 @@ class startpage{
|
||||
|
||||
if($get["npt"]){
|
||||
|
||||
$this->anubis_error = "Not ran on pagination";
|
||||
[$post, $proxy] = $this->backend->get($get["npt"], "news");
|
||||
|
||||
$post = json_decode($post, true);
|
||||
$query = $post["query"];
|
||||
$cookies = $post["cookies"];
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$query,
|
||||
true,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$post,
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
@@ -1153,67 +993,41 @@ class startpage{
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "news",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["time"] != "any"){
|
||||
|
||||
$params["with_date"] = $get["time"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
[]
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
|
||||
}
|
||||
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
// detect captcha on that page
|
||||
$this->detect_captcha($html["data"]);
|
||||
|
||||
// solve anubis if need be
|
||||
$cookies = $this->solve_anubis($proxy, $html);
|
||||
|
||||
if($cookies === false){
|
||||
$proxy = $this->backend->get_ip();
|
||||
|
||||
$cookies = [];
|
||||
$html = $html["data"];
|
||||
}else{
|
||||
$params = [
|
||||
"query" => $get["s"],
|
||||
"cat" => "news",
|
||||
"pl" => "opensearch"
|
||||
];
|
||||
|
||||
if($get["nsfw"] == "no"){
|
||||
|
||||
$params["qadf"] = "heavy";
|
||||
}
|
||||
|
||||
if($get["time"] != "any"){
|
||||
|
||||
$params["with_date"] = $get["time"];
|
||||
}
|
||||
|
||||
try{
|
||||
$html = $this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/sp/search",
|
||||
$params,
|
||||
false,
|
||||
false,
|
||||
false,
|
||||
$cookies
|
||||
$params
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
//$html = file_get_contents("scraper/startpage.html");
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch search page");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1227,14 +1041,14 @@ class startpage{
|
||||
) === 0
|
||||
){
|
||||
|
||||
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to get JSON object");
|
||||
}
|
||||
|
||||
$json = json_decode($matches[1], true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
|
||||
throw new Exception("Failed to decode JSON object");
|
||||
}
|
||||
|
||||
$out = [
|
||||
@@ -1244,7 +1058,7 @@ class startpage{
|
||||
];
|
||||
|
||||
// get npt
|
||||
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
|
||||
$out["npt"] = $this->parse_npt($json, "news", $proxy);
|
||||
|
||||
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
|
||||
|
||||
@@ -1287,7 +1101,7 @@ class startpage{
|
||||
return $out;
|
||||
}
|
||||
|
||||
private function parse_npt($json, $pagetype, $proxy, $cookies){
|
||||
private function parse_npt($json, $pagetype, $proxy){
|
||||
|
||||
foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
|
||||
|
||||
@@ -1295,27 +1109,22 @@ class startpage{
|
||||
|
||||
return
|
||||
$this->backend->store(
|
||||
json_encode([
|
||||
"query" =>
|
||||
http_build_query(
|
||||
[
|
||||
"lui" => "english",
|
||||
"language" => "english",
|
||||
"sc" => $json["render"]["search_sc"],
|
||||
"t" => "device",
|
||||
"cat" => $pagetype,
|
||||
"segment" => "startpage.udog", // "original" sometimes
|
||||
"abd" => "0",
|
||||
"abe" => "0",
|
||||
"query" => rawurldecode($json["render"]["query"]),
|
||||
"page" => $page["number"],
|
||||
"qsr" => "all",
|
||||
"qadf" => "none", // @ todo fix (??)
|
||||
...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
|
||||
]
|
||||
),
|
||||
"cookies" => $cookies
|
||||
]),
|
||||
http_build_query(
|
||||
[
|
||||
"lui" => "english",
|
||||
"language" => "english",
|
||||
"sc" => $json["render"]["search_sc"],
|
||||
"t" => "device",
|
||||
"cat" => $pagetype,
|
||||
"segment" => "organic",
|
||||
"abd" => "0",
|
||||
"abe" => "0",
|
||||
"query" => rawurldecode($json["render"]["query"]),
|
||||
"page" => $page["number"],
|
||||
"qsr" => "all",
|
||||
"qadf" => "none" // @ todo fix (??)
|
||||
]
|
||||
),
|
||||
$pagetype,
|
||||
$proxy
|
||||
);
|
||||
@@ -1461,56 +1270,4 @@ class startpage{
|
||||
throw new Exception("Startpage redirected the scraper to an unhandled page");
|
||||
}
|
||||
}
|
||||
|
||||
private function solve_anubis($proxy, $html){
|
||||
|
||||
//
|
||||
// solve anubis challenge, if theres one
|
||||
//
|
||||
$res = null;
|
||||
$cookies = [];
|
||||
|
||||
try{
|
||||
$res = $this->anubis->scrape($html["data"]);
|
||||
|
||||
}catch(Exception $error){
|
||||
|
||||
// do nothang
|
||||
$this->anubis_error = $error->getMessage();
|
||||
}
|
||||
|
||||
if($res !== null){
|
||||
|
||||
// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
|
||||
// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
|
||||
// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
|
||||
// &nonce=3098
|
||||
// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
|
||||
// &elapsedTime=111
|
||||
|
||||
$anubis_response =
|
||||
$this->get(
|
||||
$proxy,
|
||||
"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
|
||||
[
|
||||
"id" => $res["id"],
|
||||
"response" => $res["response"],
|
||||
"nonce" => $res["nonce"],
|
||||
"redir" => "https://www.startpage.com/sp/search",
|
||||
"elapsedTime" => $res["elapsedTime"]
|
||||
],
|
||||
false,
|
||||
false,
|
||||
true,
|
||||
$html["cookies"]
|
||||
);
|
||||
|
||||
// re-fetch search page AGAIN with the anubis bullshit
|
||||
$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
|
||||
|
||||
return $cookies;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
+944
-548
File diff suppressed because it is too large
Load Diff
+30
-2
@@ -87,6 +87,10 @@ $settings = [
|
||||
"value" => "kagi",
|
||||
"text" => "Kagi"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
@@ -141,6 +145,14 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "ghostery",
|
||||
"text" => "Ghostery"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
@@ -219,6 +231,10 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "naver",
|
||||
"text" => "Naver"
|
||||
@@ -235,6 +251,10 @@ $settings = [
|
||||
"value" => "pinterest",
|
||||
"text" => "Pinterest"
|
||||
],
|
||||
[
|
||||
"value" => "cara",
|
||||
"text" => "Cara"
|
||||
],
|
||||
[
|
||||
"value" => "flickr",
|
||||
"text" => "Flickr"
|
||||
@@ -309,6 +329,10 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "naver",
|
||||
"text" => "Naver"
|
||||
@@ -355,10 +379,14 @@ $settings = [
|
||||
"value" => "startpage",
|
||||
"text" => "Startpage"
|
||||
],
|
||||
/*[
|
||||
[
|
||||
"value" => "qwant",
|
||||
"text" => "Qwant"
|
||||
],
|
||||
[
|
||||
"value" => "mojeek",
|
||||
"text" => "Mojeek"
|
||||
],*/
|
||||
],
|
||||
[
|
||||
"value" => "baidu",
|
||||
"text" => "Baidu"
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user