Compare commits

2 Commits
Author SHA1 Message Date
throwaway 0efbb81d23 remove parseInt on password env 2026-08-04 23:32:43 -07:00
throwaway 4b83c47637 read values from env and add Dockerfile 2026-08-04 23:03:47 -07:00
25 changed files with 3341 additions and 4547 deletions
+1
View File
@@ -26,3 +26,4 @@ data/captcha/minecraft/
!banner/*default*
scraper/curlie.html
icons/*
node_modules/
+13 -12
View File
@@ -37,20 +37,21 @@ tl;dr 4get is the best way to browse for shit.
| Brave | Yandex | Vimeo | Brave | Swisscows | DuckDuckGo |
| Yandex | Brave | Sepia Search | Google | | Yandex |
| Google | Google | DuckDuckGo | Yahoo! JAPAN | | Google |
| Google API | Google API | Brave | Startpage | | Qwant |
| Google CSE | Google CSE | Yandex | Baidu | | Marginalia |
| Yahoo! JAPAN | Yahoo! JAPAN | Google | | | YouTube |
| Startpage | Startpage | Yahoo! JAPAN | | | SoundCloud |
| Yep | Naver | Startpage | | | Startpage |
| Mwmbl | Baidu | Naver | | | Kagi |
| Mojeek | Solofield | Baidu | | | |
| Naver | Pinterest | Cc Cc | | | |
| Baidu | Flickr | Purili | | | |
| Cốc Cốc | Pexels | Solofield | | | |
| Google API | Google API | Brave | Startpage | | Startpage |
| Google CSE | Google CSE | Yandex | Qwant | | Kagi |
| Yahoo! JAPAN | Yahoo! JAPAN | Google | Mojeek | | Qwant |
| Startpage | Startpage | Yahoo! JAPAN | Baidu | | Ghostery |
| Qwant | Qwant | Startpage | | | Yep |
| Ghostery | Baidu | Qwant | | | Marginalia |
| Yep | Solofield | Baidu | | | YouTube |
| Mwmbl | Pinterest | Coc Coc | | | SoundCloud |
| Mojeek | Cara | Solofield | | | |
| Baidu | Flickr | | | | |
| Coc Coc | Pexels | | | | |
| Solofield | Pixabay | | | | |
| Marginalia | Unsplash | | | | |
| Purili | 500px | | | | |
| wiby | VSCO | | | | |
| wiby | 500px | | | | |
| | VSCO | | | | |
| | Imgur | | | | |
| | FindThatMeme | | | | |
+3 -1
View File
@@ -15,11 +15,13 @@ class autocomplete{
"yandex" => "https://suggest.yandex.com/suggest-ff.cgi?part={searchTerms}&uil=en&v=3&sn=5&lr=21276&yu=4861394161661655015",
"google" => "https://www.google.com/complete/search?client=mobile-gws-lite&q={searchTerms}",
"qwant" => "https://api.qwant.com/v3/suggest/?q={searchTerms}&client=opensearch",
"yep" => "https://api.yep.com/ac/?query={searchTerms}",
"marginalia" => "https://search.marginalia.nu/suggest/?partial={searchTerms}",
"yt" => "https://suggestqueries-clients6.youtube.com/complete/search?client=youtube&q={searchTerms}",
"sc" => "",
"startpage" => "https://www.startpage.com/suggestions?q={searchTerms}&format=opensearch&segment=startpage.defaultffx&lui=english",
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}"
"kagi" => "https://kagi.com/api/autosuggest?q={searchTerms}",
"ghostery" => "https://ghosterysearch.com/suggest?q={searchTerms}"
];
/*
-6
View File
@@ -152,8 +152,6 @@ This should expose a webservice located at `http://localhost:3000`. I highly rec
If done correctly, you should see the red dot turn green in the extension's icon. You're ready to load pages!
**IMPORTANT:** You will want to turn off search history to protect yourself and your users. Head to `about:preferences` and search for "history". Under the "History" category, pick "Customize category", then click "Choose what you want Firefox to remember". In that sub-menu, untick everything. Make sure to **NOT** turn on the "Always use private browsing mode" option, as it will prevent the 4play extension from creating containers.
## Setting up 4get
Now, you need to tell 4get to use the 4play service. Edit `data/config.php` and change these:
@@ -165,7 +163,3 @@ const FPLAY_EXTERNAL_ENDPOINT = null; # Change null to "https://your-endpoint.co
**IMPORTANT:** By default, `localhost:3030` is what Firefox connects to. The render server is exposed at `localhost:3000`. Don't mix them up, otherwise you will get errors about an "invalid Upgrade header".
If everything was done correctly, you should be seeing results on the Google scraper! You can set proxies in `data/config.php` for it, just like any other scraper.
In case it isn't clear, here is a visual representation of what you want to achieve:
![Network map](https://i.imgur.com/jFjbBtG.png)
+4
View File
@@ -150,3 +150,7 @@ services:
##### Tor
You can route incoming and outgoing requests through tor by following [docker tor documentation](./docker_tor.md)
##### 4play
You can run the 4play sidecar by following the [docker 4play guide](./docker_4play.md)
+48
View File
@@ -0,0 +1,48 @@
### Running 4get and 4play via docker guide
This setup will run both 4get and the 4play sidecar
# Docker compose
```
# docker-compose.yaml
services:
fourget:
image: luuul/4get:latest
restart: unless-stopped
environment:
- FOURGET_PROTO=http
- FOURGET_SERVER_NAME=4get.ca
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
- FOURGET_FPLAY_PASSWORD=cnc
- FOURGET_FPLAY_TIMEOUT=30000
- FOURGET_FPLAY_ENABLE_API=true
ports:
- "80:80"
fourplay:
image: luuul/4get-4play:latest
environment:
- FOURPLAY_PORT=3030
- FOURPLAY_PASSWORD=cnc
- FOURPLAY_COMMAND_TIMEOUT=5000
ports:
- "3000:3000"
- "3030:3030"
```
There is a example docker compose file available [here](../extra/4play/docker-compose.yaml)
# Steps
1. start the services with
```
docker compose up -d
```
2. configure firefox
folllow the steps in [configure](./configure.md) for firefox setup
+19
View File
@@ -0,0 +1,19 @@
FROM alpine:3.21
WORKDIR /app
RUN apk update && apk upgrade
RUN apk add nodejs npm
COPY package*.json ./
RUN npm ci --only=production && npm cache clean --force
COPY . .
EXPOSE 3030
ENTRYPOINT ["node"]
CMD ["page-render.js"]
+24
View File
@@ -0,0 +1,24 @@
# docker-compose.yaml
services:
fourget:
image: luuul/4get:latest
restart: unless-stopped
environment:
- FOURGET_PROTO=http
- FOURGET_SERVER_NAME=4get.ca
- FOURGET_FPLAY_EXTERNAL_ENDPOINT=http://fourplay:3000
- FOURGET_FPLAY_PASSWORD=cnc
- FOURGET_FPLAY_TIMEOUT=30000
- FOURGET_FPLAY_ENABLE_API=true
ports:
- "80:80"
fourplay:
image: luuul/4get-4play:latest
environment:
- FOURPLAY_PORT=3030
- FOURPLAY_PASSWORD=cnc
- FOURPLAY_COMMAND_TIMEOUT=5000
ports:
- "3000:3000"
- "3030:3030"
+47
View File
@@ -0,0 +1,47 @@
{
"name": "4play",
"version": "1.0.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "4play",
"version": "1.0.0",
"license": "AGPL-3.0-only",
"dependencies": {
"@lawlers/4play": "^1.2.5",
"ws": "^8.21.2"
}
},
"node_modules/@lawlers/4play": {
"version": "1.2.5",
"resolved": "https://registry.npmjs.org/@lawlers/4play/-/4play-1.2.5.tgz",
"integrity": "sha512-mHxCROszXwW3XWdE+OQldJcAo3Qs9bI20VME/ZiPUvz3G2kdJaDbrMSzSwVlHmju+3dydupIE48t/ektKtewhg==",
"license": "AGPL-3.0-only",
"dependencies": {
"ws": "^8.21.1"
}
},
"node_modules/ws": {
"version": "8.21.2",
"resolved": "https://registry.npmjs.org/ws/-/ws-8.21.2.tgz",
"integrity": "sha512-54dMVAo4WIe6SKy3vBgN+9bJZqqQ8IMRevAkOLQALhi49qkkQDQfWdAZ8KQlXiEabw88ARXXdUrlvtbKQX+aKw==",
"license": "MIT",
"engines": {
"node": ">=10.0.0"
},
"peerDependencies": {
"bufferutil": "^4.0.1",
"utf-8-validate": ">=5.0.2"
},
"peerDependenciesMeta": {
"bufferutil": {
"optional": true
},
"utf-8-validate": {
"optional": true
}
}
}
}
}
+20
View File
@@ -0,0 +1,20 @@
{
"name": "4play",
"version": "1.0.0",
"description": "",
"repository": {
"type": "git",
"url": "https://git.lolcat.ca/lolcat/4get/"
},
"license": "AGPL-3.0-only",
"author": "lolcat",
"type": "commonjs",
"main": "page-render.js",
"scripts": {
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
"@lawlers/4play": "^1.2.5",
"ws": "^8.21.2"
}
}
+3 -3
View File
@@ -2,9 +2,9 @@ const fplay = require("@lawlers/4play");
const http = require("http");
// 4play settings
var port = 3030;
var timeout = 30000;
var password = "cnc";
var port = parseInt(process.env.FOURGET_FPLAY_PASSWORD) || 3030;
var timeout = parseInt(process.env.FOURGET_FPLAY_TIMEOUT) || 30000;
var password = process.env.FOURGET_FPLAY_PASSWORD || "cnc";
// ==================================================
-151
View File
@@ -1,151 +0,0 @@
<?php
$scrapers = [
"web" => [
"DuckDuckGo",
"Brave",
"Yandex",
"Google",
"Google API",
"Google CSE",
"Yahoo! JAPAN",
"Startpage",
"Yep",
"Mwmbl",
"Mojeek",
"Naver",
"Baidu",
"Cốc Cốc",
"Solofield",
"Marginalia",
"Purili",
"wiby"
],
"images" => [
"DuckDuckGo",
"Yandex",
"Brave",
"Google",
"Google API",
"Google CSE",
"Yahoo! JAPAN",
"Startpage",
"Naver",
"Baidu",
"Solofield",
"Pinterest",
"Flickr",
"Pexels",
"Pixabay",
"Unsplash",
"500px",
"VSCO",
"Imgur",
"FindThatMeme"
],
"videos" => [
"YouTube",
"Vimeo",
"Sepia Search",
"DuckDuckGo",
"Brave",
"Yandex",
"Google",
"Yahoo! JAPAN",
"Startpage",
"Naver",
"Baidu",
"Cốc Cốc",
"Purili",
"Solofield"
],
"news" => [
"DuckDuckGo",
"Brave",
"Google",
"Yahoo! JAPAN",
"Startpage",
"Baidu"
],
"music" => [
"SoundCloud",
"Swisscows"
],
"autocomplete" => [
"Brave",
"DuckDuckGo",
"Yandex",
"Google",
"Qwant",
"Marginalia",
"YouTube",
"SoundCloud",
"Startpage",
"Kagi"
]
];
$lines = [
0 => "",
1 => ""
];
// get longest scraper string
// also get list of scrapers with most entries
$biggest_category_len = 0;
foreach($scrapers as $category_name => $category){
$count = count($category);
if($count > $biggest_category_len){
$biggest_category_len = $count;
}
}
foreach($scrapers as $category_name => $category){
// get longest string of the category
$longest_len = mb_strlen($category_name);
foreach($category as $scraper){
$strlen = mb_strlen($scraper);
if($strlen > $longest_len){
$longest_len = $strlen;
}
}
// add header
$lines[0] .= mb_str_pad("| {$category_name}", $longest_len + 3);
$lines[1] .= "|" . str_repeat("-", $longest_len + 2);
for($i=2; $i<count($category) + 2; $i++){
if(!isset($lines[$i])){
$lines[$i] = "";
}
$lines[$i] .= mb_str_pad("| {$category[$i - 2]}", $longest_len + 3);
}
if($i - 2 < $biggest_category_len){
for($k=$i; $k<$biggest_category_len + 2; $k++){
if(!isset($lines[$k])){
$lines[$k] = "";
}
$lines[$k] .= "|" . str_repeat(" ", $longest_len + 2);
}
}
}
foreach($lines as $line){
echo $line . "|\n";
}
+5 -8
View File
@@ -45,15 +45,15 @@ class anubis{
if(
!isset($script["challenge"]) ||
!is_string($script["challenge"]["randomData"]) ||
!isset($script["rules"]["difficulty"]) ||
!is_int($script["rules"]["difficulty"])
!is_int($script["rules"]["difficulty"]) ||
!is_string($script["challenge"])
){
throw new Exception("Found invalid challenge data");
}
return $this->rape($script["challenge"]["id"], $script["challenge"]["randomData"], $script["rules"]["difficulty"]);
return $this->rape($script["challenge"], $script["rules"]["difficulty"]);
}
private function is_valid_hash($hash, $difficulty){
@@ -74,10 +74,9 @@ class anubis{
return true;
}
public function rape($id, $data, $difficulty = 5){
public function rape($data, $difficulty = 5){
$nonce = 0;
$microtime = microtime(true);
while(true){
@@ -88,12 +87,10 @@ class anubis{
$hash_hex = bin2hex($hash_binary);
return [
"id" => $id,
"response" => $hash_hex,
//"data" => $data,
//"difficulty" => $difficulty,
"nonce" => $nonce,
"elapsedTime" => round((microtime(true) - $microtime) * 1000)
"nonce" => $nonce
];
}
+6 -1
View File
@@ -644,6 +644,7 @@ class frontend{
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"yep" => "Yep",
"mwmbl" => "Mwmbl",
"mojeek" => "Mojeek",
@@ -670,10 +671,12 @@ class frontend{
"google_cse" => "Google CSE",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"naver" => "Naver",
"baidu" => "Baidu",
"solofield" => "Solofield",
"pinterest" => "Pinterest",
"cara" => "Cara",
"flickr" => "Flickr",
"pexels" => "Pexels",
"pixabay" => "Pixabay",
@@ -702,6 +705,7 @@ class frontend{
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
"qwant" => "Qwant",
"naver" => "Naver",
"baidu" => "Baidu",
"coccoc" => "Cốc Cốc",
@@ -720,7 +724,8 @@ class frontend{
"google" => "Google",
"yahoo_japan" => "Yahoo! JAPAN",
"startpage" => "Startpage",
//"mojeek" => "Mojeek",
"qwant" => "Qwant",
"mojeek" => "Mojeek",
"baidu" => "Baidu"
]
];
+2 -2
View File
@@ -919,8 +919,8 @@ class baidu{
$description =
$this->fuckhtml
->getElementsByAttributeValue(
"data-module",
"abstract",
"data-sanssr-cmpt",
"card/www-summary-1",
$div
);
-39
View File
@@ -383,8 +383,6 @@ class brave{
// load html
$this->fuckhtml->load($html);
$this->detect_captcha($html);
/*
Get next page "token"
*/
@@ -1177,8 +1175,6 @@ class brave{
// load html
$this->fuckhtml->load($html);
$this->detect_captcha($html);
// get npt
$out["npt"] =
$this->generatenextpagetoken(
@@ -1274,9 +1270,6 @@ class brave{
fclose($handle);*/
$this->fuckhtml->load($html);
$this->detect_captcha($html);
$json = $this->get_js();
foreach(
@@ -1381,8 +1374,6 @@ class brave{
$this->fuckhtml->load($html);
$this->detect_captcha($html);
$out = [
"status" => "ok",
"npt" => null,
@@ -1862,34 +1853,4 @@ class brave{
)
);
}
private function detect_captcha(){
$title =
$this->fuckhtml
->getElementsByTagName(
"title"
);
if(count($title) === 0){
if(strtolower($this->fuckhtml->getloadedhtml()) == "this service is not available in your region"){
throw new Exception("Brave rangebanned the IP range");
}
throw new Exception("Brave returned a malformed page");
}
$title =
strtolower(
$this->fuckhtml
->getTextContent($title[0])
);
if($title == "human verification"){
throw new Exception("Brave returned a Captcha");
}
}
}
+847
View File
@@ -0,0 +1,847 @@
<?php
class cara{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("cara");
}
public function getfilters($page){
return [
"sort" => [
"display" => "Sort by",
"option" => [
"Top" => "Top",
"MostRecent" => "Most Recent"
]
],
"type" => [
"display" => "Post type",
"option" => [
"any" => "Any type",
"portfolio" => "Portfolio", // {"posts":["portfolio"]}
"timeline" => "Timeline" // {"posts":["timeline"]}
]
],
"fields" => [
"display" => "Field/Medium",
"option" => [
"any" => "Any field",
"2D" => "2D Work",
"3D" => "3D Work",
"3DPrinting" => "3D Printing",
"Acrylic" => "Acrylic",
"AlcoholMarkers" => "Alcohol Markers",
"Animation" => "Animation",
"Chalk" => "Chalk",
"Charcoal" => "Charcoal",
"Colored pencil" => "Colored pencil",
"Conte" => "Conte",
"Crayon" => "Crayon",
"Digital" => "Digital",
"Gouache" => "Gouache",
"Ink" => "Ink",
"MixedMedia" => "Mixed-Media",
"Oil" => "Oil",
"Oil-based Markers" => "Oil-based Markers",
"Other" => "Other",
"Pastels" => "Pastels",
"Photography" => "Photography",
"Sculpture" => "Sculpture",
"Sketches" => "Sketches",
"Tattoos" => "Tattoos",
"Traditional" => "Traditional",
"VFX" => "VFX",
"Watercolor" => "Watercolor"
]
],
"category" => [
"display" => "Category",
"option" => [
"any" => "Any category",
"3DScanning" => "3D Scanning",
"Abstract" => "Abstract",
"Adoptable" => "Adoptable",
"Anatomy" => "Anatomy",
"Animals" => "Animals",
"Anime" => "Anime",
"App" => "App",
"ArchitecturalConcepts" => "Architectural Concepts",
"ArchitecturalVisualization" => "Architectural Visualization",
"AugmentedReality" => "Augmented Reality",
"Automotive" => "Automotive",
"BoardGameArt" => "Board Game Art",
"BookIllustration" => "Book Illustration",
"CardGameArt" => "Card Game Art",
"CeramicsPottery" => "Ceramics/Pottery",
"CharacterAnimation" => "Character Animation",
"CharacterDesign" => "Character Design",
"CharacterModeling" => "Character Modeling",
"ChildrensArt" => "Children's Illustration",
"Collectibles" => "Collectibles",
"ColoringPage" => "Coloring Page",
"ComicArt" => "Comic Art",
"ConceptArt" => "Concept Art",
"Cosplay" => "Cosplay",
"CostumeDesign" => "Costume Design",
"CoverArt" => "Cover Art",
"Creatures" => "Creatures",
"Diorama" => "Diorama",
"EditorialIllustration" => "Editorial Illustration",
"EmbroiderySewing" => "Embroidery/Sewing",
"EnvironmentalConceptArt" => "Environmental Concept Art",
"EnvironmentalConceptDesign" => "Environmental Concept Design",
"FanArt" => "Fan Art",
"Fantasy" => "Fantasy",
"Fashion" => "Fashion",
"FashionStyling" => "Fashion Styling",
"FiberArts" => "Fiber Arts",
"Furry" => "Furry",
"GameArt" => "Game Art",
"GameplayDesign" => "Gameplay Design",
"GamesEnvironmentArt" => "Games Environment Art",
"Gem" => "Gem",
"GraphicDesign" => "Graphic Design",
"Handicraft" => "Handicraft",
"HairStyling" => "Hair Styling",
"HardSurface" => "Hard Surface",
"Horror" => "Horror",
"Illustration" => "Illustration",
"IllustrationVisualization" => "Illustration Visualization",
"IndustrialDesign" => "Industrial Design",
"Jewelry" => "Jewelry",
"KnittingCrochet" => "Knitting/Crochet",
"Landscape" => "Landscape",
"LevelDesign" => "Level Design",
"Lighting" => "Lighting",
"Makeup" => "Makeup",
"Manga" => "Manga",
"MapsCartography" => "Maps/Cartography",
"MattePainting" => "Matte Painting",
"Materials" => "Materials",
"MechanicalDesign" => "Mechanical Design",
"Medical" => "Medical",
"Mecha" => "Mecha",
"MiniatureArt" => "Miniature Art",
"MotionGraphics" => "Motion Graphics",
"FrescoMurals" => "Fresco/Murals",
"Natural" => "Natural",
"Original Character" => "Original Character",
"Overlay" => "Overlay",
"PleinAir" => "Plein Air",
"Photogrammetry" => "Photogrammetry",
"PixelArt" => "Pixel Art",
"Portraits" => "Portraits",
"Props" => "Props",
"ProductDesign" => "Product Design",
"PublicDomain" => "Public Domain or Royalty Free",
"Real-Time3DEnvironmentArt" => "Real-Time 3D Environment Art",
"Realism" => "Realism",
"ScienceFiction" => "Science Fiction",
"ScientificVisualization" => "Scientific Visualization",
"Scripts" => "Scripts",
"StillLife" => "Still Life",
"Storyboards" => "Storyboards",
"Stylized" => "Stylized",
"Surreal" => "Surreal",
"TechnicalArt" => "Technical Art",
"Textures" => "Textures",
"Tools" => "Tools",
"Toys" => "Toys",
"ToyPackaging" => "Toy Packaging",
"Tutorials" => "Tutorials",
"UIArt" => "User Interface (UI) Art",
"UrbanSketch" => "Urban Sketch",
"VFXforAnimation" => "VFX for Animation",
"VFXforFilm" => "VFX for Film",
"VFXforGames" => "VFX for Games",
"VFXforRealTime" => "VFX for Real-Time",
"VFXforTV" => "VFX for TV",
"Vehicles" => "Vehicles",
"VirtualReality" => "Virtual Reality",
"VisualDevelopment" => "Visual Development",
"VoxelArt" => "Voxel Art",
"Vtubers" => "Vtubers",
"WIP" => "WIP (Work in Progress)",
"Web" => "Web",
"Weapons" => "Weapons",
"Wildlife" => "Wildlife",
"Woodcutting" => "Woodcutting"
]
],
"software" => [
"display" => "Software",
"option" => [
"any" => "Any software",
"123D" => "123D",
"123DCatch" => "123D Catch",
"3DBee" => "3DBee",
"3DCoat" => "3DCoat",
"3DCoatPrint" => "3DCoatPrint",
"3DCoatTextura" => "3DCoatTextura",
"3DEqualizer" => "3DEqualizer",
"3DFZephyr" => "3DF Zephyr",
"3Delight" => "3Delight",
"3dpeople" => "3dpeople",
"3dsMax" => "3ds Max",
"3DSPaint" => "3DS Paint",
"ACDSeeCanvas" => "ACDSee Canvas",
"AbletonLive" => "Ableton Live",
"Acrobat" => "Acrobat",
"AdobeDraw" => "Adobe Draw",
"AdobeFlash" => "Adobe Flash",
"AdobeFresco" => "Adobe Fresco",
"AdobeSubstance3Dassets" => "Adobe Substance 3D assets",
"AdobeXD" => "Adobe XD",
"AffinityDesigner" => "Affinity Designer",
"AffinityPhoto" => "Affinity Photo",
"AfterEffects" => "After Effects",
"Akeytsu" => "Akeytsu",
"Alchemy" => "Alchemy",
"AliasDesign" => "Alias Design",
"AlightMotion" => "Alight Motion",
"Amadine" => "Amadine",
"Amberlight" => "Amberlight",
"Animate" => "Animate",
"AnimationMaster" => "Animation:Master",
"AnimeStudio" => "Anime Studio",
"Apophysis" => "Apophysis",
"ArchiCAD" => "ArchiCAD",
"Arion" => "Arion",
"ArionFX" => "ArionFX",
"Arnold" => "Arnold",
"ArtEngine" => "ArtEngine",
"ArtFlow" => "ArtFlow",
"ArtRage" => "ArtRage",
"ArtstudioPro" => "Artstudio Pro",
"Artweaver" => "Artweaver",
"Aseprite" => "Aseprite",
"Audition" => "Audition",
"AutoCAD" => "AutoCAD",
"AutodeskSketchBook" => "Autodesk SketchBook",
"AvidMediaComposer" => "Avid Media Composer",
"AzPainter" => "AzPainter",
"babylonjs" => "babylon.js",
"BalsamiqMockup" => "Balsamiq Mockup",
"Bforartists" => "Bforartists",
"BlackInk" => "Black Ink",
"BlackmagicDesignFusion" => "Blackmagic Design Fusion",
"Blender" => "Blender",
"Blender DeepPaint" => "Blender DeepPaint",
"BlenderGreasePencil" => "Blender Grease Pencil",
"Blockbench" => "Blockbench",
"BodyPaint" => "BodyPaint",
"Boxcutter" => "Boxcutter",
"BraidMaker" => "Braid Maker",
"BrickLinkStudio" => "BrickLink Studio",
"Bridge" => "Bridge",
"Brushifyio" => "Brushify.io",
"C" => "C",
"C#" => "C#",
"C++" => "C++",
"CACANi" => "CACANi",
"CLIPSTUDIOPAINT" => "CLIP STUDIO PAINT",
"CLO" => "CLO",
"CRYENGINE" => "CRYENGINE",
"Callipeg" => "Callipeg",
"Canva" => "Canva",
"CaptureOne" => "Capture One",
"CartoonAnimator" => "Cartoon Animator",
"Carveco" => "Carveco",
"Cavalry" => "Cavalry",
"Chaotica" => "Chaotica",
"CharacterAnimator" => "Character Animator",
"CharacterCreator" => "Character Creator",
"Cinema4D" => "Cinema 4D",
"ClarisseiFX" => "Clarisse iFX",
"Coiffure" => "Coiffure",
"ColorsLive" => "Colors Live",
"Combustion" => "Combustion",
"Construct2" => "Construct 2",
"Core" => "Core",
"CorelPainter" => "Corel Painter",
"CorelDRAWGraphicsSuite" => "CorelDRAW Graphics Suite",
"CoronaRenderer" => "Corona Renderer",
"ProMotionNG" => "Cosmigo Pro Motion NG",
"CrazyBump" => "CrazyBump",
"Crocotile3D" => "Crocotile 3D",
"Curvy3D" => "Curvy 3D",
"Cycles4D" => "Cycles 4D",
"Darkroom" => "Darkroom",
"DAZStudio" => "DAZ Studio",
"DDO" => "DDO",
"DECIMA" => "DECIMA",
"Darktable" => "Darktable",
"DaVinciResolve" => "DaVinci Resolve",
"Dimension" => "Dimension",
"DragonBones" => "DragonBones",
"Dragonframe" => "Dragonframe",
"Drawpile" => "Drawpile",
"Dreams" => "Dreams",
"Dreamweaver" => "Dreamweaver",
"DxOPhotoLab" => "DxO PhotoLab",
"ECycles" => "E-Cycles",
"EmberGen" => "EmberGen",
"Encore" => "Encore",
"Expresii" => "Expresii",
"FStorm" => "FStorm",
"FadeIn" => "FadeIn",
"Feather3D" => "Feather 3D",
"FiberShop" => "FiberShop",
"Figma" => "Figma",
"FilmoraWondershare" => "Filmora Wondershare",
"FilterForge" => "Filter Forge",
"FinalCutPro" => "Final Cut Pro",
"FinalDraft" => "Final Draft",
"finalRender" => "finalRender",
"FireAlpaca" => "FireAlpaca",
"Fireworks" => "Fireworks",
"FlamePainter" => "Flame Painter",
"Flash" => "Flash",
"FlipaClip" => "FlipaClip",
"FlipnoteStudio" => "Flipnote Studio",
"Fluent" => "Fluent",
"ForestPack" => "Forest Pack",
"FormZ" => "Form-Z",
"Fractorium" => "Fractorium",
"FreeCAD" => "FreeCAD",
"FreeHand" => "FreeHand",
"Forger" => "Forger",
"FrostbiteEngine" => "Frostbite Engine",
"fSpy" => "fSpy",
"FumeFX" => "FumeFX",
"Fusion360" => "Fusion 360",
"GIMP" => "GIMP",
"GSCurveTools" => "GS CurveTools",
"GSToolbox" => "GS Toolbox",
"Gaea" => "Gaea",
"GameTextures" => "Game Textures",
"GameMakerStudio" => "GameMaker: Studio",
"GarageFarmNET" => "GarageFarm.NET",
"GeoGlyph" => "GeoGlyph",
"GigapixelAl" => "Gigapixel Al",
"Glaxnimate" => "Glaxnimate",
"GnomePaint" => "Gnome Paint",
"Godot" => "Godot",
"Goxel" => "Goxel",
"Graphite" => "Graphite",
"Graswald" => "Graswald",
"GravitySketch" => "Gravity Sketch",
"GuerillaRender" => "GuerillaRender",
"HDRLightStudio" => "HDR Light Studio",
"HairStrandDesigner" => "Hair Strand Designer",
"HairTGHairFur" => "HairTG - Hair &amp; Fur",
"HairTGSurfaceFeatherEdition" => "HairTG - Surface, Feather Edition",
"HairTGSurfaceHairEdition" => "HairTG - Surface, Hair Edition",
"Handplane" => "Handplane",
"Hansoft" => "Hansoft",
"HardOps" => "Hard Ops",
"HardMesh" => "HardMesh",
"Harmony" => "Harmony",
"HeavypaintWebbypaint" => "Heavypaint/Webbypaint",
"HelloPaint" => "HelloPaint",
"HeliconFocus" => "Helicon Focus",
"Hexels" => "Hexels",
"HiPaint" => "HiPaint",
"Houdini" => "Houdini",
"HydraRenderer" => "Hydra Renderer",
"iArtbook" => "iArtbook",
"IbisPaint" => "ibisPaint",
"Ideas" => "Ideas",
"IllustStudio" => "Illust Studio",
"Illustrator" => "Illustrator",
"IllustratorDraw" => "Illustrator Draw",
"InDesign" => "InDesign",
"Inochi2D" => "Inochi2D",
"InVision" => "InVision",
"InVisionCraft" => "InVision Craft",
"InfinitePainter" => "Infinite Painter",
"Inkscape" => "Inkscape",
"Inspirit" => "Inspirit",
"InstaLOD" => "InstaLOD",
"InstaMAT" => "InstaMAT",
"InstantLightRealtimePBR" => "Instant Light Realtime PBR",
"InstantMeshes" => "Instant Meshes",
"InstantTerra" => "Instant Terra",
"Inventor" => "Inventor",
"Iray" => "Iray",
"JWildfire" => "JWildfire",
"Java" => "Java",
"Jira" => "Jira",
"JumpPaint" => "Jump Paint by MediBang",
"JSPaint" => "JS Paint",
"Katana" => "Katana",
"Keyshot" => "Keyshot",
"KidPix" => "Kid Pix",
"KitBash3D" => "KitBash3D",
"Knald" => "Knald",
"Kodon" => "Kodon",
"KolourPaint" => "KolourPaint",
"Krakatoa" => "Krakatoa",
"KRESKA" => "KRESKA",
"Krita" => "Krita",
"LensStudio" => "Lens Studio",
"LibreSprite" => "LibreSprite",
"LightWave3D" => "LightWave 3D",
"Lightroom" => "Lightroom",
"Linearity" => "Linearity",
"LiquiGen" => "LiquiGen",
"Live2DCubism" => "Live2D Cubism",
"LookatmyHair" => "Look at my Hair",
"Lotpixel" => "Lotpixel",
"Lumion" => "Lumion",
"LuxRender" => "LuxRender",
"MacPaint" => "MacPaint",
"MagicaCSG" => "MagicaCSG",
"MagicaVoxel" => "MagicaVoxel",
"Magma" => "Magma",
"MakeHuman" => "MakeHuman",
"Malmal" => "Malmal",
"Mandelbulb3D" => "Mandelbulb 3D",
"Mandelbulber" => "Mandelbulber",
"MangaStudio" => "Manga Studio",
"Mari" => "Mari",
"MarmosetToolbag" => "Marmoset Toolbag",
"MarvelousDesigner" => "Marvelous Designer",
"MasterpieceStudioPro" => "Masterpiece Studio Pro",
"MasterpieceVR" => "MasterpieceVR",
"Maverick" => "Maverick",
"MaxwellRender" => "Maxwell Render",
"Maya" => "Maya",
"MediBangPaint" => "MediBang Paint",
"MediumbyAdobe" => "Medium by Adobe",
"Megascans" => "Megascans",
"mentalray" => "mental ray",
"MeshLab" => "MeshLab",
"Meshroom" => "Meshroom",
"MetaHumanCreator" => "MetaHuman Creator",
"Metashape" => "Metashape",
"MightyBake" => "MightyBake",
"MikuMikuDance" => "MikuMikuDance",
"Minecraft" => "Minecraft",
"Mischief" => "Mischief",
"Mixamo" => "Mixamo",
"Mixer" => "Mixer",
"MoI3D" => "MoI3D",
"Mocha" => "Mocha",
"Modo" => "Modo",
"Moho" => "Moho",
"MotionBuilder" => "MotionBuilder",
"Mudbox" => "Mudbox",
"Muse" => "Muse",
"MSPaint" => "MS Paint",
"MyPaint" => "MyPaint",
"NDO" => "NDO",
"NX" => "NX",
"NdotCAD" => "NdotCAD",
"NintendoNotes" => "Nintendo Notes",
"NomadSculpt" => "Nomad Sculpt",
"Notability" => "Notability",
"Nuke" => "Nuke",
"Nvil" => "Nvil",
"OctaneRender" => "Octane Render",
"Omniverse" => "Omniverse",
"OmniverseCreate" => "Omniverse Create",
"ON1PhotoRAW" => "ON1 Photo RAW",
"Open3DEngine" => "Open 3D Engine",
"OpenCanvas" => "OpenCanvas",
"OpenGL" => "OpenGL",
"OpenToonz" => "OpenToonz",
"Ornatrix" => "Ornatrix",
"OsciRender" => "Osci-Render",
"OurPaint" => "Our Paint",
"PBRMAX" => "PBRMAX",
"PFTrack" => "PFTrack",
"PTGui" => "PTGui",
"Paintbrush" => "Paintbrush",
"PaintNET" => "Paint.NET",
"PaintShopPro" => "PaintShop Pro",
"PaintToolSAI" => "Paint Tool SAI",
"PaintstormStudio" => "Paintstorm Studio",
"Paper" => "Paper",
"Pencil2D" => "Pencil2D",
"Penpot" => "Penpot",
"PhoenixFD" => "Phoenix FD",
"Phonto" => "Phonto",
"PhotoLab2" => "PhotoLab 2",
"Photopea" => "Photopea",
"Photoscan" => "Photoscan",
"Photoshop" => "Photoshop",
"PhotoshopElements" => "Photoshop Elements",
"PicoCAD" => "picoCAD",
"PicoCAD2" => "picoCAD 2",
"Pinta" => "Pinta",
"Piskel" => "Piskel",
"Pixilart" => "Pixilart",
"Pixelitor" => "Pixelitor",
"Pixelmator" => "Pixelmator",
"Pixelorama" => "Pixelorama",
"PixivSketch" => "pixiv Sketch",
"Pixquare" => "Pixquare",
"PlantCatalog" => "PlantCatalog",
"PlantFactory" => "PlantFactory",
"Plasticity" => "Plasticity",
"PNGtuberPlus" => "PNGtuber Plus",
"Poliigon" => "Poliigon",
"Polybrush" => "Polybrush",
"PopcornFx" => "PopcornFx",
"Poser" => "Poser",
"Premiere" => "Premiere",
"PremiereElements" => "Premiere Elements",
"PresagisCreator" => "Presagis Creator",
"ProTools" => "Pro Tools",
"Procreate" => "Procreate",
"ProcreateDreams" => "Procreate Dreams",
"Producer" => "Producer",
"PrometheanAI" => "Promethean AI",
"PureRef" => "PureRef",
"Python" => "Python",
"PyxelEdit" => "PyxelEdit",
"QuadRemesher" => "Quad Remesher",
"QuarkXPress" => "QuarkXPress",
"Qubicle" => "Qubicle",
"Quill" => "Quill",
"QuixelBridge" => "Quixel Bridge",
"QuixelMegascans" => "Quixel Megascans",
"QuixelMixer" => "Quixel Mixer",
"QuixelSuite" => "Quixel Suite",
"R3DSWrap" => "R3DS Wrap",
"R3DSZWRAP" => "R3DS ZWRAP",
"RDTextures" => "RD-Textures",
"RailClone" => "RailClone",
"RealFlow" => "RealFlow",
"RealisticPaintStudio" => "Realistic Paint Studio",
"RealityCapture" => "RealityCapture",
"RealityScan" => "RealityScan",
"RealtimeBoard" => "Realtime Board",
"Rebelle" => "Rebelle",
"Redshift" => "Redshift",
"RenderMan" => "RenderMan",
"RenderNetwork" => "Render Network",
"Revit" => "Revit",
"Rhino" => "Rhino",
"Rhinoceros" => "Rhinoceros",
"RizomUV" => "RizomUV",
"RoughAnimator" => "Rough Animator",
"SamsungNotes" => "Samsung Notes",
"SamsungPENUP" => "Samsung PENUP",
"ScansLibrary" => "ScansLibrary",
"Scrivener" => "Scrivener",
"Sculpt+" => "Sculpt+",
"Sculptris" => "Sculptris",
"ShaveandaHaircut" => "Shave and a Haircut",
"ShiVa3D" => "ShiVa3D",
"Shotgun" => "Shotgun",
"Silo" => "Silo",
"Silugen" => "Silugen",
"Sketch" => "Sketch",
"SketchApp" => "Sketch App",
"SketchBookPro" => "SketchBook Pro",
"SketchClub" => "SketchClub",
"SketchUp" => "SketchUp",
"Sketchable" => "Sketchable",
"Sketchfab" => "Sketchfab",
"Skyshop" => "Skyshop",
"Snapseed" => "Snapseed",
"Snowdrop" => "Snowdrop",
"Softimage" => "Softimage",
"SolidWorks" => "SolidWorks",
"SonySketch" => "Sony Sketch",
"Soundbooth" => "Soundbooth",
"Source2" => "Source 2",
"SourceControl" => "Source Control",
"SourceFilmmaker" => "Source Filmmaker",
"SpeedTree" => "SpeedTree",
"Speedgrade" => "Speedgrade",
"SpeedyPainter" => "SpeedyPainter",
"Spine2D" => "Spine 2D",
"Spriter" => "Spriter",
"Stingray" => "Stingray",
"Storyboarder" => "Storyboarder",
"StoryboardPro" => "Storyboard Pro",
"SublimeText" => "Sublime Text",
"Substance3DDesigner" => "Substance 3D Designer",
"Substance3DModeler" => "Substance 3D Modeler",
"Substance3DPainter" => "Substance 3D Painter",
"Substance3DSampler" => "Substance 3D Sampler",
"Substance3DStager" => "Substance 3D Stager",
"SubstanceB2M" => "Substance B2M",
"SweetHome3D" => "Sweet Home 3D",
"SynthEyes" => "SynthEyes",
"TTools" => "TTools",
"TVPaint" => "TVPaint",
"TVPaintAnimation" => "TVPaint Animation",
"TayasuiSketches" => "Tayasui Sketches",
"TayasuiSketchesMobileApp" => "Tayasui Sketches Mobile App",
"TayasuiSketchesPro" => "Tayasui Sketches Pro",
"Terragen" => "Terragen",
"Texturescom" => "Textures.com",
"Texturingxyz" => "Texturingxyz",
"TeyaConceptor" => "Teya Conceptor",
"TheGrove3D" => "The Grove 3D",
"TheaRender" => "Thea Render",
"Threejs" => "Three.js",
"Tiled" => "Tiled",
"TiltBrush" => "Tilt Brush",
"Tooll3" => "Tooll3",
"ToonBoomHarmony" => "Toon Boom Harmony",
"ToonBoomStudio" => "Toon Boom Studio",
"ToonSquid" => "ToonSquid",
"TopoGun" => "TopoGun",
"TuxPaint" => "Tux Paint",
"Tvori" => "Tvori",
"Twinmotion" => "Twinmotion",
"UNIGINEEngine" => "UNIGINE Engine",
"UVLayout" => "UVLayout",
"UltraFractal" => "Ultra Fractal",
"uMake" => "uMake",
"Unfold3D" => "Unfold 3D",
"Unity" => "Unity",
"UnrealEngine" => "Unreal Engine",
"Vengi" => "vengi",
"VRay" => "V-Ray",
"VRED" => "VRED",
"VTubeStudio" => "VTube Studio",
"Vectary" => "Vectary",
"VectorayGen" => "VectorayGen",
"Vectorworks" => "Vectorworks",
"VegasPro" => "Vegas Pro",
"VisualDesigner3D" => "Visual Designer 3D",
"VisualStudio" => "Visual Studio",
"VRoidStudio" => "VRoid Studio",
"Vue" => "Vue",
"Vuforia" => "Vuforia",
"WebGL" => "WebGL",
"WhiteboardFox" => "Whiteboard Fox",
"WickEditor" => "Wick Editor",
"Wings3D" => "Wings 3D",
"Word" => "Word",
"WorldCreator" => "World Creator",
"WorldMachine" => "World Machine",
"XParticles" => "X-Particles",
"Xfrog" => "Xfrog",
"Xgen" => "Xgen",
"xNormal" => "xNormal",
"xTex" => "xTex",
"XoliulShader" => "Xoliul Shader",
"Yafaray" => "Yafaray",
"Yeti" => "Yeti",
"ZBrush" => "ZBrush",
"ZBrushCore" => "ZBrushCore",
"ZenBrush" => "Zen Brush"
]
]
];
}
private function get($proxy, $url, $get = [], $search){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: application/json, text/plain, */*",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
//"sentry-trace: 72b0318a7141fe18cbacbd905572eddf-a60de161b66b1e6f-1
//"baggage: sentry-environment=vercel-production,sentry-release=251ff5179b4de94974f36d9b8659a487bbb8a819,sentry-public_key=2b87af2b44c84643a011838ad097735f,sentry-trace_id=72b0318a7141fe18cbacbd905572eddf,sentry-transaction=GET%20%2Fsearch,sentry-sampled=true,sentry-sample_rand=0.09967130764937493,sentry-sample_rate=0.5",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
//"Referer: https://cara.app/search?q=jak+and+daxter&type=&sortBy=Top&filters=%7B%7D",
"Referer: https://cara.app/search?q=" . urlencode($search),
//"Cookie: __Host-next-auth.csrf-token=b752c4296375bccb7b480ff010e1e916c65c35c311a4a57ac6cd871468730578%7C4d3783cfb72a98f390e534abd149806432b6cf8d50555a52d00e99216a516911; __Secure-next-auth.callback-url=https%3A%2F%2Fcara.app; crumb=BV0HDt87G5+fOWE0ZDQ5MWM0ZTQ3YTZmMzM4MGU5MGNjNDNmMzY2",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
"TE: trailers"]
);
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function image($get){
if($get["npt"]){
[$npt, $proxy] =
$this->backend->get(
$get["npt"],
"images"
);
$npt = json_decode($npt, true);
}else{
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$npt = [
"q" => $get["s"],
"sortBy" => $get["sort"],
"take" => 24,
"skip" => 0,
"filters" => []
];
// parse filters
if($get["type"] != "any"){
$npt["filters"]["posts"] = [$get["type"]];
}
if($get["fields"] != "any"){
$npt["filters"]["fields"] = [$get["fields"]];
}
if($get["category"] != "any"){
$npt["filters"]["categories"] = [$get["category"]];
}
if($get["software"] != "any"){
$npt["filters"]["softwares"] = [$get["software"]];
}
if($npt["filters"] == []){
$npt["filters"] = "{}";
}else{
$npt["filters"] = json_encode($npt["filters"]);
}
}
$out = [
"status" => "ok",
"npt" => null,
"image" => []
];
// https://cara.app/api/search/portfolio-posts?q=jak+and+daxter&sortBy=Top&take=24&skip=0&filters=%7B%7D
try{
$json =
$this->get(
$proxy,
"https://cara.app/api/search/posts",
$npt,
$npt["q"]
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$imagecount = 0;
foreach($json as $image){
if(count($image["images"]) === 0){
// sometimes the api returns no images for an object
$imagecount++;
continue;
}
$cover = null;
$sources = [];
foreach($image["images"] as $source){
if($source["isCoverImg"]){
$cover = [
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
"width" => 500,
"height" => 500
];
}else{
$sources[] = [
"url" => "https://images.cara.app/" . $this->fix_url($source["src"]),
"width" => null,
"height" => null
];
}
}
if($cover !== null){
$sources[] = $cover;
}
$out["image"][] = [
"title" => str_replace("\n", " ", $image["content"]),
"source" => $sources,
"url" => "https://cara.app/post/" . $image["id"]
];
$imagecount++;
}
if($imagecount === 24){
$npt["skip"] += 24;
$out["npt"] =
$this->backend->store(
json_encode($npt),
"images",
$proxy
);
}
return $out;
}
private function fix_url($url){
return
str_replace(
[" "],
["%20"],
$url
);
}
}
+21 -191
View File
@@ -523,7 +523,7 @@ class google{
config::FPLAY_EXTERNAL_ENDPOINT .
"?url=" . urlencode($url) .
"&pw=" . urlencode(config::FPLAY_PASSWORD) .
"&container=" .// urlencode($container) .
"&container=" . urlencode($container) .
"&proxy=" . urlencode($proxy) .
"&url_match=" . urlencode("&sei=") .
"&fail_url_match=" . urlencode("\/sorry\/");
@@ -733,7 +733,7 @@ class google{
$container = &$data["container"];
//file_put_contents("scraper/google-jp.html", $html);
//$html = file_get_contents("scraper/google-softcaptcha.html");
//$html = file_get_contents("scraper/google-jp.html");
}
$out = [
@@ -754,26 +754,9 @@ class google{
$this->fuckhtml->load($html);
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
// init
$this->detect_sorry();
$this->parsestyles();
$this->scrape_dimg($html);
$this->scrape_imagearr($html);
@@ -1398,28 +1381,7 @@ class google{
$this->fuckhtml->load($html);
// parse all <style> tags
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
$this->detect_sorry();
// get javascript images
$this->scrape_imagearr($html);
@@ -1581,7 +1543,8 @@ class google{
implode(",", $tbs);
}
}
//$html = file_get_contents("scraper/google.html");
$data = $this->get(
$proxy,
"https://www.google.com/search",
@@ -1592,8 +1555,6 @@ class google{
$html = &$data["data"];
$container = &$data["container"];
//$html = file_get_contents("scraper/google.html");
if(!isset($params["start"])){
$params["start"] = 0;
@@ -1602,32 +1563,13 @@ class google{
$this->fuckhtml->load($html);
// parse all <style> tags
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
//
// Parse web video page
//
$this->detect_sorry();
// parse all <style> tags
$this->parsestyles();
// get javascript images
$this->scrape_dimg($html);
@@ -1636,29 +1578,6 @@ class google{
$params["c"] = $container;
// get encoded links
$encoded_links =
explode(
"function(){var m={",
$html,
2
);
if(count($encoded_links) === 2){
$encoded_links =
json_decode(
$this->fuckhtml
->extract_json(
"{" . $encoded_links[1]
),
true
);
}else{
$encoded_links = [];
}
$out = [
"status" => "ok",
"npt" =>
@@ -1837,50 +1756,6 @@ class google{
}
}
$url =
$this->fuckhtml
->getTextContent(
$url[0]["attributes"]["href"]
);
if(
preg_match(
'/^\/goto\?url/',
$url
)
){
// encrypted url detected
// get reference div
$refdivs =
$this->fuckhtml
->getElementsByAttributeName(
"id",
"div"
);
foreach($refdivs as $r){
if(
preg_match(
'/^atritem-/',
$r["attributes"]["id"]
) &&
isset($r["attributes"]["jsdata"])
){
$ref = explode(";", $r["attributes"]["jsdata"]);
$ref = $ref[count($ref) - 1];
if(isset($encoded_links[$ref][32][11][7])){
$url = $encoded_links[$ref][32][11][7];
}
break;
}
}
}
$out["video"][] = [
"title" =>
$this->titledots(
@@ -1899,7 +1774,11 @@ class google{
"duration" => $duration,
"views" => null,
"thumb" => $thumb,
"url" => $url
"url" =>
$this->fuckhtml
->getTextContent(
$url[0]["attributes"]["href"]
)
];
}
@@ -2008,32 +1887,14 @@ class google{
$this->fuckhtml->load($html);
// parse styles
$this->parsestyles();
// solve softcaptcha
$softcaptcha = $this->detect_sorry();
if($softcaptcha !== false){
$params["sei"] = $softcaptcha["sei"];
$data2 = $this->get(
$proxy,
"https://www.google.com/search",
$params,
$container
);
$html = &$data2["data"];
$this->fuckhtml->load($html);
$this->parsestyles();
}
$this->detect_sorry();
// get images
$this->scrape_dimg($html);
// parse styles
$this->parsestyles();
$center_col =
$this->fuckhtml
->getElementById(
@@ -2792,7 +2653,6 @@ class google{
private function detect_sorry(){
// detect recaptcha
$captcha_form =
$this->fuckhtml
->getElementById(
@@ -2822,35 +2682,5 @@ class google{
throw new Exception("Google returned a JS challenge");
}
// bypass soft captcha
// "Verifying your request
// You'll be able to continue in a few seconds. Don't refresh this page."
$scripts =
$this->fuckhtml
->getElementsByTagName(
"script"
);
foreach($scripts as $sc){
if(
preg_match(
'/var eid ?= ?\'(.*)\'[\S\s]*, ?([0-9]+)\);/U',
$sc["innerHTML"],
$matches
)
){
sleep(10); // gotta sleep to not trigger captcha. blame google, not me
return [
"sei" => $matches[1],
"timeout" => (int)$matches[2] // set to 10000 usually
];
}
}
return false; // no need to bypass
}
}
+49 -191
View File
@@ -16,7 +16,18 @@ class mojeek{
case "web":
return [
"lang" => [ // lb
"focus" => [
"display" => "Focus",
"option" => [
"any" => "No focus",
"blogs" => "Blogs",
"Dictionary" => "Dictionary",
"Recipes" => "Recipes",
"Time" => "Time",
"Weather" => "Weather"
]
],
"lang" => [
"display" => "Language",
"option" => [
"any" => "Any language",
@@ -75,7 +86,7 @@ class mojeek{
"zu" => "Zulu"
]
],
"country" => [ // rb
"country" => [
"display" => "Country",
"option" => [
"any" => "No location bias",
@@ -329,21 +340,28 @@ class mojeek{
"zm" => "Zambia",
"zw" => "Zimbabwe"
]
],
"sort" => [
"display" => "Sort by",
],/*
"region" => [
"display" => "Region",
"option" => [
"relevance" => "Relevance",
"date" => "Date" // datewr=100
"any" => "Any region",
"eu" => "European Union",
"de" => "Germany",
"fr" => "France",
"uk" => "United Kingdom"
]
],*/
"domain" => [
"display" => "Results per domain",
"option" => [
"1" => "1 result",
"2" => "2 results",
"3" => "3 results",
"4" => "4 results",
"5" => "5 results",
"10" => "10 results",
"0" => "Unlimited",
]
],
"newer" => [
"display" => "Newer than",
"option" => "_DATE"
],
"older" => [
"display" => "Older than",
"option" => "_DATE"
]
];
break;
@@ -353,29 +371,21 @@ class mojeek{
}
}
private function get($proxy, $url, $get = [], $is_api = false){
private function get($proxy, $url, $get = []){
if($is_api){
$headers = [
"Accept: application/json"
];
}else{
$headers = [
"User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1"
];
}
$headers = [
"User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.9",
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
"Sec-Fetch-User: ?1"
];
$curlproc = curl_init();
@@ -410,154 +420,6 @@ class mojeek{
public function web($get){
return $this->web_api($get);
}
public function web_api($get){
$key = $this->backend->get_key();
$proxy = $this->backend->get_ip($key["increment"]);
if($get["npt"]){
[$params, $proxy] = $this->backend->get($get["npt"], "web");
$params = json_decode($params, true);
$params["api_key"] = $key["key"];
}else{
if(strlen($get["s"]) === 0){
throw new Exception("Search term is empty!");
}
$params = [
"api_key" => $key["key"],
"q" => $get["s"],
"date" => 1,
"tlen" => 127, // title length
"dlen" => 511, // description length
"fmt" => "json"
];
if($get["lang"] != "any"){
$params["lb"] = $get["lang"];
}
if($get["country"] != "any"){
$params["rb"] = $get["country"];
}
if($get["sort"] == "date"){
$params["datewr"] = 100;
}
if($get["newer"] !== false){
$params["since"] = date("Ymd", $get["newer"]);
}
if($get["older"] !== false){
$params["before"] = date("Ymd", $get["before"]);
}
}
try{
$json =
$this->get(
$proxy,
"https://api.mojeek.com/search",
$params
);
}catch(Exception $error){
throw new Exception("Failed to get JSON");
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
$out = [
"status" => "ok",
"spelling" => [
"type" => "no_correction",
"using" => null,
"correction" => null
],
"npt" => null,
"answer" => [],
"web" => [],
"image" => [],
"video" => [],
"news" => [],
"related" => []
];
if(!isset($json["response"]["status"])){
throw new Exception("API did not return a status field");
}
if($json["response"]["status"] != "OK"){
throw new Exception("Mojeek API returned an error: " . $json["response"]["status"]);
}
if(!isset($json["response"]["results"])){
throw new Exception("Mojeek API did not return a results object");
}
foreach($json["response"]["results"] as $entry){
$out["web"][] = [
"title" => html_entity_decode($entry["title"]),
"description" => html_entity_decode($entry["desc"]),
"url" => $entry["url"],
"date" => $entry["timestamp"],
"type" => "web",
"thumb" => [
"url" => null,
"ratio" => null
],
"sublink" => [],
"table" => []
];
}
// get npt
if($json["response"]["head"]["results"] > $json["response"]["head"]["start"] + $json["response"]["head"]["return"]){
unset($params["api_key"]);
if(!isset($params["s"])){
$params["s"] = 11;
}else{
$params["s"] = $params["s"] + 10;
}
$out["npt"] =
$this->backend->store(
json_encode($params),
"web",
$proxy
);
}
return $out;
}
public function web_browser($get){
if($get["npt"]){
[$token, $proxy] = $this->backend->get($get["npt"], "web");
@@ -595,6 +457,7 @@ class mojeek{
"tn" => 7, // number of news results/page
"date" => 1, // show date
"tlen" => 128, // max length of title
//"dlen" => 511, // max length of description
//"arc" => ($country == "any" ? "none" : $country) // location. don't use autodetect!
];
@@ -1137,12 +1000,7 @@ class mojeek{
return $out;
}
public function news(){
throw new Exception("Currently unsupported");
}
public function news_browser($get){
public function news($get){
$search = $get["s"];
+1102
View File
File diff suppressed because it is too large Load Diff
+153 -396
View File
@@ -9,10 +9,6 @@ class startpage{
include "lib/fuckhtml.php";
$this->fuckhtml = new fuckhtml();
include "lib/anubis.php";
$this->anubis = new anubis();
$this->anubis_error = "all gucci";
}
public function getfilters($page){
@@ -248,7 +244,7 @@ class startpage{
}
}
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false, $return_cookies = false, $passed_cookies = []){
private function get($proxy, $url, $get = [], $post = false, $is_xhr = false){
$curlproc = curl_init();
@@ -263,27 +259,6 @@ class startpage{
$url .= "?" . $get;
}
if($return_cookies){
$cookies = [];
curl_setopt($curlproc, CURLOPT_HEADERFUNCTION, function($curlproc, $header) use (&$cookies){
$length = strlen($header);
$header = explode(":", $header, 2);
if(trim(strtolower($header[0])) == "set-cookie"){
$cookie_tmp = explode("=", trim($header[1]), 2);
$cookies[trim($cookie_tmp[0])] =
explode(";", $cookie_tmp[1], 2)[0];
}
return $length;
});
}
curl_setopt($curlproc, CURLOPT_URL, $url);
// http2 bypass
@@ -291,15 +266,6 @@ class startpage{
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
$passed_cookies["preferences"] = "date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius";
foreach($passed_cookies as $name => $val){
$header_cookies[] = "$name=$val";
}
$header_cookies = rtrim(implode("; ", $header_cookies));
if($is_xhr === true){
curl_setopt($curlproc, CURLOPT_HTTPHEADER,
@@ -313,7 +279,7 @@ class startpage{
"Origin: https://www.startpage.com/",
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-origin",
@@ -332,7 +298,7 @@ class startpage{
"Content-Length: " . strlen($get),
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
@@ -350,7 +316,7 @@ class startpage{
"Accept-Encoding: gzip",
"DNT: 1",
"Connection: keep-alive",
"Cookie: $header_cookies",
"Cookie: preferences=date_timeEEEworldN1Ndisable_family_filterEEE1N1Ndisable_open_in_new_windowEEE0N1Nenable_post_methodEEE1N1Nenable_proxy_safety_suggestEEE0N1Nenable_stay_controlEEE0N1Ninstant_answersEEE1N1Nlang_homepageEEEs%2Fdevice%2FenN1NlanguageEEEenglishN1Nlanguage_uiEEEenglishN1Nnum_of_resultsEEE20N1Nsearch_results_regionEEEallN1NsuggestionsEEE1N1Nwt_unitEEEcelsius",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: none",
@@ -376,15 +342,6 @@ class startpage{
}
curl_close($curlproc);
if($return_cookies){
return [
"cookies" => $cookies,
"data" => $data
];
}
return $data;
}
@@ -392,22 +349,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "web");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -449,11 +398,7 @@ class startpage{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
$params
);
}catch(Exception $error){
@@ -461,41 +406,12 @@ class startpage{
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$cookies = [];
$html = $html["data"];
}else{
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
}
}
}
// detect captcha even if we passed anubis challenge
$this->detect_captcha($html);
//echo htmlspecialchars($html);
$this->detect_captcha($html);
if(
preg_match(
'/React\.createElement\(UIStartpage\.AppSerpWeb, ?(.+)\),?$/m',
@@ -504,14 +420,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to grep JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON");
}
//print_r($json);
@@ -533,7 +449,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "web", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "web", $proxy);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -742,22 +658,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "images");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -772,102 +680,76 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "images",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["size"] != "any"){
if(
$get["size"] == "Small" ||
$get["size"] == "Medium" ||
$get["size"] == "Large" ||
$get["size"] == "Wallpaper"
){
$params["flimgsize"] = $get["size"];
}else{
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
}
}
if($get["color"] != "any"){
if($get["color"] == "color"){
$params["flimgcolor"] = "ic:color";
}elseif($get["color"] == "bnw"){
$params["flimgcolor"] = "ic:gray";
}else{
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
}
}
if($get["type"] != "any"){
$params["flimgtype"] = $get["type"];
}
if($get["license"] != "any"){
$params["flimglicense"] = $get["license"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "images",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["size"] != "any"){
if(
$get["size"] == "Small" ||
$get["size"] == "Medium" ||
$get["size"] == "Large" ||
$get["size"] == "Wallpaper"
){
$params["flimgsize"] = $get["size"];
}else{
$params["image-size-select"] = "isz:lt,islt:" . $get["size"];
}
}
if($get["color"] != "any"){
if($get["color"] == "color"){
$params["flimgcolor"] = "ic:color";
}elseif($get["color"] == "bnw"){
$params["flimgcolor"] = "ic:gray";
}else{
$params["flimgcolor"] = "ic:specific,isc:" . $get["color"];
}
}
if($get["type"] != "any"){
$params["flimgtype"] = $get["type"];
}
if($get["license"] != "any"){
$params["flimglicense"] = $get["license"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -887,18 +769,18 @@ class startpage{
) === 0
){
throw new Exception("Failed to grep JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to grep JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
// get npt
$out["npt"] = $this->parse_npt($json, "images", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "images", $proxy);
// get images
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -937,22 +819,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "videos");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -967,72 +841,46 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "video",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["sort"] != "relevance"){
$params["sort_by"] = $get["sort"];
}
if($get["duration"] != "any"){
$params["with_duration"] = $get["duration"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "video",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["sort"] != "relevance"){
$params["sort_by"] = $get["sort"];
}
if($get["duration"] != "any"){
$params["with_duration"] = $get["duration"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -1046,14 +894,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to get JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
$out = [
@@ -1067,7 +915,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "video", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "video", $proxy);
// get results
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -1123,22 +971,14 @@ class startpage{
if($get["npt"]){
$this->anubis_error = "Not ran on pagination";
[$post, $proxy] = $this->backend->get($get["npt"], "news");
$post = json_decode($post, true);
$query = $post["query"];
$cookies = $post["cookies"];
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$query,
true,
false,
false,
$cookies
$post,
true
);
}catch(Exception $error){
@@ -1153,67 +993,41 @@ class startpage{
throw new Exception("Search term is empty!");
}
$proxy = $this->backend->get_ip();
$params = [
"query" => $get["s"],
"cat" => "news",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["time"] != "any"){
$params["with_date"] = $get["time"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
true,
[]
);
}catch(Exception $error){
throw new Exception("Failed to fetch search page. Anubis error: {$this->anubis_error}");
}
//$html = file_get_contents("scraper/startpage.html");
// detect captcha on that page
$this->detect_captcha($html["data"]);
// solve anubis if need be
$cookies = $this->solve_anubis($proxy, $html);
if($cookies === false){
$proxy = $this->backend->get_ip();
$cookies = [];
$html = $html["data"];
}else{
$params = [
"query" => $get["s"],
"cat" => "news",
"pl" => "opensearch"
];
if($get["nsfw"] == "no"){
$params["qadf"] = "heavy";
}
if($get["time"] != "any"){
$params["with_date"] = $get["time"];
}
try{
$html = $this->get(
$proxy,
"https://www.startpage.com/sp/search",
$params,
false,
false,
false,
$cookies
$params
);
}catch(Exception $error){
throw new Exception("Failed to re-fetch search page. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to fetch search page");
}
//$html = file_get_contents("scraper/startpage.html");
}catch(Exception $error){
throw new Exception("Failed to fetch search page");
}
}
@@ -1227,14 +1041,14 @@ class startpage{
) === 0
){
throw new Exception("Failed to get JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to get JSON object");
}
$json = json_decode($matches[1], true);
if($json === null){
throw new Exception("Failed to decode JSON object. Anubis error: {$this->anubis_error}");
throw new Exception("Failed to decode JSON object");
}
$out = [
@@ -1244,7 +1058,7 @@ class startpage{
];
// get npt
$out["npt"] = $this->parse_npt($json, "news", $proxy, $cookies);
$out["npt"] = $this->parse_npt($json, "news", $proxy);
foreach($json["render"]["presenter"]["regions"]["mainline"] as $category){
@@ -1287,7 +1101,7 @@ class startpage{
return $out;
}
private function parse_npt($json, $pagetype, $proxy, $cookies){
private function parse_npt($json, $pagetype, $proxy){
foreach($json["render"]["presenter"]["pagination"]["pages"] as $page){
@@ -1295,27 +1109,22 @@ class startpage{
return
$this->backend->store(
json_encode([
"query" =>
http_build_query(
[
"lui" => "english",
"language" => "english",
"sc" => $json["render"]["search_sc"],
"t" => "device",
"cat" => $pagetype,
"segment" => "startpage.udog", // "original" sometimes
"abd" => "0",
"abe" => "0",
"query" => rawurldecode($json["render"]["query"]),
"page" => $page["number"],
"qsr" => "all",
"qadf" => "none", // @ todo fix (??)
...(isset($page["token"]) ? ["page_token" => $page["token"]] : [])
]
),
"cookies" => $cookies
]),
http_build_query(
[
"lui" => "english",
"language" => "english",
"sc" => $json["render"]["search_sc"],
"t" => "device",
"cat" => $pagetype,
"segment" => "organic",
"abd" => "0",
"abe" => "0",
"query" => rawurldecode($json["render"]["query"]),
"page" => $page["number"],
"qsr" => "all",
"qadf" => "none" // @ todo fix (??)
]
),
$pagetype,
$proxy
);
@@ -1461,56 +1270,4 @@ class startpage{
throw new Exception("Startpage redirected the scraper to an unhandled page");
}
}
private function solve_anubis($proxy, $html){
//
// solve anubis challenge, if theres one
//
$res = null;
$cookies = [];
try{
$res = $this->anubis->scrape($html["data"]);
}catch(Exception $error){
// do nothang
$this->anubis_error = $error->getMessage();
}
if($res !== null){
// https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge
// ?id=01a00241-da22-7d62-b64f-e7a6a0e8bda8
// &response=0000460826adc23e0363e97e713e4aa36ca46e1a7cd933184442772f3d6b994c
// &nonce=3098
// &redir=https%3A%2F%2Fwww.startpage.com%2Fsp%2Fsearch
// &elapsedTime=111
$anubis_response =
$this->get(
$proxy,
"https://www.startpage.com/.within.website/x/cmd/anubis/api/pass-challenge",
[
"id" => $res["id"],
"response" => $res["response"],
"nonce" => $res["nonce"],
"redir" => "https://www.startpage.com/sp/search",
"elapsedTime" => $res["elapsedTime"]
],
false,
false,
true,
$html["cookies"]
);
// re-fetch search page AGAIN with the anubis bullshit
$cookies = array_merge($anubis_response["cookies"], $html["cookies"]);
return $cookies;
}
return false;
}
}
+944 -548
View File
File diff suppressed because it is too large Load Diff
+30 -2
View File
@@ -87,6 +87,10 @@ $settings = [
"value" => "kagi",
"text" => "Kagi"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "yep",
"text" => "Yep"
@@ -141,6 +145,14 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "ghostery",
"text" => "Ghostery"
],
[
"value" => "yep",
"text" => "Yep"
@@ -219,6 +231,10 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "naver",
"text" => "Naver"
@@ -235,6 +251,10 @@ $settings = [
"value" => "pinterest",
"text" => "Pinterest"
],
[
"value" => "cara",
"text" => "Cara"
],
[
"value" => "flickr",
"text" => "Flickr"
@@ -309,6 +329,10 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "naver",
"text" => "Naver"
@@ -355,10 +379,14 @@ $settings = [
"value" => "startpage",
"text" => "Startpage"
],
/*[
[
"value" => "qwant",
"text" => "Qwant"
],
[
"value" => "mojeek",
"text" => "Mojeek"
],*/
],
[
"value" => "baidu",
"text" => "Baidu"
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff