duckduckgo scraper rewrite
This commit is contained in:
lolcat committed
1 parent
0b3bbe0f15
commit
774f7113df
6 files changed
+1488
-2234
No files matched your search
@@ -1,10 +1,17 @@
|
||||
__ __ __
|
||||
/ // / ____ ____ / /_
|
||||
/ // /_/ __ `/ _ \/ __/
|
||||
/__ __/ /_/ / __/ /_
|
||||
/_/ \__, /\___/\__/
|
||||
/____/
|
||||
|
||||
44
|
||||
4444444 44
|
||||
44444444 44444 444
|
||||
44444444 444444 444444444
|
||||
44444 44444444 444444444
|
||||
444444444 4444444
|
||||
4444444444 444444
|
||||
4444444444444
|
||||
444444444444444444
|
||||
444444444444444
|
||||
44444444
|
||||
4444
|
||||
44
|
||||
|
||||
+ Welcome to the 4get API documentation +
|
||||
|
||||
+ Terms of use
|
||||
|
||||
+1
-1
@@ -119,7 +119,7 @@ class config{
|
||||
|
||||
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
|
||||
// Changing this might break things.
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:131.0) Gecko/20100101 Firefox/131.0";
|
||||
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:133.0) Gecko/20100101 Firefox/133.0";
|
||||
|
||||
// Proxy pool assignments for each scraper
|
||||
// false = Use server's raw IP
|
||||
|
||||
@@ -75,6 +75,7 @@ class backend{
|
||||
break;
|
||||
|
||||
case "socks5_hostname":
|
||||
case "socks5h":
|
||||
case "socks5a":
|
||||
curl_setopt($curlproc, CURLOPT_PROXYTYPE, CURLPROXY_SOCKS5_HOSTNAME);
|
||||
curl_setopt($curlproc, CURLOPT_PROXY, $address . ":" . $port);
|
||||
|
||||
@@ -526,4 +526,85 @@ class fuckhtml{
|
||||
$string
|
||||
);
|
||||
}
|
||||
|
||||
public function extract_json($json){
|
||||
|
||||
$len = strlen($json);
|
||||
$array_level = 0;
|
||||
$object_level = 0;
|
||||
$in_quote = null;
|
||||
$start = null;
|
||||
|
||||
for($i=0; $i<$len; $i++){
|
||||
|
||||
switch($json[$i]){
|
||||
|
||||
case "[":
|
||||
if($in_quote === null){
|
||||
|
||||
$array_level++;
|
||||
if($start === null){
|
||||
|
||||
$start = $i;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "]":
|
||||
if($in_quote === null){
|
||||
|
||||
$array_level--;
|
||||
}
|
||||
break;
|
||||
|
||||
case "{":
|
||||
if($in_quote === null){
|
||||
|
||||
$object_level++;
|
||||
if($start === null){
|
||||
|
||||
$start = $i;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "}":
|
||||
if($in_quote === null){
|
||||
|
||||
$object_level--;
|
||||
}
|
||||
break;
|
||||
|
||||
case "\"":
|
||||
case "'":
|
||||
if(
|
||||
$i !== 0 &&
|
||||
$json[$i - 1] !== "\\"
|
||||
){
|
||||
// found a non-escaped quote
|
||||
|
||||
if($in_quote === null){
|
||||
|
||||
// open quote
|
||||
$in_quote = $json[$i];
|
||||
}elseif($in_quote === $json[$i]){
|
||||
|
||||
// close quote
|
||||
$in_quote = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if(
|
||||
$start !== null &&
|
||||
$array_level === 0 &&
|
||||
$object_level === 0
|
||||
){
|
||||
|
||||
return substr($json, $start, $i - $start + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+1390
-2226
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1 @@
|
||||
window.execDeep = function() {DDG.deep.adUnitIndex = '9';if (DDG.deep && DDG.deep.setUpstream) DDG.deep.setUpstream("bingv7aa");DDG.deep.bn={'ivc':1};if (DDG.pageLayout) DDG.pageLayout.load('a',[], {"page_load_url":"https://duckduckgo.com/y.js?iurl=%7B2%7DIG%3DB6F0220C3D7149908C9F88B8A339DBF5%26CID%3D0A9064B4E76965BC095471FEE6D96447%26Type%3DEvent.CPT%26DATA%3D0"});DDG.duckbar.future_signal_tab({signal:'low',from:'lyrics'});nrj('/a.js?q=4get&s=lyrics&ta=the%20living%20tombstone%20%26%20cg5&tl=4get&from=lyrics&l=ca-en',null,null,null,null,'lyrics');DDG.deep.signalSummary = "lyrics:l";DDG.inject('DDG.Data.languages.resultLanguages', {"en":["https://4get.ca/","https://www.youtube.com/watch?v=rdiQ6h9p2Qo","https://www.youtube.com/watch?v=13Azi9Jnsw4","https://genius.com/The-living-tombstone-and-cg5-4get-lyrics","https://soundcloud.com/cg5-beats/4get","https://music.apple.com/us/album/4get-feat-or3o-single/1659731644","https://music.apple.com/ca/album/4get-feat-or3o-single/1659731644","https://www.shazam.com/song/1659731645/4get-feat-or3o"]});DDG.deep.pageLayoutSummary = "w6RelatedQnA1w4r1";DDG.inject('DDG.Data.languages.adLanguages', {});if (DDG.pageLayout) DDG.pageLayout.load('d',[{"a":"4get: 4get is a proxy search engine that doesn't suck. Settings \u2022 Instances \u2022 News \u2022 API \u2022 About \u2022 Source Donate to the project. Clearnet \u2022 Tor \u2022 Report a problem Running on v8!! 4get: 4get is a proxy search engine that doesn't suck.","ae":null,"c":"https://4get.ca/","d":"4get.ca","da":"","h":0,"i":"4get.ca","k":null,"l":[{"snippet":"For example, a search query like \"hello my full legal name is jonathan gallindo and i want pictures of cloacas\" would definitively blow your cover. 4get doesn't contain ads or any third party javascript applets or trackers. I don't profile you, and quite frankly, I don't give a shit about what you search on there.","targetUrl":"https://4get.ca/about","text":"About"},{"snippet":"4get: Settings. Preference link Following this link will re-apply all cookies configured here and will redirect you to the front page. Useful if your browser clears out cookies after a browsing session. Bookmark me! Settings. By clicking . Update settings!, a plaintext . key=value. cookie will be stored on your browser. When selecting a default setting, the parameter is removed from your ...","targetUrl":"https://4get.ca/settings","text":"Settings"},{"snippet":"Learn how to setup your own instance here! https://git.lolcat.ca/lolcat/4get","targetUrl":"https://4get.ca/instances","text":"Instance Browser"}],"m":0,"o":0,"p":0,"s":"bingv7aa","t":"4get","u":"https://4get.ca/"},{"a":"Watch the official music video of 4GET, a song by CG5 and The Living Tombstone featuring OR3O, a popular YouTube rapper. The song is about forgetting about a breakup and dealing with the consequences of moving on.","ae":null,"b":"yt\tYouTube\twww.youtube.com","c":"https://www.youtube.com/watch?v=rdiQ6h9p2Qo","d":"www.youtube.com/watch?v=rdiQ6h9p2Qo","da":"mlb_games,nba_games,ncaafb_games,ncaamb_games,nfl_games,nhl_games,soccer_games,videos,wheretowatch","h":0,"i":"www.youtube.com","k":null,"m":0,"o":0,"p":0,"rf":[{"hint":{"text":"MEDIA:DURATION"},"items":[{"text":"3 min"}],"label":{"text":"Video Duration"}},{"hint":{"text":"BASE:AUTHOR"},"items":[{"text":"CG5"}],"label":{"text":"Author"}},{"hint":{"text":"BASE:VIEWSCOUNT"},"items":[{"text":"6524837"}],"label":{"text":"Views"}}],"s":"bingv7aa","sn":"YouTube","t":"CG5 \u00d7 The Living Tombstone - 4GET (feat. OR3O) [Official Music Video]","u":"https://www.youtube.com/watch?v=rdiQ6h9p2Qo","video":{"duration":"2:47","height":72,"statistics":{"view_count":"6524837"},"thumbnail_url_template":"https://www.bing.com/th?id=OVP.mJd6d0rVmbZXI21G1owfewHgFo&w={width}&h={height}&c=4&rs=1&pid=6.1","width":128}},{"a":"Music Production/Composition by @CG5 & @TheLivingTombstoneLyrics by @CG5 & @OR3O_xdExecutive Producer/Writer - CG5Director - Maudolith, Luximus, CG5Project M...","ae":null,"b":"yt\tYouTube\twww.youtube.com","c":"https://www.youtube.com/watch?v=13Azi9Jnsw4","d":"www.youtube.com/watch?v=13Azi9Jnsw4","da":"mlb_games,nba_games,ncaafb_games,ncaamb_games,nfl_games,nhl_games,soccer_games,videos,wheretowatch","h":0,"i":"www.youtube.com","k":null,"m":0,"o":0,"p":0,"rf":[{"hint":{"text":"MEDIA:DURATION"},"items":[{"text":"3 min"}],"label":{"text":"Video Duration"}},{"hint":{"text":"BASE:AUTHOR"},"items":[{"text":"OlayerGames"}],"label":{"text":"Author"}},{"hint":{"text":"BASE:VIEWSCOUNT"},"items":[{"text":"28"}],"label":{"text":"Views"}}],"s":"bingv7aa","sn":"YouTube","t":"CG5 \u00d7 The Living Tombstone - 4GET (feat. OR3O) [Official ... - YouTube","u":"https://www.youtube.com/watch?v=13Azi9Jnsw4","video":{"duration":"2:40","height":72,"statistics":{"view_count":"28"},"thumbnail_url_template":"https://www.bing.com/th?id=OVP.yUNwcfcNtyQhFC_y0FpsxgEsDh&w={width}&h={height}&c=4&rs=1&pid=6.1","width":128}},{"a":"4GET is a song written about the crippling anxiety that I have always Line truncated
|
||||
Reference in new issue
Block a user