added yep web and news search
This commit is contained in:
lolcat committed
1 parent
04038e973d
commit
afb17f45ee
4 files changed
+374
-35
No files matched your search
+3
-1
@@ -892,6 +892,7 @@ class frontend{
|
||||
"brave" => "Brave",
|
||||
"yandex" => "Yandex",
|
||||
"google" => "Google",
|
||||
"yep" => "Yep",
|
||||
"mojeek" => "Mojeek",
|
||||
"marginalia" => "Marginalia",
|
||||
"wiby" => "wiby"
|
||||
@@ -936,6 +937,7 @@ class frontend{
|
||||
"ddg" => "DuckDuckGo",
|
||||
"brave" => "Brave",
|
||||
"google" => "Google",
|
||||
"yep" => "Yep",
|
||||
"mojeek" => "Mojeek"
|
||||
]
|
||||
];
|
||||
@@ -1330,7 +1332,7 @@ class frontend{
|
||||
return htmlspecialchars($image);
|
||||
}
|
||||
|
||||
return "/proxy?i=" . urlencode($image) . "&s=" . $format;
|
||||
return "/proxy.php?i=" . urlencode($image) . "&s=" . $format;
|
||||
}
|
||||
|
||||
public function htmlnextpage($gets, $npt, $page){
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
["Ok",{"results":[{"url":"https://boingboing.net/2023/03/18/what-happens-when-you-write-on-4chan-that-someone-should-kill-an-anti-nazi-sheriff.html","title":"What happens when you write on 4chan that someone should kill an anti-Nazi sheriff | Boing Boing","type":"News","snippet":"What happens when you write on 4chan that someone should kill an anti-Nazi sheriff | Boing Boing\u001dMENU SEARCH STORE MENU SEARCH STORE Blog : The posts Forums : Read the rules Store : Wonderful Products","img":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2023%2F03%2Fimage-44.png%3Ffit%3D1200%252C772%26ssl%3D1","visual_url":"boingboing.net › 2023/03/18 › what-happens-when-you-write-on-4chan-that-someone-should-kill-an-anti-nazi-sheriff.html","first_seen":"2023-03-18T14:56:15Z","srcset":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2023%2F03%2Fimage-44.png%3Ffit%3D1200%252C772%26ssl%3D1, https://api.yep.com/img/thumb/large_news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2023%2F03%2Fimage-44.png%3Ffit%3D1200%252C772%26ssl%3D1 2x"},{"url":"https://boingboing.net/2014/12/31/how-imageboard-culture-shaped.html","title":"How imageboard culture shaped Gamergate | Boing Boing","type":"News","snippet":"How imageboard culture shaped Gamergate | Boing Boing\u001dMENU SEARCH STORE MENU SEARCH STORE Blog : The posts Forums : Read the rules Store : Wonderful Products (Contact Support) Newsletter : Daily wonde","img":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2014%2F12%2FNO.png%3Ffit%3D1200%252C736%26ssl%3D1","visual_url":"boingboing.net › 2014/12/31 › how-imageboard-culture-shaped.html","first_seen":"2015-08-02T14:05:25Z","srcset":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2014%2F12%2FNO.png%3Ffit%3D1200%252C736%26ssl%3D1, https://api.yep.com/img/thumb/large_news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2014%2F12%2FNO.png%3Ffit%3D1200%252C736%26ssl%3D1 2x"},{"url":"https://boingboing.net/2010/07/16/mad-mustachioed-dad.html","title":"4chan \"backtraced,\" reported to \"cyberpolice\" by mustachioed mad dad | Boing Boing","type":"News","snippet":"4chan \"backtraced,\" reported to \"cyberpolice\" by mustachioed mad dad | Boing Boing\u001dMENU SEARCH STORE MENU SEARCH STORE Blog : The posts Forums : Read the rules Store : Wonderful Products (Contact Supp","img":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg","visual_url":"boingboing.net › 2010/07/16 › mad-mustachioed-dad.html","first_seen":"2017-08-28T20:40:00Z","srcset":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg, https://api.yep.com/img/thumb/large_news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg 2x"},{"url":"https://boingboing.net/2014/05/31/this-day-in-blogging-history-227.html","title":"This Day in Blogging History: Spelling-error bomber foiled by typos; Stephanie Meyer v 4chan; The problem with contextual advertising | Boing Boing","type":"News","snippet":"This Day in Blogging History: Spelling-error bomber foiled by typos; Stephanie Meyer v 4chan; The problem with contextual advertising | Boing Boing\u001dMENU SEARCH STORE MENU SEARCH STORE Blog : The posts","img":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg","visual_url":"boingboing.net › 2014/05/31 › this-day-in-blogging-history-227.html","first_seen":"2017-09-01T18:07:45Z","srcset":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg, https://api.yep.com/img/thumb/large_news?url=https%3A%2F%2Fs0.wp.com%2Fi%2Fblank.jpg 2x"},{"url":"https://boingboing.net/2015/09/21/4chan-sold-to-2chan-founder.html","title":"4chan sold to 2chan founder | Boing Boing","type":"News","snippet":"4chan sold to 2chan founder | Boing Boing\u001dMENU SEARCH STORE MENU SEARCH STORE Blog : The posts Forums : Read the rules Store : Wonderful Products (Contact Support) Newsletter : Daily wonderful things ","img":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2015%2F09%2Fyotsuba-4chan.jpg%3Ffit%3D889%252C500%26ssl%3D1","visual_url":"boingboing.net › 2015/09/21 › 4chan-sold-to-2chan-founder.html","first_seen":"2017-09-01T15:49:38Z","srcset":"https://api.yep.com/img/thumb/news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2015%2F09%2Fyotsuba-4chan.jpg%3Ffit%3D889%252C500%26ssl%3D1, https://api.yep.com/img/thumb/large_news?url=https%3A%2F%2Fi0.wp.com%2Fboingboing.net%2Fwp-content%2Fuploads%2F2015%2F09%2Fyotsuba-4chan.jpg%3Ffit%3D889%252C500%26ssl%3D1 2x"},{"url":"https://boingboing.net/2017/03/16/leaving-kids-in-front-of-scree.html","title":"Leaving kids in front of screens unsupervised for hours may have unpleasant consequences, parents learn | Line truncated
|
||||
+362
-34
@@ -261,7 +261,8 @@ class yep{
|
||||
"Connection: keep-alive",
|
||||
"Sec-Fetch-Dest: empty",
|
||||
"Sec-Fetch-Mode: cors",
|
||||
"Sec-Fetch-Site: same-site"]
|
||||
"Sec-Fetch-Site: same-site",
|
||||
"TE: trailers"]
|
||||
);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
@@ -283,6 +284,212 @@ class yep{
|
||||
return $data;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public function web($get){
|
||||
|
||||
$search = $get["s"];
|
||||
if(strlen($search) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$country = $get["country"];
|
||||
$nsfw = $get["nsfw"];
|
||||
|
||||
switch($nsfw){
|
||||
|
||||
case "yes": $nsfw = "off"; break;
|
||||
case "maybe": $nsfw = "moderate"; break;
|
||||
case "no": $nsfw = "strict"; break;
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"spelling" => [
|
||||
"type" => "no_correction",
|
||||
"using" => null,
|
||||
"correction" => null
|
||||
],
|
||||
"npt" => null,
|
||||
"answer" => [],
|
||||
"web" => [],
|
||||
"image" => [],
|
||||
"video" => [],
|
||||
"news" => [],
|
||||
"related" => []
|
||||
];
|
||||
|
||||
try{
|
||||
|
||||
// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
|
||||
$json =
|
||||
json_decode(
|
||||
$this->get(
|
||||
$this->backend->get_ip(),
|
||||
"https://api.yep.com/fs/2/search",
|
||||
[
|
||||
"client" => "web",
|
||||
"gl" => $country == "all" ? $country : strtoupper($country),
|
||||
"limit" => "99999",
|
||||
"no_correct" => "false",
|
||||
"q" => $search,
|
||||
"safeSearch" => $nsfw,
|
||||
"type" => "web"
|
||||
]
|
||||
),
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch JSON");
|
||||
}
|
||||
|
||||
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
if(isset($json[1]["correction"])){
|
||||
|
||||
$out["spelling"] = [
|
||||
"type" => "not_many",
|
||||
"using" => $search,
|
||||
"correction" => $json[1]["correction"][1]
|
||||
];
|
||||
}
|
||||
|
||||
if(isset($json[1]["results"])){
|
||||
foreach($json[1]["results"] as $item){
|
||||
|
||||
switch(strtolower($item["type"])){
|
||||
|
||||
case "organic":
|
||||
$sublinks = [];
|
||||
|
||||
if(isset($item["sitelinks"]["full"])){
|
||||
|
||||
foreach($item["sitelinks"]["full"] as $link){
|
||||
|
||||
$sublinks[] = [
|
||||
"title" => $link["title"],
|
||||
"date" => null,
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$link["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"url" => $link["url"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
$out["web"][] = [
|
||||
"title" => $item["title"],
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$item["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"url" => $item["url"],
|
||||
"date" => strtotime($item["first_seen"]),
|
||||
"type" => "web",
|
||||
"thumb" => [
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"sublink" => $sublinks,
|
||||
"table" => []
|
||||
];
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if(isset($json[1]["featured_news"])){
|
||||
|
||||
foreach($json[1]["featured_news"] as $news){
|
||||
|
||||
$out["news"][] = [
|
||||
"title" => $news["title"],
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$news["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"date" => strtotime($news["first_seen"]),
|
||||
"thumb" =>
|
||||
isset($news["img"]) ?
|
||||
[
|
||||
"url" => $this->unshiturl($news["img"]),
|
||||
"ratio" => "16:9"
|
||||
] :
|
||||
[
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"url" => $news["url"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
if(isset($json[1]["featured_images"])){
|
||||
|
||||
foreach($json[1]["featured_images"] as $image){
|
||||
|
||||
if(
|
||||
$image["width"] !== 0 &&
|
||||
$image["height"] !== 0
|
||||
){
|
||||
|
||||
$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
|
||||
$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
|
||||
|
||||
$width = $image["width"];
|
||||
$height = $image["height"];
|
||||
}else{
|
||||
|
||||
$thumb_width = null;
|
||||
$thumb_height = null;
|
||||
$width = null;
|
||||
$height = null;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" => $image["title"],
|
||||
"source" => [
|
||||
[
|
||||
"url" => $image["image_id"],
|
||||
"width" => $width,
|
||||
"height" => $height
|
||||
],
|
||||
[
|
||||
"url" => $image["src"],
|
||||
"width" => $thumb_width,
|
||||
"height" => $thumb_height
|
||||
]
|
||||
],
|
||||
"url" => $image["host_page"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
|
||||
|
||||
public function image($get){
|
||||
|
||||
$search = $get["s"];
|
||||
@@ -335,44 +542,165 @@ class yep{
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
foreach($json[1]["results"] as $item){
|
||||
|
||||
if(
|
||||
$item["width"] !== 0 &&
|
||||
$item["height"] !== 0
|
||||
){
|
||||
if(isset($json[1]["results"])){
|
||||
foreach($json[1]["results"] as $item){
|
||||
|
||||
$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
|
||||
$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
|
||||
if(
|
||||
$item["width"] !== 0 &&
|
||||
$item["height"] !== 0
|
||||
){
|
||||
|
||||
$thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
|
||||
$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
|
||||
|
||||
$width = $item["width"];
|
||||
$height = $item["height"];
|
||||
}else{
|
||||
|
||||
$thumb_width = null;
|
||||
$thumb_height = null;
|
||||
$width = null;
|
||||
$height = null;
|
||||
}
|
||||
|
||||
$width = $item["width"];
|
||||
$height = $item["height"];
|
||||
}else{
|
||||
|
||||
$thumb_width = null;
|
||||
$thumb_height = null;
|
||||
$width = null;
|
||||
$height = null;
|
||||
}
|
||||
|
||||
$out["image"][] = [
|
||||
"title" => $item["title"],
|
||||
"source" => [
|
||||
[
|
||||
"url" => $item["image_id"],
|
||||
"width" => $width,
|
||||
"height" => $height
|
||||
$out["image"][] = [
|
||||
"title" => $item["title"],
|
||||
"source" => [
|
||||
[
|
||||
"url" => $item["image_id"],
|
||||
"width" => $width,
|
||||
"height" => $height
|
||||
],
|
||||
[
|
||||
"url" => $item["src"],
|
||||
"width" => $thumb_width,
|
||||
"height" => $thumb_height
|
||||
]
|
||||
],
|
||||
[
|
||||
"url" => $item["src"],
|
||||
"width" => $thumb_width,
|
||||
"height" => $thumb_height
|
||||
]
|
||||
],
|
||||
"url" => $item["host_page"]
|
||||
];
|
||||
"url" => $item["host_page"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
|
||||
public function news($get){
|
||||
|
||||
$search = $get["s"];
|
||||
if(strlen($search) === 0){
|
||||
|
||||
throw new Exception("Search term is empty!");
|
||||
}
|
||||
|
||||
$country = $get["country"];
|
||||
$nsfw = $get["nsfw"];
|
||||
|
||||
switch($nsfw){
|
||||
|
||||
case "yes": $nsfw = "off"; break;
|
||||
case "maybe": $nsfw = "moderate"; break;
|
||||
case "no": $nsfw = "strict"; break;
|
||||
}
|
||||
|
||||
$out = [
|
||||
"status" => "ok",
|
||||
"npt" => null,
|
||||
"news" => []
|
||||
];
|
||||
|
||||
try{
|
||||
|
||||
// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
|
||||
$json =
|
||||
json_decode(
|
||||
$this->get(
|
||||
$this->backend->get_ip(),
|
||||
"https://api.yep.com/fs/2/search",
|
||||
[
|
||||
"client" => "web",
|
||||
"gl" => $country == "all" ? $country : strtoupper($country),
|
||||
"limit" => "99999",
|
||||
"no_correct" => "false",
|
||||
"q" => $search,
|
||||
"safeSearch" => $nsfw,
|
||||
"type" => "news"
|
||||
]
|
||||
),
|
||||
true
|
||||
);
|
||||
}catch(Exception $error){
|
||||
|
||||
throw new Exception("Failed to fetch JSON");
|
||||
}
|
||||
|
||||
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
|
||||
|
||||
if($json === null){
|
||||
|
||||
throw new Exception("Failed to decode JSON");
|
||||
}
|
||||
|
||||
if(isset($json[1]["results"])){
|
||||
foreach($json[1]["results"] as $item){
|
||||
|
||||
$out["news"][] = [
|
||||
"title" => $item["title"],
|
||||
"author" => null,
|
||||
"description" =>
|
||||
$this->titledots(
|
||||
strip_tags(
|
||||
html_entity_decode(
|
||||
$item["snippet"]
|
||||
)
|
||||
)
|
||||
),
|
||||
"date" => strtotime($item["first_seen"]),
|
||||
"thumb" =>
|
||||
isset($item["img"]) ?
|
||||
[
|
||||
"url" => $this->unshiturl($item["img"]),
|
||||
"ratio" => "16:9"
|
||||
] :
|
||||
[
|
||||
"url" => null,
|
||||
"ratio" => null
|
||||
],
|
||||
"url" => $item["url"]
|
||||
];
|
||||
}
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
|
||||
private function titledots($title){
|
||||
|
||||
$substr = substr($title, -4);
|
||||
|
||||
if(
|
||||
strpos($substr, "...") !== false ||
|
||||
strpos($substr, "…") !== false
|
||||
){
|
||||
|
||||
return trim(substr($title, 0, -4));
|
||||
}
|
||||
|
||||
return trim($title);
|
||||
}
|
||||
|
||||
private function unshiturl($url){
|
||||
|
||||
$newurl = parse_url($url, PHP_URL_QUERY);
|
||||
parse_str($newurl, $newurl);
|
||||
|
||||
if(isset($newurl["url"])){
|
||||
|
||||
return $newurl["url"];
|
||||
}
|
||||
|
||||
return $url;
|
||||
}
|
||||
}
|
||||
@@ -121,6 +121,10 @@ $settings = [
|
||||
"value" => "google",
|
||||
"text" => "Google"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
],
|
||||
[
|
||||
"value" => "mojeek",
|
||||
"text" => "Mojeek"
|
||||
@@ -215,6 +219,10 @@ $settings = [
|
||||
"value" => "google",
|
||||
"text" => "Google"
|
||||
],
|
||||
[
|
||||
"value" => "yep",
|
||||
"text" => "Yep"
|
||||
],
|
||||
[
|
||||
"value" => "mojeek",
|
||||
"text" => "Mojeek"
|
||||
|
||||
Reference in new issue
Block a user