forked from lolcat/4get
1
0
Fork 0

added yep web and news search

This commit is contained in:
lolcat 2023-12-02 19:11:52 -05:00
parent 04038e973d
commit afb17f45ee
4 changed files with 374 additions and 35 deletions

View File

@ -892,6 +892,7 @@ class frontend{
"brave" => "Brave", "brave" => "Brave",
"yandex" => "Yandex", "yandex" => "Yandex",
"google" => "Google", "google" => "Google",
"yep" => "Yep",
"mojeek" => "Mojeek", "mojeek" => "Mojeek",
"marginalia" => "Marginalia", "marginalia" => "Marginalia",
"wiby" => "wiby" "wiby" => "wiby"
@ -936,6 +937,7 @@ class frontend{
"ddg" => "DuckDuckGo", "ddg" => "DuckDuckGo",
"brave" => "Brave", "brave" => "Brave",
"google" => "Google", "google" => "Google",
"yep" => "Yep",
"mojeek" => "Mojeek" "mojeek" => "Mojeek"
] ]
]; ];
@ -1330,7 +1332,7 @@ class frontend{
return htmlspecialchars($image); return htmlspecialchars($image);
} }
return "/proxy?i=" . urlencode($image) . "&s=" . $format; return "/proxy.php?i=" . urlencode($image) . "&s=" . $format;
} }
public function htmlnextpage($gets, $npt, $page){ public function htmlnextpage($gets, $npt, $page){

1
scraper/yep.json Normal file

File diff suppressed because one or more lines are too long

View File

@ -261,7 +261,8 @@ class yep{
"Connection: keep-alive", "Connection: keep-alive",
"Sec-Fetch-Dest: empty", "Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors", "Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-site"] "Sec-Fetch-Site: same-site",
"TE: trailers"]
); );
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
@ -283,6 +284,212 @@ class yep{
return $data; return $data;
} }
public function web($get){
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$country = $get["country"];
$nsfw = $get["nsfw"];
switch($nsfw){
case "yes": $nsfw = "off"; break;
case "maybe": $nsfw = "moderate"; break;
case "no": $nsfw = "strict"; break;
}
$out = [
"status" => "ok",
"spelling" => [
"type" => "no_correction",
"using" => null,
"correction" => null
],
"npt" => null,
"answer" => [],
"web" => [],
"image" => [],
"video" => [],
"news" => [],
"related" => []
];
try{
// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
$json =
json_decode(
$this->get(
$this->backend->get_ip(),
"https://api.yep.com/fs/2/search",
[
"client" => "web",
"gl" => $country == "all" ? $country : strtoupper($country),
"limit" => "99999",
"no_correct" => "false",
"q" => $search,
"safeSearch" => $nsfw,
"type" => "web"
]
),
true
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json[1]["correction"])){
$out["spelling"] = [
"type" => "not_many",
"using" => $search,
"correction" => $json[1]["correction"][1]
];
}
if(isset($json[1]["results"])){
foreach($json[1]["results"] as $item){
switch(strtolower($item["type"])){
case "organic":
$sublinks = [];
if(isset($item["sitelinks"]["full"])){
foreach($item["sitelinks"]["full"] as $link){
$sublinks[] = [
"title" => $link["title"],
"date" => null,
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$link["snippet"]
)
)
),
"url" => $link["url"]
];
}
}
$out["web"][] = [
"title" => $item["title"],
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$item["snippet"]
)
)
),
"url" => $item["url"],
"date" => strtotime($item["first_seen"]),
"type" => "web",
"thumb" => [
"url" => null,
"ratio" => null
],
"sublink" => $sublinks,
"table" => []
];
break;
}
}
}
if(isset($json[1]["featured_news"])){
foreach($json[1]["featured_news"] as $news){
$out["news"][] = [
"title" => $news["title"],
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$news["snippet"]
)
)
),
"date" => strtotime($news["first_seen"]),
"thumb" =>
isset($news["img"]) ?
[
"url" => $this->unshiturl($news["img"]),
"ratio" => "16:9"
] :
[
"url" => null,
"ratio" => null
],
"url" => $news["url"]
];
}
}
if(isset($json[1]["featured_images"])){
foreach($json[1]["featured_images"] as $image){
if(
$image["width"] !== 0 &&
$image["height"] !== 0
){
$thumb_width = $image["width"] >= 260 ? 260 : $image["width"];
$thumb_height = ceil($image["height"] * ($thumb_width / $image["width"]));
$width = $image["width"];
$height = $image["height"];
}else{
$thumb_width = null;
$thumb_height = null;
$width = null;
$height = null;
}
$out["image"][] = [
"title" => $image["title"],
"source" => [
[
"url" => $image["image_id"],
"width" => $width,
"height" => $height
],
[
"url" => $image["src"],
"width" => $thumb_width,
"height" => $thumb_height
]
],
"url" => $image["host_page"]
];
}
}
return $out;
}
public function image($get){ public function image($get){
$search = $get["s"]; $search = $get["s"];
@ -335,44 +542,165 @@ class yep{
throw new Exception("Failed to decode JSON"); throw new Exception("Failed to decode JSON");
} }
foreach($json[1]["results"] as $item){ if(isset($json[1]["results"])){
foreach($json[1]["results"] as $item){
if( if(
$item["width"] !== 0 && $item["width"] !== 0 &&
$item["height"] !== 0 $item["height"] !== 0
){ ){
$thumb_width = $item["width"] >= 260 ? 260 : $item["width"]; $thumb_width = $item["width"] >= 260 ? 260 : $item["width"];
$thumb_height = ceil($item["height"] * ($thumb_width / $item["width"])); $thumb_height = ceil($item["height"] * ($thumb_width / $item["width"]));
$width = $item["width"]; $width = $item["width"];
$height = $item["height"]; $height = $item["height"];
}else{ }else{
$thumb_width = null; $thumb_width = null;
$thumb_height = null; $thumb_height = null;
$width = null; $width = null;
$height = null; $height = null;
} }
$out["image"][] = [ $out["image"][] = [
"title" => $item["title"], "title" => $item["title"],
"source" => [ "source" => [
[ [
"url" => $item["image_id"], "url" => $item["image_id"],
"width" => $width, "width" => $width,
"height" => $height "height" => $height
],
[
"url" => $item["src"],
"width" => $thumb_width,
"height" => $thumb_height
]
], ],
[ "url" => $item["host_page"]
"url" => $item["src"], ];
"width" => $thumb_width, }
"height" => $thumb_height
]
],
"url" => $item["host_page"]
];
} }
return $out; return $out;
} }
public function news($get){
$search = $get["s"];
if(strlen($search) === 0){
throw new Exception("Search term is empty!");
}
$country = $get["country"];
$nsfw = $get["nsfw"];
switch($nsfw){
case "yes": $nsfw = "off"; break;
case "maybe": $nsfw = "moderate"; break;
case "no": $nsfw = "strict"; break;
}
$out = [
"status" => "ok",
"npt" => null,
"news" => []
];
try{
// https://api.yep.com/fs/2/search?client=web&gl=CA&no_correct=false&q=undefined+variable+javascript&safeSearch=off&type=web
$json =
json_decode(
$this->get(
$this->backend->get_ip(),
"https://api.yep.com/fs/2/search",
[
"client" => "web",
"gl" => $country == "all" ? $country : strtoupper($country),
"limit" => "99999",
"no_correct" => "false",
"q" => $search,
"safeSearch" => $nsfw,
"type" => "news"
]
),
true
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
//$json = json_decode(file_get_contents("scraper/yep.json"), true);
if($json === null){
throw new Exception("Failed to decode JSON");
}
if(isset($json[1]["results"])){
foreach($json[1]["results"] as $item){
$out["news"][] = [
"title" => $item["title"],
"author" => null,
"description" =>
$this->titledots(
strip_tags(
html_entity_decode(
$item["snippet"]
)
)
),
"date" => strtotime($item["first_seen"]),
"thumb" =>
isset($item["img"]) ?
[
"url" => $this->unshiturl($item["img"]),
"ratio" => "16:9"
] :
[
"url" => null,
"ratio" => null
],
"url" => $item["url"]
];
}
}
return $out;
}
private function titledots($title){
$substr = substr($title, -4);
if(
strpos($substr, "...") !== false ||
strpos($substr, "") !== false
){
return trim(substr($title, 0, -4));
}
return trim($title);
}
private function unshiturl($url){
$newurl = parse_url($url, PHP_URL_QUERY);
parse_str($newurl, $newurl);
if(isset($newurl["url"])){
return $newurl["url"];
}
return $url;
}
} }

View File

@ -121,6 +121,10 @@ $settings = [
"value" => "google", "value" => "google",
"text" => "Google" "text" => "Google"
], ],
[
"value" => "yep",
"text" => "Yep"
],
[ [
"value" => "mojeek", "value" => "mojeek",
"text" => "Mojeek" "text" => "Mojeek"
@ -215,6 +219,10 @@ $settings = [
"value" => "google", "value" => "google",
"text" => "Google" "text" => "Google"
], ],
[
"value" => "yep",
"text" => "Yep"
],
[ [
"value" => "mojeek", "value" => "mojeek",
"text" => "Mojeek" "text" => "Mojeek"