added vimeo

This commit is contained in:
lolcat 2025-08-04 00:00:39 -04:00
parent f30872134f
commit 3dbcf60a3e
4 changed files with 694 additions and 0 deletions

View File

@ -144,6 +144,7 @@ class config{
const PROXY_CURLIE = false;
const PROXY_YT = false; // youtube
const PROXY_SEPIASEARCH = false;
const PROXY_VIMEO = false;
const PROXY_YEP = false;
const PROXY_PINTEREST = false;
const PROXY_SANKAKUCOMPLEX = false;

View File

@ -989,6 +989,7 @@ class frontend{
"display" => "Scraper",
"option" => [
"yt" => "YouTube",
"vimeo" => "Vimeo",
"sepiasearch" => "Sepia Search",
//"fb" => "Facebook videos",
"ddg" => "DuckDuckGo",

688
scraper/vimeo.php Normal file
View File

@ -0,0 +1,688 @@
<?php
class vimeo{
public function __construct(){
include "lib/backend.php";
$this->backend = new backend("vimeo");
include "lib/fuckhtml.php";
$this->fuckhtml = new fuckhtml();
}
public function getfilters($page){
return [
"time" => [
"display" => "Date uploaded", // &filter_uploaded=
"option" => [
"any" => "Any time",
"today" => "Last 24 hours",
"this-week" => "Last 7 days",
"this-month" => "Last 30 days",
"this-year" => "Last 365 days",
]
],
"display" => [
"display" => "Display",
"option" => [
"video" => "Videos",
"ondemand" => "On-Demand ($$)",
"people" => "People",
"channel" => "Channels",
"group" => "Groups"
]
],
"sort" => [
"display" => "Sort by",
"option" => [
"relevance" => "Relevance", // no param
"recent" => "Newest", // &sort=latest&direction=desc
"popular" => "Most popular", // &sort=popularity&direction=desc
"a_z" => "Title, A to Z", // &sort=alphabetical&direction=asc
"z_a" => "Title, Z to A", // &sort=alphabetical&direction=desc
"longest" => "Longest", // &sort=duration&direction=desc
"shortest" => "Shortest", // &sort=duration&direction=asc
]
],
"duration" => [
"display" => "Duration", // &filter_duration=
"option" => [
"any" => "Any duration",
"short" => "Short (less than 4 minutes)",
"medium" => "Medium (4-10 minutes)",
"long" => "Long (over 10 minutes)"
]
],
"resolution" => [
"display" => "Resolution",
"option" => [
"any" => "Any resolution",
"4k" => "4K" // &filter_resolution=4k
]
],
"category" => [
"display" => "Category", // &filter_category=
"option" => [
"any" => "Any category",
"animation" => "Animation",
"comedy" => "Comedy",
"music" => "Music",
"experimental" => "Experimental",
"documentary" => "Documentary",
"identsandanimatedlogos" => "Idents and Animated Logos",
"industry" => "Industry",
"instructionals" => "Instructionals",
"narrative" => "Narrative",
"personal" => "Personal"
]
],
"live" => [
"display" => "Live events",
"option" => [
"any" => "Any",
"yes" => "Live now" // &filter_live=now
]
],
"hdr" => [
"display" => "HDR", // &filter_hdr=
"option" => [
"any" => "Any",
"hdr" => "Any HDR",
"dolby_vision" => "Dolby Vision",
"hdr10" => "HDR10",
"hdr10+" => "HDR10+"
]
],
"vimeo_360" => [
"display" => "Vimeo 360°", // &filter_vimeo_360
"option" => [
"any" => "Any",
"spatial" => "Spatial",
"360" => "360°"
]
],
"price" => [ // &filter_price=
"display" => "Price",
"option" => [
"any" => "Any price",
"free" => "Free",
"paid" => "Paid"
]
],
"collection" => [
"display" => "Vimeo collections",
"option" => [
"any" => "Any collection",
"staff_pick" => "Staff picks" // &filter_staffpicked=true
]
],
"license" => [ // &filter_license=
"display" => "License",
"option" => [
"any" => "Any license",
"by-nc-nd" => "CC BY-NC-ND",
"by" => "CC BY",
"by-nc" => "CC BY-NC",
"by-nc-sa" => "CC BY-NC-SA",
"by-nd" => "CC BY-ND",
"by-sa" => "CC BY-SA",
"cc0" => "CC0"
]
]
];
}
private function get($proxy, $url, $get = [], $jwt = false){
$curlproc = curl_init();
if($get !== []){
$get = http_build_query($get);
$url .= "?" . $get;
}
curl_setopt($curlproc, CURLOPT_URL, $url);
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
if($jwt === false){
curl_setopt(
$curlproc,
CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language: en-US,en;q=0.5",
"Accept-Encoding: gzip, deflate, br, zstd",
"Referer: https://vimeo.com/watch",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Cookie: _dd_s=isExpired=1", // &aid=959c58f0-e5fe-48f6-bd54-e247d92d317a
"Upgrade-Insecure-Requests: 1",
"Sec-Fetch-Dest: document",
"Sec-Fetch-Mode: navigate",
"Sec-Fetch-Site: same-origin",
"Priority: u=0, i"]
);
}else{
curl_setopt(
$curlproc,
CURLOPT_HTTPHEADER,
["User-Agent: " . config::USER_AGENT,
"Accept: application/vnd.vimeo.*+json;version=3.3",
"Accept-Language: en",
"Accept-Encoding: gzip, deflate, br, zstd",
"Referer: https://vimeo.com/",
"Content-Type: application/json",
"Authorization: jwt $jwt",
"Vimeo-Page: /search/[[...slug]]",
"Origin: https://vimeo.com",
"DNT: 1",
"Sec-GPC: 1",
"Connection: keep-alive",
"Sec-Fetch-Dest: empty",
"Sec-Fetch-Mode: cors",
"Sec-Fetch-Site: same-site",
"Priority: u=4"]
);
}
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
$this->backend->assign_proxy($curlproc, $proxy);
$data = curl_exec($curlproc);
if(curl_errno($curlproc)){
throw new Exception(curl_error($curlproc));
}
curl_close($curlproc);
return $data;
}
public function video($get){
// parse shit
if($get["npt"]){
[$npt, $proxy] =
$this->backend
->get(
$get["npt"],
"videos"
);
$npt = json_decode($npt, true);
$pagetype = $npt["pagetype"];
$npt = $npt["npt"];
$jwt = $this->get_jwt($proxy);
try{
$json =
$this->get(
$proxy,
"https://api.vimeo.com" . $npt,
[],
$jwt
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
}else{
$proxy = null;
$jwt = $this->get_jwt($proxy); // this gives us a proxy by reference
// parse filters
$npt = [
"query" => $get["s"],
"page" => 1,
"per_page" => 24,
"facets" => "type"
];
switch($get["display"]){
case "video":
$npt["filter_type"] = "clip";
$npt["fields"] = "clip.name,stats.plays,clip.pictures,clip.user.name,clip.user.link,clip.user.pictures.sizes,clip.uri,clip.stats.plays,clip.duration,clip.created_time,clip.link,clip.description";
break;
case "ondemand":
$npt["filter_type"] = "ondemand";
$npt["sizes"] = "296x744";
$npt["fields"] = "ondemand.link,ondemand.name,ondemand.pictures.sizes,ondemand.metadata.interactions.buy,ondemand.metadata.interactions.rent,ondemand.uri";
break;
case "people":
$npt["filter_type"] = "people";
$npt["fetch_user_profile"] = "1";
$npt["fields"] = "people.name,people.location_details.formatted_address,people.metadata.public_videos.total,people.pictures.sizes,people.link,people.metadata.connections.followers.total,people.skills.name,people.skills.uri,people.background_video,people.uri";
break;
case "channel":
$npt["filter_type"] = "channel";
$npt["fields"] = "channel.name,channel.metadata.connections.users.total,channel.metadata.connections.videos.total,channel.pictures.sizes,channel.link,channel.uri";
break;
case "group":
$npt["filter_type"] = "group";
$npt["fields"] = "group.name,group.metadata.connections.users.total,group.metadata.connections.videos.total,group.pictures.sizes,group.link,group.uri";
break;
}
// only apply filters if we're searching for videos
if($get["display"] == "video"){
switch($get["sort"]){
case "relevance": break; // do nothing
case "recent":
$npt["sort"] = "latest";
$npt["direction"] = "desc";
break;
case "popular":
$npt["sort"] = "popularity";
$npt["direction"] = "desc";
break;
case "a_z":
$npt["sort"] = "alphabetical";
$npt["direction"] = "asc";
break;
case "z_a":
$npt["sort"] = "alphabetical";
$npt["direction"] = "desc";
break;
case "longest":
$npt["sort"] = "duration";
$npt["direction"] = "desc";
break;
case "shortest":
$npt["sort"] = "duration";
$npt["direction"] = "asc";
break;
}
if($get["time"] != "any"){
$npt["filter_uploaded"] = $get["time"];
}
if($get["duration"] != "any"){
$npt["filter_duration"] = $get["duration"];
}
if($get["resolution"] != "any"){
$npt["filter_resolution"] = $get["resolution"];
}
if($get["category"] != "any"){
$npt["filter_category"] = $get["category"];
}
if($get["live"] != "any"){
$npt["filter_live"] = "now";
}
if($get["hdr"] != "any"){
$npt["filter_hdr"] = $get["hdr"];
}
if($get["vimeo_360"] != "any"){
$npt["filter_vimeo_360"] = $get["vimeo_360"];
}
if($get["price"] != "any"){
$npt["filter_price"] = $get["price"];
}
if($get["collection"] == "staff_pick"){
$npt["filter_staffpicked"] = "true";
}
if($get["license"] != "any"){
$npt["filter_license"] = $get["license"];
}
}
$pagetype = $npt["filter_type"];
try{
$json =
$this->get(
$proxy,
"https://api.vimeo.com/search",
$npt,
$jwt
);
}catch(Exception $error){
throw new Exception("Failed to fetch JSON");
}
}
$json = json_decode($json, true);
if($json === null){
throw new Exception("Failed to parse JSON");
}
$out = [
"status" => "ok",
"npt" => null,
"video" => [],
"author" => [],
"livestream" => [],
"playlist" => [],
"reel" => []
];
if(isset($json["error"])){
$error = $json["error"];
if(isset($json["developer_message"])){
$error .= " ({$json["developer_message"]})";
}
throw new Exception("Vimeo returned an error: " . $error);
}
if(!isset($json["data"])){
throw new Exception("Vimeo did not return a data object");
}
switch($pagetype){
case "clip":
foreach($json["data"] as $video){
$video = $video["clip"];
if(isset($video["user"]["pictures"]["sizes"])){
$avatar = $video["user"]["pictures"]["sizes"][count($video["user"]["pictures"]["sizes"]) - 1]["link"];
}else{
$avatar = null;
}
$out["video"][] = [
"title" => $video["name"],
"description" =>
$this->limitstrlen(
$video["description"]
),
"author" => [
"name" => $video["user"]["name"],
"url" => $video["user"]["link"],
"avatar" => $avatar
],
"date" => strtotime($video["created_time"]),
"duration" => (int)$video["duration"],
"views" => (int)$video["stats"]["plays"],
"thumb" => [
"ratio" => "16:9",
"url" => $video["pictures"]["base_link"]
],
"url" => $video["link"]
];
}
break;
case "ondemand":
foreach($json["data"] as $video){
$video = $video["ondemand"];
$description = [];
if(isset($video["metadata"]["interactions"]["rent"]["display_price"])){
$description[] = "Rent for " . $video["metadata"]["interactions"]["rent"]["display_price"];
}
if(isset($video["metadata"]["interactions"]["buy"]["display_price"])){
$description[] = "Buy for " . $video["metadata"]["interactions"]["buy"]["display_price"];
}
$description = implode(", ", $description);
$out["video"][] = [
"title" => $video["name"],
"description" => $description,
"author" => [
"name" => null,
"url" => null,
"avatar" => null
],
"date" => null,
"duration" => null,
"views" => null,
"thumb" => [
"ratio" => "9:16",
"url" => $video["pictures"]["sizes"][0]["link"]
],
"url" => $video["link"]
];
}
break;
case "people":
foreach($json["data"] as $user){
$user = $user["people"];
if(
isset($user["pictures"]["sizes"]) &&
count($user["pictures"]["sizes"]) !== 0
){
$thumb = [
"ratio" => "1:1",
"url" => $user["pictures"]["sizes"][count($user["pictures"]["sizes"]) - 1]["link"]
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["author"][] = [
"title" => $user["name"],
"followers" => (int)$user["metadata"]["connections"]["followers"]["total"],
"description" => $user["metadata"]["public_videos"]["total"] . " videos.",
"thumb" => $thumb,
"url" => $user["link"]
];
}
break;
case "channel":
case "group":
foreach($json["data"] as $channel){
$channel = $channel[$npt["filter_type"]];
if(
isset($channel["pictures"]["sizes"]) &&
count($channel["pictures"]["sizes"]) !== 0
){
$thumb = [
"ratio" => "16:9",
"url" => $channel["pictures"]["sizes"][count($channel["pictures"]["sizes"]) - 1]["link"]
];
}else{
$thumb = [
"ratio" => null,
"url" => null
];
}
$out["author"][] = [
"title" => $channel["name"],
"followers" => (int)$channel["metadata"]["connections"]["users"]["total"],
"description" => $channel["metadata"]["connections"]["videos"]["total"] . " videos.",
"thumb" => $thumb,
"url" => $channel["link"]
];
}
break;
}
//
// get next page
//
if(
isset($json["paging"]["next"]) &&
is_string($json["paging"]["next"])
){
$out["npt"] =
$this->backend
->store(
json_encode([
"npt" => $json["paging"]["next"],
"pagetype" => $pagetype
]),
"videos",
$proxy
);
}
return $out;
}
private function get_jwt(&$proxy){
//
// get jwt token
// it's probably safe to cache this across proxies, cause the jwt doesnt contain an userID
// only an appID, whatever shit that is
// we can only cache it for 5 minutes though, otherwise vimeo cries about it
//
if($proxy === null){
$proxy = $this->backend->get_ip();
}
$jwt = apcu_fetch("vimeo_jwt");
if($jwt === false){
$html =
$this->get(
$proxy,
"https://vimeo.com/search",
[],
false
);
$html =
explode(
'<script id="viewer-bootstrap" type="application/json">',
$html,
2
);
if(count($html) !== 2){
throw new Exception("Failed to find JWT json");
}
$jwt =
json_decode(
$this->fuckhtml
->extract_json(
$html[1]
),
true
);
if($jwt === null){
throw new Exception("Failed to decode JWT json");
}
if(!isset($jwt["jwt"])){
throw new Exception("Failed to grep JWT");
}
$jwt = $jwt["jwt"];
apcu_store("vimeo_jwt", $jwt, 300);
}
return $jwt;
}
private function titledots($title){
$substr = substr($title, -3);
if(
$substr == "..." ||
$substr == ""
){
return trim(substr($title, 0, -3), " \n\r\t\v\x00\0\x0B\xc2\xa0");
}
return trim($title, " \n\r\t\v\x00\0\x0B\xc2\xa0");
}
private function limitstrlen($text){
return
explode(
"\n",
wordwrap(
str_replace(
["\n\r", "\r\n", "\n", "\r"],
" ",
$text
),
300,
"\n"
),
2
)[0];
}
}

View File

@ -265,6 +265,10 @@ $settings = [
"value" => "yt",
"text" => "YouTube"
],
[
"value" => "vimeo",
"text" => "Vimeo"
],
[
"value" => "sepiasearch",
"text" => "Sepia Search"