lolcat/4get
4
2
Fork 0
Code Issues 24 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

fix request count not resetting, added request graph, im so tired

Browse Source
This commit is contained in:
lolcat
2026-09-05 19:02:06 -04:00
parent 90232152e9
commit be3abbef16
17 changed files with 526 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ami4get.php
+16 -4
View File
@@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *");
include "data/config.php";
$real_requests = apcu_fetch("real_requests");
$bot_requests = apcu_fetch("captcha_gen");
// get request count in last 24 hours
$bot_requests = 0;
$real_requests = 0;
$time = time();
for($i=0; $i<24; $i++){
$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
}
for($i=0; $i<24; $i++){
$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
}
echo json_encode(
[
@@ -16,8 +28,8 @@ echo json_encode(
"name" => config::SERVER_NAME,
"description" => config::SERVER_LONG_DESCRIPTION,
"bot_protection" => config::BOT_PROTECTION,
"real_requests" => $real_requests === false ? 0 : $real_requests,
"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
"real_requests" => $real_requests,
"bot_requests" => $bot_requests,
"api_enabled" => config::API_ENABLED,
"alt_addresses" => config::ALT_ADDRESSES,
"version" => config::VERSION
api/v1/images.php
+3
View File
@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->image($get),
api/v1/music.php
+3
View File
@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->music($get),
api/v1/news.php
+3
View File
@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->news($get),
api/v1/videos.php
+3
View File
@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false);
$get = $frontend->parsegetfilters($_GET, $filters);
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo json_encode(
$scraper->video($get),
api/v1/web.php
+3
View File
@@ -38,6 +38,9 @@ if(
$get["extendedsearch"] = "no";
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
try{
echo
api/v2/graph.php
+76
View File
@@ -0,0 +1,76 @@
<?php
date_default_timezone_set('America/Toronto');
function get_requests($slug){
$time = time();
$requests = [];
for($i=0; $i<73; $i++){
$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
$requests[] = [$i, ($data === false ? 0 : $data)];
}
// rtrim() out all entries that are 0
$out = [];
$found = false;
for($i=count($requests) - 1; $i>=0; $i--){
if($requests[$i][1] !== 0){
$found = $i;
break;
}
}
if($found === false){
return [];
}
$requests = array_slice($requests, 0, $found + 1);
return $requests;
}
$bot_requests = get_requests("bot_requests");
$real_requests = get_requests("real_requests");
$real_reqs_24h = 0;
for($i=0; $i<count($real_requests); $i++){
if($real_requests[$i][1] !== false){
$real_reqs_24h += $real_requests[$i][1];
}
}
$real_reqs_72h = 0;
for($i=0; $i<count($real_requests); $i++){
if($real_requests[$i][1] !== false){
$real_reqs_72h += $real_requests[$i][1];
}
}
header("Content-Type: image/png");
include "../../lib/graph_gen.php";
$graph = new graph();
echo $graph->render([
"title" => "Number of requests within the last 72 hours (non-cummulative)",
"subtitle" => date("jS M y @ g:ia", time()) . " || 24h=" . number_format($real_reqs_24h) . ", 72h=" . number_format($real_reqs_72h) . " || localhost:8000",
"x_label" => "<= Hours passed =>",
"y_label" => "<= Request count/hour =>",
"grad_x" => 25,
"grad_y" => 20,
"data" => [
"Bot requests" => $bot_requests,
"Real requests" => $real_requests,
]
]);
data/config.php
+2 -2
View File
@@ -5,7 +5,7 @@ class config{
// any parameters.
// 4get version. Please keep this updated
const VERSION = 8;
const VERSION = 9;
// Will be shown pretty much everywhere.
const SERVER_NAME = "4get";
@@ -134,7 +134,7 @@ class config{
// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
// Changing this might break things.
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0";
// User agent to use with 4get-friendly APIs
const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";
images.php
+3
View File
@@ -32,6 +32,9 @@ try{
$frontend->drawscrapererror($error->getMessage(), $get, "images", $payload["timetaken"]);
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
if(count($results["image"]) === 0){
$payload["images"] =
lib/bot_protection.php
+2 -5
View File
@@ -7,7 +7,6 @@ class bot_protection{
// check if we want captcha
if(config::BOT_PROTECTION !== 1){
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@@ -46,8 +45,6 @@ class bot_protection{
}else{
// the cookie is OK! dont die() and give results
apcu_inc("real_requests");
if($output === true){
$frontend->loadheader(
$get,
@@ -164,8 +161,7 @@ class bot_protection{
$key = "k" . $inc . "." . $this->randomchars();
apcu_inc($key, 1, $stupid, 86400);
apcu_inc("real_requests");
apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
setcookie(
"pass",
@@ -190,6 +186,7 @@ class bot_protection{
}
}
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200);
$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
$payload = [
lib/frontend.php
+7 -1
View File
@@ -177,7 +177,7 @@ class frontend{
){
// bot detected !!
apcu_inc("captcha_gen");
apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
$this->drawerror(
"Tshh, blocked!",
@@ -1052,6 +1052,12 @@ class frontend{
return http_build_query($out);
}
public function increment_real_reqs($scraper){
apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
}
public function htmlimage($image, $format){
if(
lib/graph_gen.php
+388
View File
@@ -0,0 +1,388 @@
<?php
class graph{
public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
$graph = array_merge([
"title" => "Title",
"subtitle" => "description",
"grad_x" => 25,
"grad_y" => 20,
"x_label" => "<= x =>",
"y_label" => "<= y =>",
"antialias" => false,
"data" => []
], $graph);
$size = array_merge([
"image_width" => 750,
"image_height" => 500,
"x" => 90,
"y" => 50,
"width" => 650,
"height" => 300,
"square" => 11,
"column" => 6
], $size);
$colors = array_merge([
"bg" => new ImagickPixel("#1d2021"),
"lines_bg" => new ImagickPixel("#3c3836"),
"outline" => new ImagickPixel("#a89984"),
"text" => new ImagickPixel("#ebdbb2"),
"transparent" => new ImagickPixel("transparent")
], $colors);
$colors_line = array_merge([
new ImagickPixel("#83a598"),
new ImagickPixel("#fb4934"),
new ImagickPixel("#b8bb26"),
new ImagickPixel("#fabd2f"),
new ImagickPixel("#d3869b"),
new ImagickPixel("#8ec07c"),
new ImagickPixel("#fe8019"),
new ImagickPixel("#458588"),
new ImagickPixel("#cc241d"),
new ImagickPixel("#98971a"),
new ImagickPixel("#d79921"),
new ImagickPixel("#b16286"),
new ImagickPixel("#689d6a"),
new ImagickPixel("#d65d0e"),
new ImagickPixel("#a89984") // other
], $colors_line);
$not_enough_data = false;
// compute datapoint max ranges
$max_range_x = 0;
$max_range_y = 0;
if(count($graph["data"]) === 0){
$not_enough_data = true;
}else{
foreach($graph["data"] as $datapoint => $value){
if(count($value) < 1){
$not_enough_data = true;
}else{
$nv1 = [];
$nv2 = [];
foreach($value as $k => $v){
$nv1[] = $v[0];
$nv2[] = $v[1];
}
$max1 = max($nv1);
$max2 = max($nv2);
if($max_range_x < $max1){ $max_range_x = $max1; }
if($max_range_y < $max2){ $max_range_y = $max2; }
}
}
}
$im = new Imagick();
$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
$im->setImageFormat("png");
$draw = new ImagickDraw();
//
// Draw background grid
//
$draw->setStrokeColor($colors["lines_bg"]);
$draw->setStrokeAntialias($graph["antialias"]);
$draw->setStrokeWidth(1);
$draw->setFillColor($colors["transparent"]);
$text = new ImagickDraw();
$text->setFontSize(11);
$text->setTextAntialias($graph["antialias"]);
$text->setFillColor($colors["text"]);
// vertical lines
$offset = $size["width"] / ($graph["grad_x"] - 1);
for($i=0; $i<$graph["grad_x"]; $i++){
$x = round($size["x"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_x"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$x,
$size["y"],
$x,
$size["y"] + $size["height"],
);
if($not_enough_data === false){
// add numbas
$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
$im->annotateImage(
$text,
round($x - ($number_width / 2)),
$size["y"] + $size["height"] + 14,
0,
$number
);
}
}
// horizontal lines
$text->setTextAlignment(Imagick::ALIGN_RIGHT);
$offset = $size["height"] / ($graph["grad_y"] - 1);
$vertical_text_size = 0;
for($i=0; $i<$graph["grad_y"]; $i++){
$y = round($size["y"] + ($offset * $i));
if(
$i === 0 ||
$i === $graph["grad_y"] - 1
){
$draw->setStrokeColor($colors["outline"]);
}else{
$draw->setStrokeColor($colors["lines_bg"]);
}
$draw->line(
$size["x"] + 1,
$y,
$size["x"] + $size["width"] - 1,
$y,
);
if($not_enough_data === false){
$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
$query = $im->queryFontMetrics($text, $val);
if($query["textWidth"] > $vertical_text_size){
$vertical_text_size = $query["textWidth"];
}
// add numbas
$im->annotateImage(
$text,
$size["x"] - 4,
$y + 4,
0,
$val
);
}
}
//
// Draw legend
//
$text->setTextAlignment(Imagick::ALIGN_LEFT);
$top_padding = 44;
$c = count($colors_line);
$padding = $size["width"] / $size["column"];
$line = new ImagickDraw();
//$line->setStrokeWidth(1);
$line->setStrokeAntialias($graph["antialias"]);
if($not_enough_data === false){
$i = 0;
foreach($graph["data"] as $datafield => $data){
$used_color = $colors_line[$i % $c];
$draw->setStrokeColor($used_color);
$row = floor($i / $size["column"]);
$x = round($size["x"] + (($i % $size["column"]) * $padding));
$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
// draw legend
$draw->rectangle(
$x,
$y,
$x + $size["square"],
$y + $size["square"]
);
$im->annotateImage(
$text,
$x + $size["square"] + 4,
$y + 10,
0,
$datafield
);
//
// Draw lines
//
$line->setStrokeColor($used_color);
$old_x = null;
$old_y = null;
$continue = true;
foreach($data as $field){
// field 0 = hour (sorted)
// field 1 = datapoint
$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
if($continue){
$continue = false;
$old_x = $x;
$old_y = $y;
continue;
}
$line->line($old_x, $old_y, $x, $y);
$old_x = $x;
$old_y = $y;
}
$i++;
}
}
//
// add background labels
//
$text->setFontSize(13);
$text->setFillColor($colors["bg"]);
$draw->setFillColor($colors["text"]);
$draw->setStrokeColor($colors["transparent"]);
// horizontal
$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
$draw->rectangle(
$x_x - 2,
$x_y - $label_width_x["textHeight"] + 4,
$x_x + $label_width_x["textWidth"] + 2,
$x_y + 3,
);
// vertical
$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
$draw->rectangle(
$y_x + 4,
$y_y - 2,
$y_x + $label_width_y["textHeight"] + 3,
$y_y + $label_width_y["textWidth"] + 2,
);
$im->drawImage($draw);
$im->drawImage($line);
// horizontal text
$im->annotateImage(
$text,
$x_x,
$x_y,
0,
$graph["x_label"]
);
// vertical text
$im->annotateImage(
$text,
$y_x + $label_width_y["textHeight"],
$y_y + $label_width_y["textWidth"],
270,
$graph["y_label"]
);
//
// draw title + subtext
//
$text->setFontSize(20);
$text->setFontWeight(900);
$text->setFillColor($colors["text"]);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 25,
0,
$graph["title"]
);
$text->setFontSize(13);
$text->setFontWeight(100);
$im->annotateImage(
$text,
$size["x"],
$size["y"] - 10,
0,
$graph["subtitle"]
);
if($not_enough_data){
// add not enough data notice
$rect = new ImagickDraw();
$rect->setFillColor($colors["bg"]);
$rect->setStrokeColor($colors_line[1]);
$rect_w = 200;
$rect_h = 24;
$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
$rect->rectangle(
$rect_x,
$rect_y,
$rect_x + $rect_w,
$rect_y + $rect_h
);
$im->drawImage($rect);
$label = "Not enough data";
$text->setFontWeight(900);
$query = $im->queryFontMetrics($text, $label);
$im->annotateImage(
$text,
round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
0,
$label
);
}
return $im->getImageBlob();
}
}
music.php
+3
View File
@@ -34,6 +34,9 @@ try{
$frontend->drawscrapererror($error->getMessage(), $get, "music", $payload["timetaken"]);
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
$categories = [
"song" => "",
"author" => "",
news.php
+3
View File
@@ -34,6 +34,9 @@ try{
$frontend->drawscrapererror($error->getMessage(), $get, "news", $payload["timetaken"]);
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
/*
Populate links
*/
static/serverping.js
+3 -3
View File
@@ -22,7 +22,7 @@ var list = [];
var pinged_list = [];
var reqs = 0;
var errors = 0;
var sort = 6; // highest version first
var sort = 12;
// check for instance redirect stuff
var redir = [];
@@ -68,8 +68,8 @@ table.innerHTML =
'<th class="extend">Server</th>' +
'<th>Address</th>' +
'<th>Bot protection</th>' +
'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
'<th>API</th>' +
'<th><div class="arrow up"></div>Version</th>' +
'</tr>' +
videos.php
+3
View File
@@ -34,6 +34,9 @@ try{
$frontend->drawscrapererror($error->getMessage(), $get, "videos", $payload["timetaken"]);
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
$categories = [
"video" => "",
"author" => "",
web.php
+3
View File
@@ -34,6 +34,9 @@ try{
$frontend->drawscrapererror($error->getMessage(), $get, "web", $payload["timetaken"]);
}
// increment request count
$frontend->increment_real_reqs($get["scraper"]);
/*
Prepend Oracle output, if applicable
*/