fix request count not resetting, added request graph, im so tired

ami4get.php

@@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *");
 
 include "data/config.php";
 
-$real_requests = apcu_fetch("real_requests");
+// get request count in last 24 hours
-$bot_requests = apcu_fetch("captcha_gen");
+$bot_requests = 0;
+$real_requests = 0;
+$time = time();
+
+for($i=0; $i<24; $i++){
+	
+	$bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests");
+}
+
+for($i=0; $i<24; $i++){
+	
+	$real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests");
+}
 
 echo json_encode(
 	[
@@ -16,8 +28,8 @@ echo json_encode(
 			"name" => config::SERVER_NAME,
 			"description" => config::SERVER_LONG_DESCRIPTION,
 			"bot_protection" => config::BOT_PROTECTION,
-			"real_requests" => $real_requests === false ? 0 : $real_requests,
+			"real_requests" => $real_requests,
-			"bot_requests" => $bot_requests === false ? 0 : $bot_requests,
+			"bot_requests" => $bot_requests,
 			"api_enabled" => config::API_ENABLED,
 			"alt_addresses" => config::ALT_ADDRESSES,
 			"version" => config::VERSION

api/v1/images.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->image($get),

api/v1/music.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->music($get),

api/v1/news.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->news($get),

api/v1/videos.php

@@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false);
 
 $get = $frontend->parsegetfilters($_GET, $filters);
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	echo json_encode(
 		$scraper->video($get),

api/v1/web.php

@@ -38,6 +38,9 @@ if(
 	$get["extendedsearch"] = "no";
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 try{
 	
 	echo

api/v2/graph.php

@@ -0,0 +1,76 @@
+<?php
+
+date_default_timezone_set('America/Toronto');
+
+function get_requests($slug){
+	
+	$time = time();
+	$requests = [];
+	
+	for($i=0; $i<73; $i++){
+		
+		$data = apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".$slug");
+		
+		$requests[] = [$i, ($data === false ? 0 : $data)];
+	}
+	
+	// rtrim() out all entries that are 0
+	$out = [];
+	$found = false;
+	for($i=count($requests) - 1; $i>=0; $i--){
+		
+		if($requests[$i][1] !== 0){
+			
+			$found = $i;
+			break;
+		}
+	}
+	
+	if($found === false){
+		
+		return [];
+	}
+	
+	$requests = array_slice($requests, 0, $found + 1);
+	
+	return $requests;
+}
+
+$bot_requests = get_requests("bot_requests");
+$real_requests = get_requests("real_requests");
+
+$real_reqs_24h = 0;
+for($i=0; $i<count($real_requests); $i++){
+	
+	if($real_requests[$i][1] !== false){
+		
+		$real_reqs_24h += $real_requests[$i][1];
+	}
+}
+
+$real_reqs_72h = 0;
+for($i=0; $i<count($real_requests); $i++){
+	
+	if($real_requests[$i][1] !== false){
+		
+		$real_reqs_72h += $real_requests[$i][1];
+	}
+}
+
+header("Content-Type: image/png");
+
+include "../../lib/graph_gen.php";
+$graph = new graph();
+
+echo $graph->render([
+	"title" => "Number of requests within the last 72 hours (non-cummulative)",
+	"subtitle" => date("jS M y @ g:ia", time()) . " || 24h=" . number_format($real_reqs_24h) . ", 72h=" . number_format($real_reqs_72h) . " || localhost:8000",
+	"x_label" => "<= Hours passed =>",
+	"y_label" => "<= Request count/hour =>",
+	"grad_x" => 25,
+	"grad_y" => 20,
+	"data" => [
+		"Bot requests" => $bot_requests,
+		"Real requests" => $real_requests,
+	]
+]);

data/config.php

@@ -5,7 +5,7 @@ class config{
 	// any parameters.
 	
 	// 4get version. Please keep this updated
-	const VERSION = 8;
+	const VERSION = 9;
 	
 	// Will be shown pretty much everywhere.
 	const SERVER_NAME = "4get";
@@ -134,7 +134,7 @@ class config{
 	
 	// Default user agent to use for scraper requests. Sometimes ignored to get specific webpages
 	// Changing this might break things.
-	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0";
+	const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0";
 	
 	// User agent to use with 4get-friendly APIs
 	const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)";

images.php

@@ -32,6 +32,9 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "images", $payload["timetaken"]);
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 if(count($results["image"]) === 0){
 	
 	$payload["images"] =

lib/bot_protection.php

@@ -7,7 +7,6 @@ class bot_protection{
 		// check if we want captcha
 		if(config::BOT_PROTECTION !== 1){
 			
-			apcu_inc("real_requests");
 			if($output === true){
 				$frontend->loadheader(
 					$get,
@@ -32,7 +31,7 @@ class bot_protection{
 				// does key exist
 				apcu_exists($_COOKIE["pass"])
 			){
-					
+				
 				// exists, increment counter
 				$inc = apcu_inc($_COOKIE["pass"]);
 				
@@ -46,8 +45,6 @@ class bot_protection{
 				}else{
 					
 					// the cookie is OK! dont die() and give results
-					apcu_inc("real_requests");
-					
 					if($output === true){
 						$frontend->loadheader(
 							$get,
@@ -164,8 +161,7 @@ class bot_protection{
 				$key = "k" . $inc . "." . $this->randomchars();
 				
 				apcu_inc($key, 1, $stupid, 86400);
-				
+				apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
-				apcu_inc("real_requests");
 				
 				setcookie(
 					"pass",
@@ -190,6 +186,7 @@ class bot_protection{
 			}
 		}
 		
+		apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200);
 		$key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars();
 		
 		$payload = [

lib/frontend.php

@@ -177,7 +177,7 @@ class frontend{
 		){
 			
 			// bot detected !!
-			apcu_inc("captcha_gen");
+			apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800);
 			
 			$this->drawerror(
 				"Tshh, blocked!",
@@ -221,7 +221,7 @@ class frontend{
 			'This scraper returned an error:' .
 			'<div class="code">' . htmlspecialchars($error) . '</div>' .
 			'Things you can try:' .
-			'<ul>' . 
+			'<ul>' .
 				'<li>Use a different scraper</li>' .
 				'<li>Remove keywords that could cause errors</li>' .
 				'<li><a href="/instances?target=' . $target . "&" . $this->buildquery($get, false) . '">Try your search on another 4get instance</a></li>' .
@@ -1052,6 +1052,12 @@ class frontend{
 		return http_build_query($out);
 	}
 	
+	public function increment_real_reqs($scraper){
+		
+		apcu_inc(intdiv(time(), 3600) . ".real_requests", 1, $s, 262800);
+		apcu_inc(intdiv(time(), 3600) . ".$scraper.requests", 1, $s, 262800);
+	}
+	
 	public function htmlimage($image, $format){
 		
 		if(

lib/graph_gen.php

@@ -0,0 +1,388 @@
+<?php
+
+class graph{
+	
+	public function render(array $graph = [], array $size = [], array $colors = [], array $colors_line = []){
+		
+		$graph = array_merge([
+			"title" => "Title",
+			"subtitle" => "description",
+			"grad_x" => 25,
+			"grad_y" => 20,
+			"x_label" => "<= x =>",
+			"y_label" => "<= y =>",
+			"antialias" => false,
+			"data" => []
+		], $graph);
+		
+		$size = array_merge([
+			"image_width" => 750,
+			"image_height" => 500,
+			"x" => 90,
+			"y" => 50,
+			"width" => 650,
+			"height" => 300,
+			"square" => 11,
+			"column" => 6
+		], $size);
+		
+		$colors = array_merge([
+			"bg" =>				new ImagickPixel("#1d2021"),
+			"lines_bg" =>		new ImagickPixel("#3c3836"),
+			"outline" =>		new ImagickPixel("#a89984"),
+			"text" =>			new ImagickPixel("#ebdbb2"),
+			"transparent" =>	new ImagickPixel("transparent")
+		], $colors);
+		
+		$colors_line = array_merge([
+			new ImagickPixel("#83a598"),
+			new ImagickPixel("#fb4934"),
+			new ImagickPixel("#b8bb26"),
+			new ImagickPixel("#fabd2f"),
+			new ImagickPixel("#d3869b"),
+			new ImagickPixel("#8ec07c"),
+			new ImagickPixel("#fe8019"),
+			new ImagickPixel("#458588"),
+			new ImagickPixel("#cc241d"),
+			new ImagickPixel("#98971a"),
+			new ImagickPixel("#d79921"),
+			new ImagickPixel("#b16286"),
+			new ImagickPixel("#689d6a"),
+			new ImagickPixel("#d65d0e"),
+			new ImagickPixel("#a89984") // other
+		], $colors_line);
+		
+		$not_enough_data = false;
+		
+		// compute datapoint max ranges
+		$max_range_x = 0;
+		$max_range_y = 0;
+		if(count($graph["data"]) === 0){
+			
+			$not_enough_data = true;
+		}else{
+			
+			foreach($graph["data"] as $datapoint => $value){
+				
+				if(count($value) < 1){
+					
+					$not_enough_data = true;
+				}else{
+					
+					$nv1 = [];
+					$nv2 = [];
+					foreach($value as $k => $v){
+						
+						$nv1[] = $v[0];
+						$nv2[] = $v[1];
+					}
+					
+					$max1 = max($nv1);
+					$max2 = max($nv2);
+					if($max_range_x < $max1){ $max_range_x = $max1; }
+					if($max_range_y < $max2){ $max_range_y = $max2; }
+				}
+			}
+		}
+
+		$im = new Imagick();
+		$im->newImage($size["image_width"], $size["image_height"], $colors["bg"]);
+		$im->setImageFormat("png");
+
+		$draw = new ImagickDraw();
+
+		//
+		// Draw background grid
+		//
+		$draw->setStrokeColor($colors["lines_bg"]);
+		$draw->setStrokeAntialias($graph["antialias"]);
+		$draw->setStrokeWidth(1);
+		$draw->setFillColor($colors["transparent"]);
+
+		$text = new ImagickDraw();
+		$text->setFontSize(11);
+		$text->setTextAntialias($graph["antialias"]);
+		$text->setFillColor($colors["text"]);
+
+
+		// vertical lines
+		$offset = $size["width"] / ($graph["grad_x"] - 1);
+
+		for($i=0; $i<$graph["grad_x"]; $i++){
+			
+			$x = round($size["x"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_x"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$x,
+				$size["y"],
+				$x,
+				$size["y"] + $size["height"],
+			);
+			
+			if($not_enough_data === false){
+				// add numbas
+				$number = number_format(round(($graph["grad_x"] - 1 - $i) * ($max_range_x / ($graph["grad_x"] - 1))));
+				$number_width = $im->queryFontMetrics($text, $number)["textWidth"];
+				
+				$im->annotateImage(
+					$text,
+					round($x - ($number_width / 2)),
+					$size["y"] + $size["height"] + 14,
+					0,
+					$number
+				);
+			}
+		}
+
+
+		// horizontal lines
+		$text->setTextAlignment(Imagick::ALIGN_RIGHT);
+		$offset = $size["height"] / ($graph["grad_y"] - 1);
+
+		$vertical_text_size = 0;
+
+		for($i=0; $i<$graph["grad_y"]; $i++){
+			
+			$y = round($size["y"] + ($offset * $i));
+			
+			if(
+				$i === 0 ||
+				$i === $graph["grad_y"] - 1
+			){
+				
+				$draw->setStrokeColor($colors["outline"]);
+			}else{
+				
+				$draw->setStrokeColor($colors["lines_bg"]);
+			}
+			
+			$draw->line(
+				$size["x"] + 1,
+				$y,
+				$size["x"] + $size["width"] - 1,
+				$y,
+			);
+			
+			if($not_enough_data === false){
+				$val = number_format(round(($graph["grad_y"] - 1 - $i) * ($max_range_y / ($graph["grad_y"] - 1))));
+				$query = $im->queryFontMetrics($text, $val);
+				
+				if($query["textWidth"] > $vertical_text_size){
+					
+					$vertical_text_size = $query["textWidth"];
+				}
+				
+				// add numbas
+				$im->annotateImage(
+					$text,
+					$size["x"] - 4,
+					$y + 4,
+					0,
+					$val
+				);
+			}
+		}
+
+
+		//
+		// Draw legend
+		//
+		$text->setTextAlignment(Imagick::ALIGN_LEFT);
+		$top_padding = 44;
+		$c = count($colors_line);
+		$padding = $size["width"] / $size["column"];
+
+		$line = new ImagickDraw();
+		//$line->setStrokeWidth(1);
+		$line->setStrokeAntialias($graph["antialias"]);
+
+		if($not_enough_data === false){
+			$i = 0;
+			foreach($graph["data"] as $datafield => $data){
+				
+				$used_color = $colors_line[$i % $c];
+				$draw->setStrokeColor($used_color);
+				$row = floor($i / $size["column"]);
+				
+				$x = round($size["x"] + (($i % $size["column"]) * $padding));
+				$y = $size["y"] + $size["height"] + $top_padding + ($row * 17);
+				
+				// draw legend
+				$draw->rectangle(
+					$x,
+					$y,
+					$x + $size["square"],
+					$y + $size["square"]
+				);
+				
+				$im->annotateImage(
+					$text,
+					$x + $size["square"] + 4,
+					$y + 10,
+					0,
+					$datafield
+				);
+				
+				//
+				// Draw lines
+				//
+				$line->setStrokeColor($used_color);
+				$old_x = null;
+				$old_y = null;
+				$continue = true;
+				
+				foreach($data as $field){
+					
+					// field 0 = hour (sorted)
+					// field 1 = datapoint
+					
+					$x = $size["x"] + (($max_range_x - $field[0]) * ($size["width"] / $max_range_x));
+					$y = $size["y"] + (($max_range_y - $field[1]) * ($size["height"] / $max_range_y));
+					
+					if($continue){
+						
+						$continue = false;
+						$old_x = $x;
+						$old_y = $y;
+						continue;
+					}
+					
+					$line->line($old_x, $old_y, $x, $y);
+							
+					$old_x = $x;
+					$old_y = $y;
+				}
+				
+				$i++;
+			}
+		}
+
+		//
+		// add background labels
+		//
+		$text->setFontSize(13);
+		$text->setFillColor($colors["bg"]);
+		$draw->setFillColor($colors["text"]);
+		$draw->setStrokeColor($colors["transparent"]);
+
+		// horizontal
+		$label_width_x = $im->queryFontMetrics($text, $graph["x_label"]);
+		$x_x = $size["x"] + ($size["width"] / 2) - ($label_width_x["textWidth"] / 2);
+		$x_y = $size["y"] + $size["height"] + ($not_enough_data ? 21 : 30);
+
+		$draw->rectangle(
+			$x_x - 2,
+			$x_y - $label_width_x["textHeight"] + 4,
+			$x_x + $label_width_x["textWidth"] + 2,
+			$x_y + 3,
+		);
+
+		// vertical
+		$label_width_y = $im->queryFontMetrics($text, $graph["y_label"]);
+		$y_x = $size["x"] - $label_width_y["textHeight"] - 13 - $vertical_text_size;
+		$y_y = $size["y"] + ($size["height"] / 2) - ($label_width_y["textWidth"] / 2);
+
+		$draw->rectangle(
+			$y_x + 4,
+			$y_y - 2,
+			$y_x + $label_width_y["textHeight"] + 3,
+			$y_y + $label_width_y["textWidth"] + 2,
+		);
+
+		$im->drawImage($draw);
+		$im->drawImage($line);
+
+		// horizontal text
+		$im->annotateImage(
+			$text,
+			$x_x,
+			$x_y,
+			0,
+			$graph["x_label"]
+		);
+
+		// vertical text
+		$im->annotateImage(
+			$text,
+			$y_x + $label_width_y["textHeight"],
+			$y_y + $label_width_y["textWidth"],
+			270,
+			$graph["y_label"]
+		);
+
+		//
+		// draw title + subtext
+		//
+		$text->setFontSize(20);
+		$text->setFontWeight(900);
+		$text->setFillColor($colors["text"]);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 25,
+			0,
+			$graph["title"]
+		);
+
+		$text->setFontSize(13);
+		$text->setFontWeight(100);
+
+		$im->annotateImage(
+			$text,
+			$size["x"],
+			$size["y"] - 10,
+			0,
+			$graph["subtitle"]
+		);
+
+		if($not_enough_data){
+			
+			// add not enough data notice
+			$rect = new ImagickDraw();
+			$rect->setFillColor($colors["bg"]);
+			$rect->setStrokeColor($colors_line[1]);
+			
+			$rect_w = 200;
+			$rect_h = 24;
+				
+			$rect_x = round($size["x"] + ($size["width"] / 2) - ($rect_w / 2));
+			$rect_y = round($size["y"] + ($size["height"] / 2) - ($rect_h / 2));
+			
+			$rect->rectangle(
+				$rect_x,
+				$rect_y,
+				$rect_x + $rect_w,
+				$rect_y + $rect_h
+			);
+			
+			$im->drawImage($rect);
+			
+			$label = "Not enough data";
+			
+			$text->setFontWeight(900);
+			$query = $im->queryFontMetrics($text, $label);
+			
+			$im->annotateImage(
+				$text,
+				round($size["x"] + ($size["width"] / 2) - ($query["textWidth"] / 2)),
+				round($size["y"] + ($size["height"] / 2) + ($query["textHeight"] / 2)) - 3,
+				0,
+				$label
+			);
+		}
+
+		return $im->getImageBlob();
+	}
+}

music.php

@@ -34,6 +34,9 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "music", $payload["timetaken"]);
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 $categories = [
 	"song" => "",
 	"author" => "",

news.php

@@ -34,6 +34,9 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "news", $payload["timetaken"]);
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 /*
 	Populate links
 */

static/serverping.js

@@ -22,7 +22,7 @@ var list = [];
 var pinged_list = [];
 var reqs = 0;
 var errors = 0;
-var sort = 6; // highest version first
+var sort = 12;
 
 // check for instance redirect stuff
 var redir = [];
@@ -68,8 +68,8 @@ table.innerHTML =
 		'<th class="extend">Server</th>' +
 		'<th>Address</th>' +
 		'<th>Bot protection</th>' +
-		'<th title="Amount of legit requests processed since the last APCU cache clear (usually happens at midnight)">Real reqs (?)</th>' +
+		'<th title="Amount of legit requests processed within the last 24 hours">Real reqs (?)</th>' +
-		'<th title="Amount of filtered requests processed since the last APCU cache clear (usually happens at midnight)">Bot reqs (?)</th>' +
+		'<th title="Amount of filtered requests processed within the last 24 hours">Bot reqs (?)</th>' +
 		'<th>API</th>' +
 		'<th><div class="arrow up"></div>Version</th>' +
 	'</tr>' +

videos.php

@@ -34,6 +34,9 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "videos", $payload["timetaken"]);
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 $categories = [
 	"video" => "",
 	"author" => "",

web.php

@@ -34,6 +34,9 @@ try{
 	$frontend->drawscrapererror($error->getMessage(), $get, "web", $payload["timetaken"]);
 }
 
+// increment request count
+$frontend->increment_real_reqs($get["scraper"]);
+
 /*
 	Prepend Oracle output, if applicable
 */