diff --git a/ami4get.php b/ami4get.php index 5bb9273..993dd6f 100644 --- a/ami4get.php +++ b/ami4get.php @@ -5,8 +5,20 @@ header("Access-Control-Allow-Origin: *"); include "data/config.php"; -$real_requests = apcu_fetch("real_requests"); -$bot_requests = apcu_fetch("captcha_gen"); +// get request count in last 24 hours +$bot_requests = 0; +$real_requests = 0; +$time = time(); + +for($i=0; $i<24; $i++){ + + $bot_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".bot_requests"); +} + +for($i=0; $i<24; $i++){ + + $real_requests += apcu_fetch(intdiv($time - (3600 * $i), 3600) . ".real_requests"); +} echo json_encode( [ @@ -16,8 +28,8 @@ echo json_encode( "name" => config::SERVER_NAME, "description" => config::SERVER_LONG_DESCRIPTION, "bot_protection" => config::BOT_PROTECTION, - "real_requests" => $real_requests === false ? 0 : $real_requests, - "bot_requests" => $bot_requests === false ? 0 : $bot_requests, + "real_requests" => $real_requests, + "bot_requests" => $bot_requests, "api_enabled" => config::API_ENABLED, "alt_addresses" => config::ALT_ADDRESSES, "version" => config::VERSION diff --git a/api/v1/images.php b/api/v1/images.php index de2c5a9..de5dd64 100644 --- a/api/v1/images.php +++ b/api/v1/images.php @@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "images", false); $get = $frontend->parsegetfilters($_GET, $filters); +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + try{ echo json_encode( $scraper->image($get), diff --git a/api/v1/music.php b/api/v1/music.php index 58985e3..4c8b2a4 100644 --- a/api/v1/music.php +++ b/api/v1/music.php @@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "music", false); $get = $frontend->parsegetfilters($_GET, $filters); +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + try{ echo json_encode( $scraper->music($get), diff --git a/api/v1/news.php b/api/v1/news.php index ab38781..d837c2c 100644 --- a/api/v1/news.php +++ b/api/v1/news.php @@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "news", false); $get = $frontend->parsegetfilters($_GET, $filters); +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + try{ echo json_encode( $scraper->news($get), diff --git a/api/v1/videos.php b/api/v1/videos.php index 1d23780..65065b0 100644 --- a/api/v1/videos.php +++ b/api/v1/videos.php @@ -27,6 +27,9 @@ new bot_protection($null, $null, $null, "videos", false); $get = $frontend->parsegetfilters($_GET, $filters); +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + try{ echo json_encode( $scraper->video($get), diff --git a/api/v1/web.php b/api/v1/web.php index 6a9c030..dfde3bd 100644 --- a/api/v1/web.php +++ b/api/v1/web.php @@ -38,6 +38,9 @@ if( $get["extendedsearch"] = "no"; } +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + try{ echo diff --git a/api/v2/graph.php b/api/v2/graph.php new file mode 100644 index 0000000..2c09bda --- /dev/null +++ b/api/v2/graph.php @@ -0,0 +1,76 @@ +=0; $i--){ + + if($requests[$i][1] !== 0){ + + $found = $i; + break; + } + } + + if($found === false){ + + return []; + } + + $requests = array_slice($requests, 0, $found + 1); + + return $requests; +} + +$bot_requests = get_requests("bot_requests"); +$real_requests = get_requests("real_requests"); + +$real_reqs_24h = 0; +for($i=0; $irender([ + "title" => "Number of requests within the last 72 hours (non-cummulative)", + "subtitle" => date("jS M y @ g:ia", time()) . " || 24h=" . number_format($real_reqs_24h) . ", 72h=" . number_format($real_reqs_72h) . " || localhost:8000", + "x_label" => "<= Hours passed =>", + "y_label" => "<= Request count/hour =>", + "grad_x" => 25, + "grad_y" => 20, + "data" => [ + "Bot requests" => $bot_requests, + "Real requests" => $real_requests, + ] +]); diff --git a/data/config.php b/data/config.php index fdf2aff..04059f8 100644 --- a/data/config.php +++ b/data/config.php @@ -5,7 +5,7 @@ class config{ // any parameters. // 4get version. Please keep this updated - const VERSION = 8; + const VERSION = 9; // Will be shown pretty much everywhere. const SERVER_NAME = "4get"; @@ -134,7 +134,7 @@ class config{ // Default user agent to use for scraper requests. Sometimes ignored to get specific webpages // Changing this might break things. - const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0"; + const USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:155.0) Gecko/20100101 Firefox/155.0"; // User agent to use with 4get-friendly APIs const USER_AGENT_FRIENDLY = "4get-scrapist (+https://4get.ca)"; diff --git a/images.php b/images.php index 99fc9d6..b3e3bdf 100644 --- a/images.php +++ b/images.php @@ -32,6 +32,9 @@ try{ $frontend->drawscrapererror($error->getMessage(), $get, "images", $payload["timetaken"]); } +// increment request count +$frontend->increment_real_reqs($get["scraper"]); + if(count($results["image"]) === 0){ $payload["images"] = diff --git a/lib/bot_protection.php b/lib/bot_protection.php index 5a823e0..59f36ce 100644 --- a/lib/bot_protection.php +++ b/lib/bot_protection.php @@ -7,7 +7,6 @@ class bot_protection{ // check if we want captcha if(config::BOT_PROTECTION !== 1){ - apcu_inc("real_requests"); if($output === true){ $frontend->loadheader( $get, @@ -32,7 +31,7 @@ class bot_protection{ // does key exist apcu_exists($_COOKIE["pass"]) ){ - + // exists, increment counter $inc = apcu_inc($_COOKIE["pass"]); @@ -46,8 +45,6 @@ class bot_protection{ }else{ // the cookie is OK! dont die() and give results - apcu_inc("real_requests"); - if($output === true){ $frontend->loadheader( $get, @@ -164,8 +161,7 @@ class bot_protection{ $key = "k" . $inc . "." . $this->randomchars(); apcu_inc($key, 1, $stupid, 86400); - - apcu_inc("real_requests"); + apcu_dec(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800); setcookie( "pass", @@ -190,6 +186,7 @@ class bot_protection{ } } + apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 259200); $key = "c" . apcu_inc("captcha_gen", 1) . "." . $this->randomchars(); $payload = [ diff --git a/lib/frontend.php b/lib/frontend.php index 3f7e54a..8af0e1f 100644 --- a/lib/frontend.php +++ b/lib/frontend.php @@ -177,7 +177,7 @@ class frontend{ ){ // bot detected !! - apcu_inc("captcha_gen"); + apcu_inc(intdiv(time(), 3600) . ".bot_requests", 1, $s, 262800); $this->drawerror( "Tshh, blocked!", @@ -221,7 +221,7 @@ class frontend{ 'This scraper returned an error:' . '
' . htmlspecialchars($error) . '
' . 'Things you can try:' . - '