Detect residential+datacenter IPs
This commit is contained in:
216
mmdb.php
216
mmdb.php
@@ -6,9 +6,15 @@ class mmdb_creator{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
$this->ip_list = [];
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
|
||||
//
|
||||
// Import database
|
||||
// Download data
|
||||
//
|
||||
/*
|
||||
if(!file_exists("data")){ mkdir("data"); }
|
||||
chdir("data");
|
||||
if(!file_exists("blocklist-ipsets")){
|
||||
@@ -23,6 +29,9 @@ class mmdb_creator{
|
||||
chdir("..");
|
||||
}
|
||||
|
||||
echo "Downloading MaxMind's GeoLite2-ASN database...\n";
|
||||
$this->dl("https://git.io/GeoLite2-ASN.mmdb", "GeoLite2-ASN.mmdb");
|
||||
|
||||
echo "Downloading Tunnelbear list\n";
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
|
||||
|
||||
@@ -37,24 +46,95 @@ class mmdb_creator{
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
|
||||
|
||||
echo "Downloading hosting lists... (";
|
||||
if(!file_exists("x4bnet-hosting")){ mkdir("x4bnet-hosting"); }
|
||||
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt", "x4bnet-hosting/ipv4.txt"); echo "1,";
|
||||
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt", "x4bnet-hosting/ipv6.txt"); echo "2)\n";
|
||||
|
||||
echo "Downloading CDN list...\n";
|
||||
$this->dl("https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst", "cdn.lst");
|
||||
|
||||
echo "Downloading Tor exit node list...\n";
|
||||
$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
|
||||
|
||||
chdir("..");
|
||||
echo "Done! Creating MMDB database...\n";
|
||||
chdir("..");*/
|
||||
|
||||
//
|
||||
// Construct ASN blocklist
|
||||
//
|
||||
echo "Extracting ASNs from GeoLite database...\n";
|
||||
|
||||
require "lib/MMDB_ASN_Extractor.php";
|
||||
|
||||
$asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb");
|
||||
$mmdb_asns = $asn_extractor->extract();
|
||||
|
||||
|
||||
//
|
||||
// Scrape ASN classifications from BGP.tools
|
||||
//
|
||||
$good =
|
||||
array_unique(
|
||||
array_merge(
|
||||
$this->parse_bgptools("dsl"),
|
||||
$this->parse_bgptools("mobile")
|
||||
)
|
||||
);
|
||||
|
||||
$bad =
|
||||
array_unique(
|
||||
array_merge(
|
||||
$this->parse_bgptools("cdn"),
|
||||
$this->parse_bgptools("vpsh"),
|
||||
$this->parse_bgptools("vpn")
|
||||
)
|
||||
);
|
||||
|
||||
echo "Processing hosting ASNs...\n";
|
||||
foreach($bad as $bad_asn){
|
||||
|
||||
// if a bad ASN is found to be a DSL/mobile provider, ignore
|
||||
if(in_array($bad_asn, $good)){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!isset($mmdb_asns[$bad_asn])){
|
||||
|
||||
// no IP range available for that ASN
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Hosting: $bad_asn (" . count($mmdb_asns[$bad_asn]) . " ranges)\n";
|
||||
|
||||
foreach($mmdb_asns[$bad_asn] as $ip_range){
|
||||
|
||||
$this->ip_list[$ip_range]["hosting"] = true;
|
||||
}
|
||||
}
|
||||
|
||||
echo "Processing residential ASNs...\n";
|
||||
foreach($good as $good_asn){
|
||||
|
||||
// if a residential ASN is found to be a CDN/VPSH/VPN provider, ignore
|
||||
if(in_array($good_asn, $bad)){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!isset($mmdb_asns[$good_asn])){
|
||||
|
||||
// no IP range available for that ASN
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Residential: $good_asn (" . count($mmdb_asns[$good_asn]) . " ranges)\n";
|
||||
|
||||
foreach($mmdb_asns[$good_asn] as $ip_range){
|
||||
|
||||
$this->ip_list[$ip_range]["residential"] = true;
|
||||
}
|
||||
}
|
||||
|
||||
echo "Clearing memory...\n";
|
||||
unset($good);
|
||||
unset($bad);
|
||||
|
||||
//
|
||||
// Construct database
|
||||
//
|
||||
$this->ip_list = [];
|
||||
|
||||
// import miscelaneous lists
|
||||
$this->import_firehol_lists(
|
||||
@@ -66,9 +146,6 @@ class mmdb_creator{
|
||||
],
|
||||
"tor" => [
|
||||
"tor-exit-list.txt"
|
||||
],
|
||||
"hosting" => [
|
||||
"cdn.lst" // mansourjabin's
|
||||
]
|
||||
],
|
||||
""
|
||||
@@ -85,16 +162,6 @@ class mmdb_creator{
|
||||
"x4bnet-vpn/"
|
||||
);
|
||||
|
||||
$this->import_firehol_lists(
|
||||
[
|
||||
"hosting" => [
|
||||
"ipv4.txt",
|
||||
"ipv6.txt",
|
||||
]
|
||||
],
|
||||
"x4bnet-hosting/"
|
||||
);
|
||||
|
||||
// import firehol lists
|
||||
$this->import_firehol_lists(
|
||||
[
|
||||
@@ -121,13 +188,13 @@ class mmdb_creator{
|
||||
"blocklist-ipsets/"
|
||||
);
|
||||
|
||||
require __DIR__ . '/MMDBWriter.php';
|
||||
require "lib/MMDBWriter.php";
|
||||
|
||||
$w = new MMDBWriter(
|
||||
6, // accepts IPv4 and IPv6 networks
|
||||
'GeoIP-Custom', // database_type
|
||||
['en'], // languages
|
||||
['en' => 'Cloudfish']
|
||||
"GeoIP-Custom", // database_type
|
||||
["en"], // languages
|
||||
["en" => "Cloudfish"]
|
||||
);
|
||||
|
||||
echo "Generating mmdb file\n";
|
||||
@@ -139,12 +206,14 @@ class mmdb_creator{
|
||||
//"city" => "Ashburn",
|
||||
...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
|
||||
...(isset($data["tor"]) ? ["is_tor" => true] : []),
|
||||
...(isset($data["hosting"]) ? ["is_hosting" => true] : [])
|
||||
...(isset($data["hosting"]) ? ["is_hosting" => true] : []),
|
||||
...(isset($data["residential"]) ? ["is_residential" => true] : [])
|
||||
//"asn" => MMDBValue::uint32(15169),
|
||||
]);
|
||||
}
|
||||
|
||||
$w->write(__DIR__ . '/output.mmdb');
|
||||
echo "Saving database...\n";
|
||||
$w->write(__DIR__ . "/output.mmdb");
|
||||
|
||||
echo "done\n";
|
||||
}
|
||||
@@ -182,4 +251,89 @@ class mmdb_creator{
|
||||
$data = file_get_contents($url);
|
||||
file_put_contents($path, $data);
|
||||
}
|
||||
|
||||
public function curl($url){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
|
||||
"User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip, deflate, br",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
]);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function parse_bgptools($tag){
|
||||
|
||||
$page = $this->curl("https://bgp.tools/tags/{$tag}");
|
||||
|
||||
$asns = [];
|
||||
|
||||
$this->fuckhtml->load($page);
|
||||
|
||||
$table =
|
||||
$this->fuckhtml
|
||||
->getElementById("upstreamTable", "table");
|
||||
|
||||
if($table === false){
|
||||
|
||||
throw new Exception("Failed to grep table element on bgptool's {$tag} page");
|
||||
}
|
||||
|
||||
$this->fuckhtml->load($table);
|
||||
|
||||
$trs =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("tr");
|
||||
|
||||
foreach($trs as $tr){
|
||||
|
||||
$this->fuckhtml->load($tr);
|
||||
|
||||
$tds =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("td");
|
||||
|
||||
if(!isset($tds[1])){ continue; }
|
||||
|
||||
$asns[] =
|
||||
strtolower(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$tds[1]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
echo "Scraped bgp.tools/tags/{$tag} (got " . number_format(count($asns)) . " ASNs)\n";
|
||||
|
||||
return $asns;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user