lolcat/Cloudfish
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

detect residential/datacenter flags on ipv6

Browse Source
This commit is contained in:
lolcat
2026-08-16 03:48:17 -04:00
parent 51a0909499
commit 6c427baab9
7 changed files with 297 additions and 738 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
mmdb.php
View File

@@ -14,6 +14,7 @@ class mmdb_creator{
//
// Download data
//
if(!file_exists("data")){ mkdir("data"); }
chdir("data");
if(!file_exists("blocklist-ipsets")){
@@ -58,8 +59,7 @@ class mmdb_creator{
require "lib/MMDB_ASN_Extractor.php";
$asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb");
$mmdb_asns = $asn_extractor->extract();
$mmdb_asns = $asn_extractor->extract(true);
//
// Scrape ASN classifications from BGP.tools
@@ -100,7 +100,7 @@ class mmdb_creator{
foreach($mmdb_asns[$bad_asn] as $ip_range){
$this->ip_list[$ip_range]["hosting"] = true;
$this->ip_list[$ip_range]["is_hosting"] = true;
}
}
@@ -123,7 +123,7 @@ class mmdb_creator{
foreach($mmdb_asns[$good_asn] as $ip_range){
$this->ip_list[$ip_range]["residential"] = true;
$this->ip_list[$ip_range]["is_residential"] = true;
}
}
@@ -138,12 +138,12 @@ class mmdb_creator{
// import miscelaneous lists
$this->import_firehol_lists(
[
"proxy" => [ // tn3w's
"is_proxy" => [ // tn3w's
"tunnelbear_ips.txt",
"protonvpn_ips.txt",
"windscribe_ips.txt"
],
"tor" => [
"is_tor" => [
"tor-exit-list.txt"
]
],
@@ -153,7 +153,7 @@ class mmdb_creator{
// import X4BNet's lists
$this->import_firehol_lists(
[
"proxy" => [
"is_proxy" => [
"ipv4.txt",
"ipv6.txt",
]
@@ -164,7 +164,7 @@ class mmdb_creator{
// import firehol lists
$this->import_firehol_lists(
[
"proxy" => [
"is_proxy" => [
"firehol_anonymous.netset",
"firehol_abusers_30d.netset",
"abuseipdb_30d.ipset",
@@ -180,39 +180,48 @@ class mmdb_creator{
"blocklist_net_ua.ipset",
"botscout_30d.ipset"
],
"tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
"is_tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
"tor_exits.ipset"
]
],
"blocklist-ipsets/"
);
require "lib/MMDBWriter.php";
$w = new MMDBWriter(
6, // accepts IPv4 and IPv6 networks
"GeoIP-Custom", // database_type
["en"], // languages
["en" => "Cloudfish"]
echo "Sending database as JSON payload to Go helper...\n";
$output = json_encode($this->ip_list);
$process = proc_open(
"mmdb_writer/ipmmdb",
[
0 => ["pipe", "r"], // stdin
1 => ["pipe", "w"], // stdout
2 => ["pipe", "w"], // stderr
],
$pipes
);
echo "Generating mmdb file\n";
foreach($this->ip_list as $ip => $data){
$w->addRecord($ip, [
//"country" => "US",
//"city" => "Ashburn",
...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
...(isset($data["tor"]) ? ["is_tor" => true] : []),
...(isset($data["hosting"]) ? ["is_hosting" => true] : []),
...(isset($data["residential"]) ? ["is_residential" => true] : [])
//"asn" => MMDBValue::uint32(15169),
]);
if(!is_resource($process)){
echo "Failed to start ipmmdb";
die();
}
fwrite($pipes[0], $output);
fclose($pipes[0]);
$stdout = stream_get_contents($pipes[1]);
$stderr = stream_get_contents($pipes[2]);
fclose($pipes[1]);
fclose($pipes[2]);
$returnCode = proc_close($process);
if($returnCode !== 0){
echo "ipmmdb failed: $stderr";
}
echo "Saving database...\n";
$w->write(__DIR__ . "/output.mmdb");
file_put_contents("output.mmdb", $stdout);
echo "done\n";
}