339 lines
7.7 KiB
PHP
339 lines
7.7 KiB
PHP
<?php
|
|
|
|
$s = new mmdb_creator();
|
|
|
|
class mmdb_creator{
|
|
|
|
public function __construct(){
|
|
|
|
$this->ip_list = [];
|
|
|
|
include "lib/fuckhtml.php";
|
|
$this->fuckhtml = new fuckhtml();
|
|
|
|
//
|
|
// Download data
|
|
//
|
|
if(!file_exists("data")){ mkdir("data"); }
|
|
chdir("data");
|
|
if(!file_exists("blocklist-ipsets")){
|
|
|
|
echo "Downloading firehol lists\n";
|
|
shell_exec("git clone https://github.com/firehol/blocklist-ipsets");
|
|
}else{
|
|
|
|
echo "Updating firehol lists\n";
|
|
chdir("blocklist-ipsets");
|
|
echo shell_exec("git pull");
|
|
chdir("..");
|
|
}
|
|
|
|
echo "Downloading MaxMind's GeoLite2-ASN database...\n";
|
|
$this->dl("https://git.io/GeoLite2-ASN.mmdb", "GeoLite2-ASN.mmdb");
|
|
|
|
echo "Downloading Tunnelbear list\n";
|
|
$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
|
|
|
|
echo "Downloading ProtonVPN list\n";
|
|
$this->dl("https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/refs/heads/master/protonvpn_ips.txt", "protonvpn_ips.txt");
|
|
|
|
echo "Downloading Windscribe list\n";
|
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "windscribe_ips.txt");
|
|
|
|
echo "Downloading pia/proton/apple/mullvad lists... (";
|
|
if(!file_exists("x4bnet-vpn")){ mkdir("x4bnet-vpn"); }
|
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
|
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
|
|
|
|
echo "Downloading Tor exit node list...\n";
|
|
$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
|
|
|
|
chdir("..");
|
|
|
|
//
|
|
// Construct ASN blocklist
|
|
//
|
|
echo "Extracting ASNs from GeoLite database...\n";
|
|
|
|
require "lib/MMDB_ASN_Extractor.php";
|
|
|
|
$asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb");
|
|
$mmdb_asns = $asn_extractor->extract();
|
|
|
|
|
|
//
|
|
// Scrape ASN classifications from BGP.tools
|
|
//
|
|
$good =
|
|
array_unique(
|
|
array_merge(
|
|
$this->parse_bgptools("dsl"),
|
|
$this->parse_bgptools("mobile")
|
|
)
|
|
);
|
|
|
|
$bad =
|
|
array_unique(
|
|
array_merge(
|
|
$this->parse_bgptools("cdn"),
|
|
$this->parse_bgptools("vpsh"),
|
|
$this->parse_bgptools("vpn")
|
|
)
|
|
);
|
|
|
|
echo "Processing hosting ASNs...\n";
|
|
foreach($bad as $bad_asn){
|
|
|
|
// if a bad ASN is found to be a DSL/mobile provider, ignore
|
|
if(in_array($bad_asn, $good)){
|
|
|
|
continue;
|
|
}
|
|
|
|
if(!isset($mmdb_asns[$bad_asn])){
|
|
|
|
// no IP range available for that ASN
|
|
continue;
|
|
}
|
|
|
|
echo "Hosting: $bad_asn (" . count($mmdb_asns[$bad_asn]) . " ranges)\n";
|
|
|
|
foreach($mmdb_asns[$bad_asn] as $ip_range){
|
|
|
|
$this->ip_list[$ip_range]["hosting"] = true;
|
|
}
|
|
}
|
|
|
|
echo "Processing residential ASNs...\n";
|
|
foreach($good as $good_asn){
|
|
|
|
// if a residential ASN is found to be a CDN/VPSH/VPN provider, ignore
|
|
if(in_array($good_asn, $bad)){
|
|
|
|
continue;
|
|
}
|
|
|
|
if(!isset($mmdb_asns[$good_asn])){
|
|
|
|
// no IP range available for that ASN
|
|
continue;
|
|
}
|
|
|
|
echo "Residential: $good_asn (" . count($mmdb_asns[$good_asn]) . " ranges)\n";
|
|
|
|
foreach($mmdb_asns[$good_asn] as $ip_range){
|
|
|
|
$this->ip_list[$ip_range]["residential"] = true;
|
|
}
|
|
}
|
|
|
|
echo "Clearing memory...\n";
|
|
unset($good);
|
|
unset($bad);
|
|
|
|
//
|
|
// Construct database
|
|
//
|
|
|
|
// import miscelaneous lists
|
|
$this->import_firehol_lists(
|
|
[
|
|
"proxy" => [ // tn3w's
|
|
"tunnelbear_ips.txt",
|
|
"protonvpn_ips.txt",
|
|
"windscribe_ips.txt"
|
|
],
|
|
"tor" => [
|
|
"tor-exit-list.txt"
|
|
]
|
|
],
|
|
""
|
|
);
|
|
|
|
// import X4BNet's lists
|
|
$this->import_firehol_lists(
|
|
[
|
|
"proxy" => [
|
|
"ipv4.txt",
|
|
"ipv6.txt",
|
|
]
|
|
],
|
|
"x4bnet-vpn/"
|
|
);
|
|
|
|
// import firehol lists
|
|
$this->import_firehol_lists(
|
|
[
|
|
"proxy" => [
|
|
"firehol_anonymous.netset",
|
|
"firehol_abusers_30d.netset",
|
|
"abuseipdb_30d.ipset",
|
|
"spamhaus_drop.netset",
|
|
"dshield_30d.netset",
|
|
"greensnow.ipset",
|
|
"blocklist_de.ipset",
|
|
"bruteforceblocker.ipset",
|
|
"ciarmy.ipset",
|
|
"stopforumspam_90d.ipset",
|
|
"myip.ipset",
|
|
"vxvault.ipset",
|
|
"blocklist_net_ua.ipset",
|
|
"botscout_30d.ipset"
|
|
],
|
|
"tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
|
|
"tor_exits.ipset"
|
|
]
|
|
],
|
|
"blocklist-ipsets/"
|
|
);
|
|
|
|
require "lib/MMDBWriter.php";
|
|
|
|
$w = new MMDBWriter(
|
|
6, // accepts IPv4 and IPv6 networks
|
|
"GeoIP-Custom", // database_type
|
|
["en"], // languages
|
|
["en" => "Cloudfish"]
|
|
);
|
|
|
|
echo "Generating mmdb file\n";
|
|
|
|
foreach($this->ip_list as $ip => $data){
|
|
|
|
$w->addRecord($ip, [
|
|
//"country" => "US",
|
|
//"city" => "Ashburn",
|
|
...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
|
|
...(isset($data["tor"]) ? ["is_tor" => true] : []),
|
|
...(isset($data["hosting"]) ? ["is_hosting" => true] : []),
|
|
...(isset($data["residential"]) ? ["is_residential" => true] : [])
|
|
//"asn" => MMDBValue::uint32(15169),
|
|
]);
|
|
}
|
|
|
|
echo "Saving database...\n";
|
|
$w->write(__DIR__ . "/output.mmdb");
|
|
|
|
echo "done\n";
|
|
}
|
|
|
|
public function import_firehol_lists($targets_assoc, $path = "blocklist-ipsets/"){
|
|
|
|
foreach($targets_assoc as $key => $targets){
|
|
|
|
foreach($targets as $target){
|
|
|
|
$path_use = "data/{$path}{$target}";
|
|
echo "Reading {$path_use}\n";
|
|
|
|
$lines = explode("\n", file_get_contents($path_use));
|
|
|
|
foreach($lines as $line){
|
|
|
|
$line = trim($line);
|
|
if(
|
|
strlen($line) === 0 ||
|
|
$line[0] == "#"
|
|
){
|
|
|
|
continue;
|
|
}
|
|
|
|
$this->ip_list[$line][$key] = true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
public function dl($url, $path){
|
|
|
|
$data = file_get_contents($url);
|
|
file_put_contents($path, $data);
|
|
}
|
|
|
|
public function curl($url){
|
|
|
|
$curlproc = curl_init();
|
|
|
|
curl_setopt($curlproc, CURLOPT_URL, $url);
|
|
|
|
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
|
curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
|
|
"User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
|
|
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
|
"Accept-Language: en-US,en;q=0.9",
|
|
"Accept-Encoding: gzip, deflate, br",
|
|
"DNT: 1",
|
|
"Connection: keep-alive",
|
|
"Upgrade-Insecure-Requests: 1",
|
|
"Sec-Fetch-Dest: document",
|
|
"Sec-Fetch-Mode: navigate",
|
|
"Sec-Fetch-Site: none",
|
|
"Sec-Fetch-User: ?1"
|
|
]);
|
|
|
|
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
|
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
|
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
|
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
|
|
|
$data = curl_exec($curlproc);
|
|
|
|
if(curl_errno($curlproc)){
|
|
|
|
throw new Exception(curl_error($curlproc));
|
|
}
|
|
|
|
curl_close($curlproc);
|
|
return $data;
|
|
}
|
|
|
|
public function parse_bgptools($tag){
|
|
|
|
$page = $this->curl("https://bgp.tools/tags/{$tag}");
|
|
|
|
$asns = [];
|
|
|
|
$this->fuckhtml->load($page);
|
|
|
|
$table =
|
|
$this->fuckhtml
|
|
->getElementById("upstreamTable", "table");
|
|
|
|
if($table === false){
|
|
|
|
throw new Exception("Failed to grep table element on bgptool's {$tag} page");
|
|
}
|
|
|
|
$this->fuckhtml->load($table);
|
|
|
|
$trs =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("tr");
|
|
|
|
foreach($trs as $tr){
|
|
|
|
$this->fuckhtml->load($tr);
|
|
|
|
$tds =
|
|
$this->fuckhtml
|
|
->getElementsByTagName("td");
|
|
|
|
if(!isset($tds[1])){ continue; }
|
|
|
|
$asns[] =
|
|
strtolower(
|
|
$this->fuckhtml
|
|
->getTextContent(
|
|
$tds[1]
|
|
)
|
|
);
|
|
}
|
|
|
|
echo "Scraped bgp.tools/tags/{$tag} (got " . number_format(count($asns)) . " ASNs)\n";
|
|
|
|
return $asns;
|
|
}
|
|
}
|