Cloudfish/mmdb.php

<?php

$s = new mmdb_creator();

class mmdb_creator{
	
	public function __construct(){
		
		//
		// Import database
		//
		if(!file_exists("data")){ mkdir("data"); }
		chdir("data");
		if(!file_exists("blocklist-ipsets")){
			
			echo "Downloading firehol lists\n";
			shell_exec("git clone https://github.com/firehol/blocklist-ipsets");
		}else{
			
			echo "Updating firehol lists\n";
			chdir("blocklist-ipsets");
			echo shell_exec("git pull");
			chdir("..");
		}
		
		echo "Downloading Tunnelbear list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
		
		echo "Downloading ProtonVPN list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/refs/heads/master/protonvpn_ips.txt", "protonvpn_ips.txt");
		
		echo "Downloading Windscribe list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "windscribe_ips.txt");
		
		echo "Downloading pia/proton/apple/mullvad lists... (";
		if(!file_exists("x4bnet-vpn")){ mkdir("x4bnet-vpn"); }
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
		
		echo "Downloading hosting lists... (";
		if(!file_exists("x4bnet-hosting")){ mkdir("x4bnet-hosting"); }
		$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt", "x4bnet-hosting/ipv4.txt"); echo "1,";
		$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt", "x4bnet-hosting/ipv6.txt"); echo "2)\n";
		
		echo "Downloading CDN list...\n";
		$this->dl("https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst", "cdn.lst");
		
		echo "Downloading Tor exit node list...\n";
		$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
		
		chdir("..");
		echo "Done! Creating MMDB database...\n";
		
		//
		// Construct database
		//
		$this->ip_list = [];
		
		// import miscelaneous lists
		$this->import_firehol_lists(
			[
				"proxy" => [ // tn3w's
					"tunnelbear_ips.txt",
					"protonvpn_ips.txt",
					"windscribe_ips.txt"
				],
				"tor" => [
					"tor-exit-list.txt"
				],
				"hosting" => [
					"cdn.lst" // mansourjabin's
				]
			],
			""
		);
		
		// import X4BNet's lists
		$this->import_firehol_lists(
			[
				"proxy" => [
					"ipv4.txt",
					"ipv6.txt",
				]
			],
			"x4bnet-vpn/"
		);
		
		$this->import_firehol_lists(
			[
				"hosting" => [
					"ipv4.txt",
					"ipv6.txt",
				]
			],
			"x4bnet-hosting/"
		);
		
		// import firehol lists
		$this->import_firehol_lists(
			[
				"proxy" => [
					"firehol_anonymous.netset",
					"spamhaus_drop.netset",
					"dshield_30d.netset",
					"greensnow.ipset",
					"blocklist_de.ipset",
					"bruteforceblocker.ipset",
					"ciarmy.ipset",
					"stopforumspam_90d.ipset",
					"myip.ipset",
					"vxvault.ipset",
					"blocklist_net_ua.ipset",
					"botscout_30d.ipset",
					"cybercure.ipset",
					"cybercrime.ipset"
				],
				"tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
					"dm_tor.ipset"
				]
			],
			"blocklist-ipsets/"
		);
		
		require __DIR__ . '/MMDBWriter.php';

		$w = new MMDBWriter(
			6, // accepts IPv4 and IPv6 networks
			'GeoIP-Custom', // database_type
			['en'], // languages
			['en' => 'Cloudfish']
		);
		
		echo "Generating mmdb file\n";
		
		foreach($this->ip_list as $ip => $data){
			
			$w->addRecord($ip, [
				//"country" => "US",
				//"city" => "Ashburn",
				...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
				...(isset($data["tor"]) ? ["is_tor" => true] : []),
				...(isset($data["hosting"]) ? ["is_hosting" => true] : [])
				//"asn" => MMDBValue::uint32(15169),
			]);
		}
		
		$w->write(__DIR__ . '/output.mmdb');
		
		echo "done\n";
	}
	
	public function import_firehol_lists($targets_assoc, $path = "blocklist-ipsets/"){
		
		foreach($targets_assoc as $key => $targets){
			
			foreach($targets as $target){
				
				$path_use = "data/{$path}{$target}";
				echo "Reading {$path_use}\n";
				
				$lines = explode("\n", file_get_contents($path_use));
				
				foreach($lines as $line){
					
					$line = trim($line);
					if(
						strlen($line) === 0 ||
						$line[0] == "#"
					){
						
						continue;
					}
					
					$this->ip_list[$line][$key] = true;
				}
			}
		}
	}
	
	public function dl($url, $path){
		
		$data = file_get_contents($url);
		file_put_contents($path, $data);
	}
}