<?php

$s = new mmdb_creator();

class mmdb_creator{
	
	public function __construct(){
		
		$this->ip_list = [];
		
		include "lib/fuckhtml.php";
		$this->fuckhtml = new fuckhtml();
		
		//
		// Download data
		//
		
		if(!file_exists("data")){ mkdir("data"); }
		chdir("data");
		if(!file_exists("blocklist-ipsets")){
			
			echo "Downloading firehol lists\n";
			shell_exec("git clone https://github.com/firehol/blocklist-ipsets");
		}else{
			
			echo "Updating firehol lists\n";
			chdir("blocklist-ipsets");
			echo shell_exec("git pull");
			chdir("..");
		}
		
		echo "Downloading MaxMind's GeoLite2-ASN database...\n";
		$this->dl("https://git.io/GeoLite2-ASN.mmdb", "GeoLite2-ASN.mmdb");
		
		echo "Downloading Tunnelbear list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
		
		echo "Downloading ProtonVPN list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/refs/heads/master/protonvpn_ips.txt", "protonvpn_ips.txt");
		
		echo "Downloading Windscribe list\n";
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "windscribe_ips.txt");
		
		echo "Downloading pia/proton/apple/mullvad lists... (";
		if(!file_exists("x4bnet-vpn")){ mkdir("x4bnet-vpn"); }
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
		$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
		
		echo "Downloading Tor exit node list...\n";
		$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
		
		chdir("..");
		
		//
		// Construct ASN blocklist
		//
		echo "Extracting ASNs from GeoLite database...\n";
		
		require "lib/MMDB_ASN_Extractor.php";
		
		$asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb");
		$mmdb_asns = $asn_extractor->extract(true);
		
		//
		// Scrape ASN classifications from BGP.tools
		//
		$good =
			array_unique(
				array_merge(
					$this->parse_bgptools("dsl"),
					$this->parse_bgptools("mobile")
				)
			);
		
		$bad =
			array_unique(
				array_merge(
					$this->parse_bgptools("cdn"),
					$this->parse_bgptools("vpsh"),
					$this->parse_bgptools("vpn")
				)
			);
		
		echo "Processing hosting ASNs...\n";
		foreach($bad as $bad_asn){
			
			// if a bad ASN is found to be a DSL/mobile provider, ignore
			if(in_array($bad_asn, $good)){
				
				continue;
			}
			
			if(!isset($mmdb_asns[$bad_asn])){
				
				// no IP range available for that ASN
				continue;
			}
			
			echo "Hosting: $bad_asn (" . count($mmdb_asns[$bad_asn]) . " ranges)\n";
			
			foreach($mmdb_asns[$bad_asn] as $ip_range){
				
				$this->ip_list[$ip_range]["is_hosting"] = true;
			}
		}
		
		echo "Processing residential ASNs...\n";
		foreach($good as $good_asn){
			
			// if a residential ASN is found to be a CDN/VPSH/VPN provider, ignore
			if(in_array($good_asn, $bad)){
				
				continue;
			}
			
			if(!isset($mmdb_asns[$good_asn])){
				
				// no IP range available for that ASN
				continue;
			}
			
			echo "Residential: $good_asn (" . count($mmdb_asns[$good_asn]) . " ranges)\n";
			
			foreach($mmdb_asns[$good_asn] as $ip_range){
				
				$this->ip_list[$ip_range]["is_residential"] = true;
			}
		}
		
		echo "Clearing memory...\n";
		unset($good);
		unset($bad);
		
		//
		// Construct database
		//
		
		// import miscelaneous lists
		$this->import_firehol_lists(
			[
				"is_proxy" => [ // tn3w's
					"tunnelbear_ips.txt",
					"protonvpn_ips.txt",
					"windscribe_ips.txt"
				],
				"is_tor" => [
					"tor-exit-list.txt"
				]
			],
			""
		);
		
		// import X4BNet's lists
		$this->import_firehol_lists(
			[
				"is_proxy" => [
					"ipv4.txt",
					"ipv6.txt",
				]
			],
			"x4bnet-vpn/"
		);
		
		// import firehol lists
		$this->import_firehol_lists(
			[
				"is_proxy" => [
					"firehol_anonymous.netset",
					"firehol_abusers_30d.netset",
					"abuseipdb_30d.ipset",
					"spamhaus_drop.netset",
					"dshield_30d.netset",
					"greensnow.ipset",
					"blocklist_de.ipset",
					"bruteforceblocker.ipset",
					"ciarmy.ipset",
					"stopforumspam_90d.ipset",
					"myip.ipset",
					"vxvault.ipset",
					"blocklist_net_ua.ipset",
					"botscout_30d.ipset"
				],
				"is_tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
					"tor_exits.ipset"
				]
			],
			"blocklist-ipsets/"
		);
		
		echo "Sending database as JSON payload to Go helper...\n";
		
		$output = json_encode($this->ip_list);
		
		$process = proc_open(
			"mmdb_writer/ipmmdb",
			[
				0 => ["pipe", "r"], // stdin
				1 => ["pipe", "w"], // stdout
				2 => ["pipe", "w"], // stderr
			],
			$pipes
		);
		
		if(!is_resource($process)){
			echo "Failed to start ipmmdb";
			die();
		}

		fwrite($pipes[0], $output);
		fclose($pipes[0]);

		$stdout = stream_get_contents($pipes[1]);
		$stderr = stream_get_contents($pipes[2]);

		fclose($pipes[1]);
		fclose($pipes[2]);

		$returnCode = proc_close($process);

		if($returnCode !== 0){
			echo "ipmmdb failed: $stderr";
		}
		
		file_put_contents("output.mmdb", $stdout);
		
		echo "done\n";
	}
	
	public function import_firehol_lists($targets_assoc, $path = "blocklist-ipsets/"){
		
		foreach($targets_assoc as $key => $targets){
			
			foreach($targets as $target){
				
				$path_use = "data/{$path}{$target}";
				echo "Reading {$path_use}\n";
				
				$lines = explode("\n", file_get_contents($path_use));
				
				foreach($lines as $line){
					
					$line = trim($line);
					if(
						strlen($line) === 0 ||
						$line[0] == "#"
					){
						
						continue;
					}
					
					$this->ip_list[$line][$key] = true;
				}
			}
		}
	}
	
	public function dl($url, $path){
		
		$data = file_get_contents($url);
		file_put_contents($path, $data);
	}
	
	public function curl($url){
		
		$curlproc = curl_init();
		
		curl_setopt($curlproc, CURLOPT_URL, $url);
		
		curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
		curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
			"User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
			"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
			"Accept-Language: en-US,en;q=0.9",
			"Accept-Encoding: gzip, deflate, br",
			"DNT: 1",
			"Connection: keep-alive",
			"Upgrade-Insecure-Requests: 1",
			"Sec-Fetch-Dest: document",
			"Sec-Fetch-Mode: navigate",
			"Sec-Fetch-Site: none",
			"Sec-Fetch-User: ?1"
		]);
		
		curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
		curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
		curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
		curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
		
		$data = curl_exec($curlproc);
		
		if(curl_errno($curlproc)){
			
			throw new Exception(curl_error($curlproc));
		}
		
		curl_close($curlproc);
		return $data;
	}
	
	public function parse_bgptools($tag){
		
		$page = $this->curl("https://bgp.tools/tags/{$tag}");
		
		$asns = [];
		
		$this->fuckhtml->load($page);
		
		$table =
			$this->fuckhtml
			->getElementById("upstreamTable", "table");
		
		if($table === false){
			
			throw new Exception("Failed to grep table element on bgptool's {$tag} page");
		}
		
		$this->fuckhtml->load($table);
		
		$trs =
			$this->fuckhtml
			->getElementsByTagName("tr");
		
		foreach($trs as $tr){
			
			$this->fuckhtml->load($tr);
			
			$tds =
				$this->fuckhtml
				->getElementsByTagName("td");
			
			if(!isset($tds[1])){ continue; }
			
			$asns[] =
				strtolower(
					$this->fuckhtml
					->getTextContent(
						$tds[1]
					)
				);
		}
		
		echo "Scraped bgp.tools/tags/{$tag} (got " . number_format(count($asns)) . " ASNs)\n";
		
		return $asns;
	}
}