From d61e2e7b351a3e826eef2fb8ea637260e9bfc20c Mon Sep 17 00:00:00 2001 From: lolcat Date: Sun, 16 Aug 2026 02:29:19 -0400 Subject: [PATCH] Detect residential+datacenter IPs --- README.md | 29 +- MMDBWriter.php => lib/MMDBWriter.php | 321 ++++++++++++-- lib/MMDB_ASN_Extractor.php | 395 +++++++++++++++++ lib/fuckhtml.php | 628 +++++++++++++++++++++++++++ mmdb.php | 216 +++++++-- 5 files changed, 1515 insertions(+), 74 deletions(-) rename MMDBWriter.php => lib/MMDBWriter.php (60%) create mode 100644 lib/MMDB_ASN_Extractor.php create mode 100644 lib/fuckhtml.php diff --git a/README.md b/README.md index 178bc6b..9ca527c 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,13 @@ print_r($cf_lookup); // ["is_hosting" => true] **Warning**: Any field may be missing. Fields that are set are always set to `true`. +# Generate database +Just run that shit and hope it works + +```sh +php mmdb.php +``` + # Data sources ## Proxy detection ([firehol](https://iplists.firehol.org/)) @@ -45,8 +52,21 @@ print_r($cf_lookup); // ["is_hosting" => true] - pia/proton/apple/mullvad [ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv4.txt), [ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/)) ## Hosting detection -- [datacenter-ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt), [datacenter-ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/)) -- [CDN list](https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst) ([source](https://github.com/mansourjabin/cdn-ip-database)) +- [GeoLite2-ASN](https://git.io/GeoLite2-ASN.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb)) +- ASN purpose index ([source](https://bgp.tools)): + - Residential signals + - [Home ISP](https://bgp.tools/tags/dsl) + - [Mobile Data/Carrier](https://bgp.tools/tags/mobile) + - Hosting signals + - [Content Delivery Network](https://bgp.tools/tags/cdn) + - [Server Hosting](https://bgp.tools/tags/vpsh) + - [VPN Host](https://bgp.tools/tags/vpn) + +### Important! +- An ASN's IP range(s) are assigned `is_hosting` if they report *ANY* hosting signal, but it *MUST NOT* have a residential signal. +- An ASN's IP range(s) are assigned `is_residential` if they report a residential signal, but it *MUST NOT* have a hosting signal. + +This method cover everything, some IPs will not receive a category. But it shouldn't falseflag. ## Tor detection - [tor exit node list](https://openinternet.io/tor/tor-exit-list.txt) ([source](https://openinternet.io)) @@ -55,9 +75,6 @@ print_r($cf_lookup); // ["is_hosting" => true] # Recommendation I recommend these additional MMDB databases to complement Cloudfish. -## ASN detection -- [GeoLite2-ASN](https://git.io/GeoLite2-ASN.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb)) - ## Country/city detection - [GeoLite2-City](https://git.io/GeoLite2-City.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb)) @@ -65,7 +82,7 @@ I recommend these additional MMDB databases to complement Cloudfish. I wrote a simple IP lookup script. It's available at `ip.lolcat.ca`, `ip4.lolcat.ca` and `ip6.lolcat.ca`. ### Disclaimer -I transpiled MMDBWriter.php from it's original Go implementation using Claude. I'll make my own when I have more time, no idea if the filesize could be reduced. +I transpiled MMDBWriter.php from it's original Go implementation using Claude. I'll make my own when I have more time, no idea if the filesize could be reduced. I also asked it to create a MMDB ASN extractor. Lol. # License AGPLv3, make sure to credit all blocklists used, they all have their own fuckass licenses. diff --git a/MMDBWriter.php b/lib/MMDBWriter.php similarity index 60% rename from MMDBWriter.php rename to lib/MMDBWriter.php index 70b286d..ed9891f 100644 --- a/MMDBWriter.php +++ b/lib/MMDBWriter.php @@ -114,6 +114,31 @@ final class MMDBEncoder } private static function encodeTyped(int $type, $value): string + { + if ($type !== MMDBType::MAP && $type !== MMDBType::ARRAY_T) { + return self::encodeScalar($type, $value); + } + switch ($type) { + case MMDBType::MAP: + $out = self::controlAndSize($type, count($value)); + foreach ($value as $k => $v) { + $out .= self::encodeTyped(MMDBType::STRING, (string) $k); + $out .= self::encode($v); + } + return $out; + + case MMDBType::ARRAY_T: + $out = self::controlAndSize($type, count($value)); + foreach ($value as $v) { + $out .= self::encode($v); + } + return $out; + } + } + + /** Encode a single non-composite (leaf) MMDB value to bytes. Public so + * MMDBDataSection can reuse it for the leaves of its deduped structures. */ + public static function encodeScalar(int $type, $value): string { switch ($type) { case MMDBType::STRING: @@ -156,21 +181,6 @@ final class MMDBEncoder // Boolean has no payload; the value lives in the size field. return self::controlAndSize($type, $value ? 1 : 0); - case MMDBType::MAP: - $out = self::controlAndSize($type, count($value)); - foreach ($value as $k => $v) { - $out .= self::encodeTyped(MMDBType::STRING, (string) $k); - $out .= self::encode($v); - } - return $out; - - case MMDBType::ARRAY_T: - $out = self::controlAndSize($type, count($value)); - foreach ($value as $v) { - $out .= self::encode($v); - } - return $out; - default: throw new InvalidArgumentException("Unsupported MMDB type: $type"); } @@ -191,7 +201,7 @@ final class MMDBEncoder * for a given MMDB type and payload size, per the spec's control * byte format. */ - private static function controlAndSize(int $type, int $size): string + public static function controlAndSize(int $type, int $size): string { $typeBits = $type <= 7 ? $type : 0; // 0 = "look at next byte for real type" $out = ''; @@ -217,6 +227,157 @@ final class MMDBEncoder } } +// --------------------------------------------------------------------- +// Deduplicating data section builder. Any value (string, number, or +// whole map/array structure) that has already been written once is +// replaced everywhere else with a 2-5 byte MMDB "pointer" record instead +// of being re-encoded, which is where most of the size savings on +// real-world data (repeated keys like "country", repeated country codes, +// repeated sub-structures) comes from. +// --------------------------------------------------------------------- +final class MMDBDataSection +{ + private $buffer = ''; + /** @var array cache key => byte offset of its real (non-pointer) encoding */ + private $cache = []; + + public function bytes(): string + { + return $this->buffer; + } + + /** + * Ensure $value is present in the data section (writing it if this is + * the first time it's been seen) and return the offset of its real + * encoding. Used for top-level records referenced directly by tree + * leaves, so this always returns a real offset, never a pointer. + */ + public function internTop($value): int + { + [$type, $raw, $cacheKey] = $this->classify($value); + if ($cacheKey !== null && isset($this->cache[$cacheKey])) { + return $this->cache[$cacheKey]; + } + $offset = strlen($this->buffer); + if ($cacheKey !== null) { + $this->cache[$cacheKey] = $offset; + } + $this->appendEncoding($type, $raw); + return $offset; + } + + /** Append bytes for a value nested inside a map/array: a pointer if + * it's a repeat, otherwise its full encoding (registering it for + * future reuse). */ + private function writeRef($value): void + { + [$type, $raw, $cacheKey] = $this->classify($value); + if ($cacheKey !== null && isset($this->cache[$cacheKey])) { + $this->buffer .= $this->pointerBytes($this->cache[$cacheKey]); + return; + } + $offset = strlen($this->buffer); + if ($cacheKey !== null) { + $this->cache[$cacheKey] = $offset; + } + $this->appendEncoding($type, $raw); + } + + private function appendEncoding(int $type, $raw): void + { + if ($type === MMDBType::MAP) { + $this->buffer .= MMDBEncoder::controlAndSize(MMDBType::MAP, count($raw)); + foreach ($raw as $k => $v) { + $this->writeRef((string) $k); + $this->writeRef($v); + } + return; + } + if ($type === MMDBType::ARRAY_T) { + $this->buffer .= MMDBEncoder::controlAndSize(MMDBType::ARRAY_T, count($raw)); + foreach ($raw as $v) { + $this->writeRef($v); + } + return; + } + $this->buffer .= MMDBEncoder::encodeScalar($type, $raw); + } + + /** @return array{0:int,1:mixed,2:?string} [type, raw value, cache key or null if not worth caching] */ + private function classify($value): array + { + if ($value instanceof MMDBValue) { + $type = $value->type; + $raw = $value->value; + } elseif (is_bool($value)) { + $type = MMDBType::BOOLEAN; + $raw = $value; + } elseif (is_int($value)) { + if ($value >= 0 && $value <= 0xFFFFFFFF) { + $type = MMDBType::UINT32; + } elseif ($value < 0 && $value >= -2147483648) { + $type = MMDBType::INT32; + } else { + $type = MMDBType::UINT64; + } + $raw = $value; + } elseif (is_float($value)) { + $type = MMDBType::DOUBLE; + $raw = $value; + } elseif (is_string($value)) { + $type = MMDBType::STRING; + $raw = $value; + } elseif (is_array($value)) { + $type = $this->isList($value) ? MMDBType::ARRAY_T : MMDBType::MAP; + $raw = $value; + } else { + throw new InvalidArgumentException('Unsupported value type: ' . gettype($value)); + } + + // A pointer costs 2-5 bytes; booleans already cost 1 byte with no + // payload, so caching them can only ever waste space -- skip. + $cacheKey = $type === MMDBType::BOOLEAN ? null : ($type . ':' . serialize($raw)); + + return [$type, $raw, $cacheKey]; + } + + private function isList(array $a): bool + { + $i = 0; + foreach ($a as $k => $_) { + if ($k !== $i++) { + return false; + } + } + return true; + } + + /** Encode an MMDB pointer (type 1) record per the spec's 4 size classes. */ + private function pointerBytes(int $offset): string + { + if ($offset <= 0x7FF) { // 11-bit value, 1 extra byte + $b0 = (MMDBType::POINTER << 5) | (0 << 3) | (($offset >> 8) & 0x07); + return chr($b0) . chr($offset & 0xFF); + } + + $v = $offset - 2048; + if ($v <= 0x7FFFF) { // 19-bit value, 2 extra bytes + $b0 = (MMDBType::POINTER << 5) | (1 << 3) | (($v >> 16) & 0x07); + return chr($b0) . chr(($v >> 8) & 0xFF) . chr($v & 0xFF); + } + + $v2 = $offset - 2048 - 524288; + if ($v2 <= 0x7FFFFFF) { // 27-bit value, 3 extra bytes + $b0 = (MMDBType::POINTER << 5) | (2 << 3) | (($v2 >> 24) & 0x07); + return chr($b0) . chr(($v2 >> 16) & 0xFF) . chr(($v2 >> 8) & 0xFF) . chr($v2 & 0xFF); + } + + // Size class 3: full absolute 32-bit offset, 4 extra bytes. + $b0 = (MMDBType::POINTER << 5) | (3 << 3); + return chr($b0) . pack('N', $offset); + } +} + // --------------------------------------------------------------------- // Binary trie used to build the MMDB search tree // --------------------------------------------------------------------- @@ -266,6 +427,28 @@ final class MMDBTree } } + /** + * Look up the data offset of whatever already-inserted network covers + * this bit path (the "parent" network), without mutating the tree. + * Returns null if no covering network has been inserted yet. + */ + public function lookupCoveringData(array $bits, int $prefixLen): ?int + { + $current = 0; + for ($i = 0; $i < $prefixLen; $i++) { + $bit = $bits[$i]; + $rec = $this->nodes[$current][$bit]; + if ($rec === null) { + return null; + } + if (is_array($rec)) { + return $rec['data']; + } + $current = $rec; + } + return null; + } + /** * Serialize the tree to bytes using the smallest record size (24, 28, * or 32 bits) that fits every record value. @@ -337,8 +520,9 @@ final class MMDBWriter private $ipVersion; private $totalDepth; private $tree; - private $dataSection = ''; - private $dataCache = []; + private $data; + private $rawDataByOffset = []; + private $pending = []; private $databaseType; private $languages; private $description; @@ -361,26 +545,101 @@ final class MMDBWriter $this->ipVersion = $ipVersion; $this->totalDepth = $ipVersion === 4 ? 32 : 128; $this->tree = new MMDBTree(); + $this->data = new MMDBDataSection(); $this->databaseType = $databaseType; $this->languages = $languages; $this->description = $description; } /** - * @param string $cidr e.g. "1.2.3.0/24", "2001:db8::/32", or a bare IP - * @param mixed $data scalar/array/MMDBValue data to associate with the network + * Queue a network to be written. Records can be added in ANY order -- + * broader and narrower networks alike -- regardless of which one you + * call this with first. At write() time, records are resolved from + * broadest to narrowest so inheritance (see $inherit) always works + * correctly no matter the call order. + * + * @param string $cidr e.g. "1.2.3.0/24", "2001:db8::/32", or a bare IP + * @param mixed $data scalar/array/MMDBValue data to associate with the network + * @param bool $inherit if true (default), and this network falls inside + * a broader network also added to this writer, + * fields from that broader record are merged in + * first (this record's own fields still win on + * conflicts). */ - public function addRecord(string $cidr, $data): void + public function addRecord(string $cidr, $data, bool $inherit = true): void { [$ip, $prefixLen] = $this->parseCidr($cidr); [$bits, $treePrefixLen] = $this->ipToTreeBits($ip, $prefixLen); - $offset = $this->internData($data); - $this->tree->insert($bits, $treePrefixLen, $offset); + $this->pending[] = [$bits, $treePrefixLen, $data, $inherit, count($this->pending)]; + } + + /** + * Resolve all queued records (broadest network first, so inheritance + * is correct regardless of the order addRecord() was called in) and + * insert them into the tree. + */ + private function resolveRecords(): void + { + $records = $this->pending; + usort($records, function ($a, $b) { + return $a[1] <=> $b[1] ?: $a[4] <=> $b[4]; // prefix length asc, then original call order + }); + + foreach ($records as [$bits, $treePrefixLen, $data, $inherit]) { + if ($inherit) { + $parentOffset = $this->tree->lookupCoveringData($bits, $treePrefixLen); + if ($parentOffset !== null && isset($this->rawDataByOffset[$parentOffset])) { + $data = $this->mergeWithParent($this->rawDataByOffset[$parentOffset], $data); + } + } + + $offset = $this->data->internTop($data); + $this->rawDataByOffset[$offset] = $data; + $this->tree->insert($bits, $treePrefixLen, $offset); + } + } + + /** + * Shallow-merge a child record over a parent record: parent fields are + * inherited, and any key the child also sets overrides the parent's + * value for that key. Non-map values (or MMDBValue-wrapped non-maps) + * are left untouched -- the child value simply replaces the parent. + */ + private function mergeWithParent($parentData, $data) + { + $parentArr = $parentData instanceof MMDBValue && $parentData->type === MMDBType::MAP + ? $parentData->value + : $parentData; + $childArr = $data instanceof MMDBValue && $data->type === MMDBType::MAP + ? $data->value + : $data; + + if (!$this->isMap($parentArr) || !$this->isMap($childArr)) { + return $data; // nothing sensible to merge; child value wins outright + } + + return array_replace($parentArr, $childArr); + } + + private function isMap($v): bool + { + if (!is_array($v) || $v === []) { + return false; + } + $i = 0; + foreach ($v as $k => $_) { + if ($k !== $i++) { + return true; // has at least one non-sequential/string key -> it's a map + } + } + return false; // it's a plain list } public function write(string $path): void { - [$treeBytes, $recordSize] = $this->tree->serialize(strlen($this->dataSection)); + $this->resolveRecords(); + + [$treeBytes, $recordSize] = $this->tree->serialize(strlen($this->data->bytes())); $metadata = [ 'node_count' => MMDBValue::uint32($this->tree->nodeCount()), @@ -399,25 +658,13 @@ final class MMDBWriter $separator = str_repeat("\x00", 16); $marker = "\xAB\xCD\xEFMaxMind.com"; - $blob = $treeBytes . $separator . $this->dataSection . $marker . $metadataBytes; + $blob = $treeBytes . $separator . $this->data->bytes() . $marker . $metadataBytes; if (file_put_contents($path, $blob) === false) { throw new RuntimeException("Failed to write $path"); } } - private function internData($data): int - { - $key = serialize($data); - if (isset($this->dataCache[$key])) { - return $this->dataCache[$key]; - } - $offset = strlen($this->dataSection); - $this->dataSection .= MMDBEncoder::encode($data); - $this->dataCache[$key] = $offset; - return $offset; - } - private function parseCidr(string $cidr): array { if (strpos($cidr, '/') !== false) { diff --git a/lib/MMDB_ASN_Extractor.php b/lib/MMDB_ASN_Extractor.php new file mode 100644 index 0000000..630d27d --- /dev/null +++ b/lib/MMDB_ASN_Extractor.php @@ -0,0 +1,395 @@ + ["1.0.0.0/32", "1.0.0.1/32", ...], + * "as13335" => ["1.1.1.0/24", ...], + * ] + * + * No Composer dependency required — implements the MaxMind DB binary + * format (search tree + data section) directly per the public spec: + * https://maxmind.github.io/MaxMind-DB/ + * + * Usage: + * $extractor = new GeoLite2ASNExtractor('/path/to/GeoLite2-ASN.mmdb'); + * $asnMap = $extractor->extract(); + * + * Notes: + * - GeoLite2-ASN.mmdb is normally built as an IPv6 tree with the IPv4 + * address space embedded under ::/96. This class detects that and + * emits plain IPv4 CIDRs (e.g. "1.0.0.0/32"), skipping true + * IPv6-only ranges. Set $includeIPv6 = true in extract() if you + * want IPv6 ranges included too (returned as e.g. "2606:4700::/32"). + * - The whole file is loaded into memory once (file_get_contents). + * GeoLite2-ASN.mmdb is tens of MB, which is fine for most setups, + * but be aware if running under a memory-constrained environment. + * - Walking the full tree is O(number of tree nodes), which can be a + * few million recursive calls for the ASN database. This can take + * anywhere from several seconds to a couple of minutes depending on + * hardware. Consider raising max_execution_time for CLI/cron use. + */ +class GeoLite2ASNExtractor +{ + private const METADATA_MARKER = "\xab\xcd\xefMaxMind.com"; + + private string $data; + private array $metadata = []; + + private int $nodeCount = 0; + private int $recordSize = 0; + private int $nodeByteSize = 0; + private int $searchTreeSize = 0; + private int $dataSectionStart = 0; + private int $ipVersion = 6; + + public function __construct(string $mmdbPath) + { + if (!is_readable($mmdbPath)) { + throw new \RuntimeException("Cannot read file: {$mmdbPath}"); + } + + $contents = file_get_contents($mmdbPath); + if ($contents === false) { + throw new \RuntimeException("Failed to read file: {$mmdbPath}"); + } + + $this->data = $contents; + $this->parseMetadata(); + } + + /** + * Optional convenience helper: download an .mmdb file over HTTP(S) + * and construct an extractor from it. + */ + public static function fromUrl(string $url, ?string $saveTo = null): self + { + $tmpPath = $saveTo ?? tempnam(sys_get_temp_dir(), 'mmdb_'); + + $ch = curl_init($url); + $fh = fopen($tmpPath, 'wb'); + if ($fh === false) { + throw new \RuntimeException("Cannot open temp file for writing: {$tmpPath}"); + } + + curl_setopt_array($ch, [ + CURLOPT_FILE => $fh, + CURLOPT_FOLLOWLOCATION => true, + CURLOPT_MAXREDIRS => 5, + CURLOPT_TIMEOUT => 300, + CURLOPT_FAILONERROR => true, + CURLOPT_USERAGENT => 'GeoLite2ASNExtractor/1.0', + ]); + + $ok = curl_exec($ch); + $err = curl_error($ch); + curl_close($ch); + fclose($fh); + + if ($ok === false) { + if ($saveTo === null) { + @unlink($tmpPath); + } + throw new \RuntimeException("Download failed: {$err}"); + } + + return new self($tmpPath); + } + + /** + * Walk the entire tree and return ["as" => ["a.b.c.d/x", ...], ...] + */ + public function extract(bool $includeIPv6 = false): array + { + $result = []; + $this->walk(0, '', $result, $includeIPv6); + return $result; + } + + public function getMetadata(): array + { + return $this->metadata; + } + + // ------------------------------------------------------------------ + // Search tree traversal + // ------------------------------------------------------------------ + + /** + * @param string $bits '0'/'1' characters representing the path taken so far + */ + private function walk(int $recordValue, string $bits, array &$result, bool $includeIPv6): void + { + if ($recordValue === $this->nodeCount) { + // Empty branch, no data assigned here. + return; + } + + if ($recordValue > $this->nodeCount) { + // Leaf: record points into the data section. + // + // Per the MaxMind DB spec, the offset within the data section is + // (record_value - node_count - 16), and the data section itself + // starts at (search_tree_size + 16) — the two 16s cancel out, so + // the correct absolute file offset is search_tree_size plus + // (record_value - node_count). Using data_section_start here + // instead would shift every leaf read by 16 bytes. + $absoluteOffset = $this->searchTreeSize + ($recordValue - $this->nodeCount); + [$record, ] = $this->decodeData($absoluteOffset); + $this->addResult($bits, $record, $result, $includeIPv6); + return; + } + + // Otherwise it's an index of another node — recurse both children. + [$left, $right] = $this->readNode($recordValue); + $this->walk($left, $bits . '0', $result, $includeIPv6); + $this->walk($right, $bits . '1', $result, $includeIPv6); + } + + private function readNode(int $nodeNumber): array + { + $offset = $nodeNumber * $this->nodeByteSize; + $bytes = substr($this->data, $offset, $this->nodeByteSize); + + if ($this->recordSize === 24) { + $left = (ord($bytes[0]) << 16) | (ord($bytes[1]) << 8) | ord($bytes[2]); + $right = (ord($bytes[3]) << 16) | (ord($bytes[4]) << 8) | ord($bytes[5]); + } elseif ($this->recordSize === 28) { + $middle = ord($bytes[3]); + $left = (ord($bytes[0]) << 16) | (ord($bytes[1]) << 8) | ord($bytes[2]); + $left |= ($middle >> 4) << 24; + $right = (($middle & 0x0f) << 24) | (ord($bytes[4]) << 16) | (ord($bytes[5]) << 8) | ord($bytes[6]); + } elseif ($this->recordSize === 32) { + $left = (ord($bytes[0]) << 24) | (ord($bytes[1]) << 16) | (ord($bytes[2]) << 8) | ord($bytes[3]); + $right = (ord($bytes[4]) << 24) | (ord($bytes[5]) << 16) | (ord($bytes[6]) << 8) | ord($bytes[7]); + } else { + throw new \RuntimeException("Unsupported record_size: {$this->recordSize}"); + } + + return [$left, $right]; + } + + private function addResult(string $bits, $record, array &$result, bool $includeIPv6): void + { + if (!is_array($record) || !isset($record['autonomous_system_number'])) { + return; // no ASN data at this leaf + } + + $asn = 'as' . $record['autonomous_system_number']; + $prefixLen = strlen($bits); + + if ($this->ipVersion === 6) { + if ($prefixLen >= 96 && substr($bits, 0, 96) === str_repeat('0', 96)) { + // Embedded IPv4 space (::/96). + $ipv4Bits = substr($bits, 96); + $cidr = $this->bitsToIPv4($ipv4Bits) . '/' . strlen($ipv4Bits); + $result[$asn][] = $cidr; + } elseif ($includeIPv6) { + $cidr = $this->bitsToIPv6($bits) . '/' . $prefixLen; + $result[$asn][] = $cidr; + } + // else: real IPv6 range but caller doesn't want IPv6 — skip. + } else { + // Pure IPv4 tree (older / rare db builds). + $cidr = $this->bitsToIPv4($bits) . '/' . $prefixLen; + $result[$asn][] = $cidr; + } + } + + private function bitsToIPv4(string $bits): string + { + $bits = str_pad($bits, 32, '0'); // pad host bits with 0 -> network address + $octets = []; + for ($i = 0; $i < 4; $i++) { + $octets[] = bindec(substr($bits, $i * 8, 8)); + } + return implode('.', $octets); + } + + private function bitsToIPv6(string $bits): string + { + $bits = str_pad($bits, 128, '0'); + $groups = []; + for ($i = 0; $i < 8; $i++) { + $groups[] = dechex(bindec(substr($bits, $i * 16, 16))); + } + $expanded = implode(':', $groups); + return inet_ntop(inet_pton($expanded)) ?: $expanded; + } + + // ------------------------------------------------------------------ + // Metadata / data section decoding (MaxMind DB data format) + // ------------------------------------------------------------------ + + private function parseMetadata(): void + { + $pos = strrpos($this->data, self::METADATA_MARKER); + if ($pos === false) { + throw new \RuntimeException('MaxMind DB metadata marker not found — not a valid .mmdb file'); + } + + $offset = $pos + strlen(self::METADATA_MARKER); + [$metadata, ] = $this->decodeData($offset); + + if (!is_array($metadata) || !isset($metadata['node_count'], $metadata['record_size'], $metadata['ip_version'])) { + throw new \RuntimeException('Malformed MaxMind DB metadata'); + } + + $this->metadata = $metadata; + $this->nodeCount = (int) $metadata['node_count']; + $this->recordSize = (int) $metadata['record_size']; + $this->ipVersion = (int) $metadata['ip_version']; + $this->nodeByteSize = (int) (($this->recordSize * 2) / 8); + $this->searchTreeSize = $this->nodeCount * $this->nodeByteSize; + $this->dataSectionStart = $this->searchTreeSize + 16; // 16-byte all-zero separator + } + + /** + * Decodes one MaxMind DB data item starting at $offset. + * Returns [decodedValue, offsetAfterThisItem]. + */ + private function decodeData(int $offset): array + { + $ctrl = ord($this->data[$offset]); + $offset++; + + $type = $ctrl >> 5; // top 3 bits + $size = $ctrl & 0x1f; // bottom 5 bits + + if ($type === 0) { + // Extended type: real type = next byte + 7 + $type = 7 + ord($this->data[$offset]); + $offset++; + } + + if ($type === 1) { + return $this->decodePointer($ctrl, $offset); + } + + if ($type !== 14) { // not boolean — booleans store their value in $size directly + if ($size === 29) { + $size = 29 + ord($this->data[$offset]); + $offset += 1; + } elseif ($size === 30) { + $size = 285 + $this->readUint($offset, 2); + $offset += 2; + } elseif ($size === 31) { + $size = 65821 + $this->readUint($offset, 3); + $offset += 3; + } + } + + switch ($type) { + case 2: // utf8_string + case 4: // bytes + $value = substr($this->data, $offset, $size); + $offset += $size; + return [$value, $offset]; + + case 3: // double + $bytes = substr($this->data, $offset, 8); + $offset += 8; + $u = unpack('E', $bytes); + return [$u[1], $offset]; + + case 5: // uint16 + case 6: // uint32 + case 9: // uint64 (may lose precision above 2^63 — not expected for ASN fields) + $value = $this->readUint($offset, $size); + $offset += $size; + return [$value, $offset]; + + case 7: // map + $map = []; + for ($i = 0; $i < $size; $i++) { + [$key, $offset] = $this->decodeData($offset); + [$val, $offset] = $this->decodeData($offset); + $map[$key] = $val; + } + return [$map, $offset]; + + case 8: // int32 + $value = $this->readUint($offset, $size); + if ($size > 0 && ($value & (1 << (8 * $size - 1)))) { + $value -= (1 << (8 * $size)); + } + $offset += $size; + return [$value, $offset]; + + case 10: // uint128 — returned as a hex string (not used by ASN records) + $hex = bin2hex(substr($this->data, $offset, $size)); + $offset += $size; + return [$hex, $offset]; + + case 11: // array + $arr = []; + for ($i = 0; $i < $size; $i++) { + [$val, $offset] = $this->decodeData($offset); + $arr[] = $val; + } + return [$arr, $offset]; + + case 13: // end marker + return [null, $offset]; + + case 14: // boolean — value is $size itself (0 or 1) + return [$size === 1, $offset]; + + case 15: // float + $bytes = substr($this->data, $offset, 4); + $offset += 4; + $u = unpack('G', $bytes); + return [$u[1], $offset]; + + default: + throw new \RuntimeException("Unsupported MaxMind DB data type: {$type} at offset {$offset}"); + } + } + + private function decodePointer(int $ctrl, int $offset): array + { + $ptrSize = ($ctrl >> 3) & 0x03; + $valueHigh = $ctrl & 0x07; + + switch ($ptrSize) { + case 0: + $pointer = ($valueHigh << 8) | ord($this->data[$offset]); + $offset += 1; + $base = 0; + break; + case 1: + $pointer = ($valueHigh << 16) | $this->readUint($offset, 2); + $offset += 2; + $base = 2048; + break; + case 2: + $pointer = ($valueHigh << 24) | $this->readUint($offset, 3); + $offset += 3; + $base = 526336; + break; + default: // 3 + $pointer = $this->readUint($offset, 4); + $offset += 4; + $base = 0; + break; + } + + $target = $this->dataSectionStart + $pointer + $base; + [$value, ] = $this->decodeData($target); + + return [$value, $offset]; + } + + private function readUint(int $offset, int $numBytes): int + { + $value = 0; + for ($i = 0; $i < $numBytes; $i++) { + $value = ($value << 8) | ord($this->data[$offset + $i]); + } + return $value; + } +} diff --git a/lib/fuckhtml.php b/lib/fuckhtml.php new file mode 100644 index 0000000..143d113 --- /dev/null +++ b/lib/fuckhtml.php @@ -0,0 +1,628 @@ +load($html, $isfile); + } + } + + public function load($html, $isfile = false){ + + if(is_array($html)){ + + if(!array_key_exists("innerHTML", $html)){ + + throw new Exception("(load) Supplied array doesn't contain an innerHTML index"); + } + $html = $html["innerHTML"]; + } + + if($isfile){ + + $handle = fopen($html, "r"); + $fetch = fread($handle, filesize($html)); + fclose($handle); + + $this->html = $fetch; + }else{ + + $this->html = $html; + } + + $this->strlen = strlen($this->html); + } + + public function getloadedhtml(){ + + return $this->html; + } + + public function getElementsByTagName(string $tagname){ + + $out = []; + + /* + Scrape start of the tag. Example +
... + */ + + if($tagname == "*"){ + + $tagname = '[A-Za-z0-9._-]+'; + }else{ + + $tagname = preg_quote(strtolower($tagname)); + } + + preg_match_all( + '/<\s*(' . $tagname . ')(\s(?:[^>\'"]*|"[^"]*"|\'[^\']*\')+)?\s*>/i', + /* '/<\s*(' . $tagname . ')(\s[\S\s]*?)?>/i', */ + $this->html, + $starting_tags, + PREG_OFFSET_CAPTURE + ); + + for($i=0; $i strtolower($starting_tags[1][$i][0]), + "startPos" => $starting_tags[0][$i][1], + "endPos" => 0, + "startTag" => $starting_tags[0][$i][0], + "attributes" => $attributes, + "innerHTML" => null + ]; + } + + /* + Get innerHTML + */ + // get closing tag positions + preg_match_all( + '/<\s*\/\s*(' . $tagname . ')\s*>/i', + $this->html, + $regex_closing_tags, + PREG_OFFSET_CAPTURE + ); + + // merge opening and closing tags together + for($i=0; $i strtolower($regex_closing_tags[1][$i][0]), + "endTag" => $regex_closing_tags[0][$i][0], + "startPos" => $regex_closing_tags[0][$i][1] + ]; + } + + usort( + $out, + function($a, $b){ + + return $a["startPos"] > $b["startPos"]; + } + ); + + // compute the indent level for each element + $level = []; + $count = count($out); + + for($i=0; $i<$count; $i++){ + + if(!isset($level[$out[$i]["tagName"]])){ + + $level[$out[$i]["tagName"]] = 0; + } + + if(isset($out[$i]["startTag"])){ + + // encountered starting tag + $level[$out[$i]["tagName"]]++; + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + }else{ + + // encountered closing tag + $out[$i]["level"] = $level[$out[$i]["tagName"]]; + $level[$out[$i]["tagName"]]--; + } + } + + // if the indent level is the same for a div, + // we encountered _THE_ closing tag + for($i=0; $i<$count; $i++){ + + if(!isset($out[$i]["startTag"])){ + + continue; + } + + for($k=$i; $k<$count; $k++){ + + if( + isset($out[$k]["endTag"]) && + $out[$i]["tagName"] == $out[$k]["tagName"] && + $out[$i]["level"] + === $out[$k]["level"] + ){ + + $startlen = strlen($out[$i]["startTag"]); + $endlen = strlen($out[$k]["endTag"]); + + $out[$i]["endPos"] = $out[$k]["startPos"] + $endlen; + + $out[$i]["innerHTML"] = + substr( + $this->html, + $out[$i]["startPos"] + $startlen, + $out[$k]["startPos"] - ($out[$i]["startPos"] + $startlen) + ); + + $out[$i]["outerHTML"] = + substr( + $this->html, + $out[$i]["startPos"], + $out[$k]["startPos"] - $out[$i]["startPos"] + $endlen + ); + + break; + } + } + } + + // filter out ending divs + for($i=0; $i<$count; $i++){ + + if(isset($out[$i]["endTag"])){ + + unset($out[$i]); + } + + unset($out[$i]["startTag"]); + } + + return array_values($out); + } + + public function getElementsByAttributeName(string $name, $collection = null){ + + if($collection === null){ + + $collection = $this->getElementsByTagName("*"); + }elseif(is_string($collection)){ + + $collection = $this->getElementsByTagName($collection); + } + + $return = []; + foreach($collection as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_name == $name){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + + $value = + explode( + " ", + trim( + preg_replace( + '/\s+/', + " ", + $value + ) + ) + ); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + $attrib_value = + explode( + " ", + trim( + preg_replace( + '/\s+/', + " ", + $attrib_value + ) + ) + ); + + $ac = count($attrib_value); + $nc = count($value); + $cr = 0; + + for($i=0; $i<$nc; $i++){ + + for($k=0; $k<$ac; $k++){ + + if($value[$i] == $attrib_value[$k]){ + + $cr++; + } + } + } + + if($cr === $nc){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementsByAttributeValue(string $name, string $value, $collection = null){ + + $elems = $this->getElementsByAttributeName($name, $collection); + + $return = []; + + foreach($elems as $elem){ + + foreach($elem["attributes"] as $attrib_name => $attrib_value){ + + if($attrib_value == $value){ + + $return[] = $elem; + continue 2; + } + } + } + + return $return; + } + + public function getElementById(string $idname, $collection = null){ + + $id = $this->getElementsByAttributeValue("id", $idname, $collection); + + if(count($id) !== 0){ + + return $id[0]; + } + + return false; + } + + public function getElementsByClassName(string $classname, $collection = null){ + + return $this->getElementsByFuzzyAttributeValue("class", $classname, $collection); + } + + public function getTextContent($html, $whitespace = false, $trim = true){ + + if(is_array($html)){ + + if(!array_key_exists("innerHTML", $html)){ + + throw new Exception("(getTextContent) Supplied array doesn't contain an innerHTML index"); + } + + $html = $html["innerHTML"]; + } + + $html = preg_split('/\n|<\/?br>/i', $html); + + $out = ""; + for($i=0; $i= 0 && $json[$j] === "\\"){ + $bsCount++; + $j--; + } + + // quote is NOT escaped if even number of backslashes + if($bsCount % 2 === 0){ + if($in_quote === null){ + // open quote + $in_quote = $json[$i]; + }elseif($in_quote === $json[$i]){ + // close quote + $in_quote = null; + } + } + break; + + case "[": + if($in_quote === null){ + + $array_level++; + if($start === null){ + + $start = $i; + } + } + break; + + case "]": + if($in_quote === null){ + + $array_level--; + } + break; + + case "{": + if($in_quote === null){ + + $object_level++; + if($start === null){ + + $start = $i; + } + } + break; + + case "}": + if($in_quote === null){ + + $object_level--; + } + break; + } + + if( + $array_level === 0 && + $object_level === 0 && + $start !== null + ){ + + return substr($json, $start, $i - $start + 1); + break; + } + } + + // fallback + return "[]"; + } +} diff --git a/mmdb.php b/mmdb.php index ef3cd5d..0bb2d81 100644 --- a/mmdb.php +++ b/mmdb.php @@ -6,9 +6,15 @@ class mmdb_creator{ public function __construct(){ + $this->ip_list = []; + + include "lib/fuckhtml.php"; + $this->fuckhtml = new fuckhtml(); + // - // Import database + // Download data // + /* if(!file_exists("data")){ mkdir("data"); } chdir("data"); if(!file_exists("blocklist-ipsets")){ @@ -23,6 +29,9 @@ class mmdb_creator{ chdir(".."); } + echo "Downloading MaxMind's GeoLite2-ASN database...\n"; + $this->dl("https://git.io/GeoLite2-ASN.mmdb", "GeoLite2-ASN.mmdb"); + echo "Downloading Tunnelbear list\n"; $this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt"); @@ -37,24 +46,95 @@ class mmdb_creator{ $this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,"; $this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n"; - echo "Downloading hosting lists... ("; - if(!file_exists("x4bnet-hosting")){ mkdir("x4bnet-hosting"); } - $this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt", "x4bnet-hosting/ipv4.txt"); echo "1,"; - $this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt", "x4bnet-hosting/ipv6.txt"); echo "2)\n"; - - echo "Downloading CDN list...\n"; - $this->dl("https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst", "cdn.lst"); - echo "Downloading Tor exit node list...\n"; $this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt"); - chdir(".."); - echo "Done! Creating MMDB database...\n"; + chdir("..");*/ + + // + // Construct ASN blocklist + // + echo "Extracting ASNs from GeoLite database...\n"; + + require "lib/MMDB_ASN_Extractor.php"; + + $asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb"); + $mmdb_asns = $asn_extractor->extract(); + + + // + // Scrape ASN classifications from BGP.tools + // + $good = + array_unique( + array_merge( + $this->parse_bgptools("dsl"), + $this->parse_bgptools("mobile") + ) + ); + + $bad = + array_unique( + array_merge( + $this->parse_bgptools("cdn"), + $this->parse_bgptools("vpsh"), + $this->parse_bgptools("vpn") + ) + ); + + echo "Processing hosting ASNs...\n"; + foreach($bad as $bad_asn){ + + // if a bad ASN is found to be a DSL/mobile provider, ignore + if(in_array($bad_asn, $good)){ + + continue; + } + + if(!isset($mmdb_asns[$bad_asn])){ + + // no IP range available for that ASN + continue; + } + + echo "Hosting: $bad_asn (" . count($mmdb_asns[$bad_asn]) . " ranges)\n"; + + foreach($mmdb_asns[$bad_asn] as $ip_range){ + + $this->ip_list[$ip_range]["hosting"] = true; + } + } + + echo "Processing residential ASNs...\n"; + foreach($good as $good_asn){ + + // if a residential ASN is found to be a CDN/VPSH/VPN provider, ignore + if(in_array($good_asn, $bad)){ + + continue; + } + + if(!isset($mmdb_asns[$good_asn])){ + + // no IP range available for that ASN + continue; + } + + echo "Residential: $good_asn (" . count($mmdb_asns[$good_asn]) . " ranges)\n"; + + foreach($mmdb_asns[$good_asn] as $ip_range){ + + $this->ip_list[$ip_range]["residential"] = true; + } + } + + echo "Clearing memory...\n"; + unset($good); + unset($bad); // // Construct database // - $this->ip_list = []; // import miscelaneous lists $this->import_firehol_lists( @@ -66,9 +146,6 @@ class mmdb_creator{ ], "tor" => [ "tor-exit-list.txt" - ], - "hosting" => [ - "cdn.lst" // mansourjabin's ] ], "" @@ -85,16 +162,6 @@ class mmdb_creator{ "x4bnet-vpn/" ); - $this->import_firehol_lists( - [ - "hosting" => [ - "ipv4.txt", - "ipv6.txt", - ] - ], - "x4bnet-hosting/" - ); - // import firehol lists $this->import_firehol_lists( [ @@ -121,13 +188,13 @@ class mmdb_creator{ "blocklist-ipsets/" ); - require __DIR__ . '/MMDBWriter.php'; + require "lib/MMDBWriter.php"; $w = new MMDBWriter( 6, // accepts IPv4 and IPv6 networks - 'GeoIP-Custom', // database_type - ['en'], // languages - ['en' => 'Cloudfish'] + "GeoIP-Custom", // database_type + ["en"], // languages + ["en" => "Cloudfish"] ); echo "Generating mmdb file\n"; @@ -139,12 +206,14 @@ class mmdb_creator{ //"city" => "Ashburn", ...(isset($data["proxy"]) ? ["is_proxy" => true] : []), ...(isset($data["tor"]) ? ["is_tor" => true] : []), - ...(isset($data["hosting"]) ? ["is_hosting" => true] : []) + ...(isset($data["hosting"]) ? ["is_hosting" => true] : []), + ...(isset($data["residential"]) ? ["is_residential" => true] : []) //"asn" => MMDBValue::uint32(15169), ]); } - $w->write(__DIR__ . '/output.mmdb'); + echo "Saving database...\n"; + $w->write(__DIR__ . "/output.mmdb"); echo "done\n"; } @@ -182,4 +251,89 @@ class mmdb_creator{ $data = file_get_contents($url); file_put_contents($path, $data); } + + public function curl($url){ + + $curlproc = curl_init(); + + curl_setopt($curlproc, CURLOPT_URL, $url); + + curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding + curl_setopt($curlproc, CURLOPT_HTTPHEADER, [ + "User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0", + "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8", + "Accept-Language: en-US,en;q=0.9", + "Accept-Encoding: gzip, deflate, br", + "DNT: 1", + "Connection: keep-alive", + "Upgrade-Insecure-Requests: 1", + "Sec-Fetch-Dest: document", + "Sec-Fetch-Mode: navigate", + "Sec-Fetch-Site: none", + "Sec-Fetch-User: ?1" + ]); + + curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2); + curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true); + curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30); + curl_setopt($curlproc, CURLOPT_TIMEOUT, 30); + + $data = curl_exec($curlproc); + + if(curl_errno($curlproc)){ + + throw new Exception(curl_error($curlproc)); + } + + curl_close($curlproc); + return $data; + } + + public function parse_bgptools($tag){ + + $page = $this->curl("https://bgp.tools/tags/{$tag}"); + + $asns = []; + + $this->fuckhtml->load($page); + + $table = + $this->fuckhtml + ->getElementById("upstreamTable", "table"); + + if($table === false){ + + throw new Exception("Failed to grep table element on bgptool's {$tag} page"); + } + + $this->fuckhtml->load($table); + + $trs = + $this->fuckhtml + ->getElementsByTagName("tr"); + + foreach($trs as $tr){ + + $this->fuckhtml->load($tr); + + $tds = + $this->fuckhtml + ->getElementsByTagName("td"); + + if(!isset($tds[1])){ continue; } + + $asns[] = + strtolower( + $this->fuckhtml + ->getTextContent( + $tds[1] + ) + ); + } + + echo "Scraped bgp.tools/tags/{$tag} (got " . number_format(count($asns)) . " ASNs)\n"; + + return $asns; + } }