Detect residential+datacenter IPs
This commit is contained in:
29
README.md
29
README.md
@@ -17,6 +17,13 @@ print_r($cf_lookup); // ["is_hosting" => true]
|
||||
|
||||
**Warning**: Any field may be missing. Fields that are set are always set to `true`.
|
||||
|
||||
# Generate database
|
||||
Just run that shit and hope it works
|
||||
|
||||
```sh
|
||||
php mmdb.php
|
||||
```
|
||||
|
||||
# Data sources
|
||||
|
||||
## Proxy detection ([firehol](https://iplists.firehol.org/))
|
||||
@@ -45,8 +52,21 @@ print_r($cf_lookup); // ["is_hosting" => true]
|
||||
- pia/proton/apple/mullvad [ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv4.txt), [ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/))
|
||||
|
||||
## Hosting detection
|
||||
- [datacenter-ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt), [datacenter-ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/))
|
||||
- [CDN list](https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst) ([source](https://github.com/mansourjabin/cdn-ip-database))
|
||||
- [GeoLite2-ASN](https://git.io/GeoLite2-ASN.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb))
|
||||
- ASN purpose index ([source](https://bgp.tools)):
|
||||
- Residential signals
|
||||
- [Home ISP](https://bgp.tools/tags/dsl)
|
||||
- [Mobile Data/Carrier](https://bgp.tools/tags/mobile)
|
||||
- Hosting signals
|
||||
- [Content Delivery Network](https://bgp.tools/tags/cdn)
|
||||
- [Server Hosting](https://bgp.tools/tags/vpsh)
|
||||
- [VPN Host](https://bgp.tools/tags/vpn)
|
||||
|
||||
### Important!
|
||||
- An ASN's IP range(s) are assigned `is_hosting` if they report *ANY* hosting signal, but it *MUST NOT* have a residential signal.
|
||||
- An ASN's IP range(s) are assigned `is_residential` if they report a residential signal, but it *MUST NOT* have a hosting signal.
|
||||
|
||||
This method cover everything, some IPs will not receive a category. But it shouldn't falseflag.
|
||||
|
||||
## Tor detection
|
||||
- [tor exit node list](https://openinternet.io/tor/tor-exit-list.txt) ([source](https://openinternet.io))
|
||||
@@ -55,9 +75,6 @@ print_r($cf_lookup); // ["is_hosting" => true]
|
||||
# Recommendation
|
||||
I recommend these additional MMDB databases to complement Cloudfish.
|
||||
|
||||
## ASN detection
|
||||
- [GeoLite2-ASN](https://git.io/GeoLite2-ASN.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb))
|
||||
|
||||
## Country/city detection
|
||||
- [GeoLite2-City](https://git.io/GeoLite2-City.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb))
|
||||
|
||||
@@ -65,7 +82,7 @@ I recommend these additional MMDB databases to complement Cloudfish.
|
||||
I wrote a simple IP lookup script. It's available at `ip.lolcat.ca`, `ip4.lolcat.ca` and `ip6.lolcat.ca`.
|
||||
|
||||
### Disclaimer
|
||||
I transpiled MMDBWriter.php from it's original Go implementation using Claude. I'll make my own when I have more time, no idea if the filesize could be reduced.
|
||||
I transpiled MMDBWriter.php from it's original Go implementation using Claude. I'll make my own when I have more time, no idea if the filesize could be reduced. I also asked it to create a MMDB ASN extractor. Lol.
|
||||
|
||||
# License
|
||||
AGPLv3, make sure to credit all blocklists used, they all have their own fuckass licenses.
|
||||
|
||||
@@ -114,6 +114,31 @@ final class MMDBEncoder
|
||||
}
|
||||
|
||||
private static function encodeTyped(int $type, $value): string
|
||||
{
|
||||
if ($type !== MMDBType::MAP && $type !== MMDBType::ARRAY_T) {
|
||||
return self::encodeScalar($type, $value);
|
||||
}
|
||||
switch ($type) {
|
||||
case MMDBType::MAP:
|
||||
$out = self::controlAndSize($type, count($value));
|
||||
foreach ($value as $k => $v) {
|
||||
$out .= self::encodeTyped(MMDBType::STRING, (string) $k);
|
||||
$out .= self::encode($v);
|
||||
}
|
||||
return $out;
|
||||
|
||||
case MMDBType::ARRAY_T:
|
||||
$out = self::controlAndSize($type, count($value));
|
||||
foreach ($value as $v) {
|
||||
$out .= self::encode($v);
|
||||
}
|
||||
return $out;
|
||||
}
|
||||
}
|
||||
|
||||
/** Encode a single non-composite (leaf) MMDB value to bytes. Public so
|
||||
* MMDBDataSection can reuse it for the leaves of its deduped structures. */
|
||||
public static function encodeScalar(int $type, $value): string
|
||||
{
|
||||
switch ($type) {
|
||||
case MMDBType::STRING:
|
||||
@@ -156,21 +181,6 @@ final class MMDBEncoder
|
||||
// Boolean has no payload; the value lives in the size field.
|
||||
return self::controlAndSize($type, $value ? 1 : 0);
|
||||
|
||||
case MMDBType::MAP:
|
||||
$out = self::controlAndSize($type, count($value));
|
||||
foreach ($value as $k => $v) {
|
||||
$out .= self::encodeTyped(MMDBType::STRING, (string) $k);
|
||||
$out .= self::encode($v);
|
||||
}
|
||||
return $out;
|
||||
|
||||
case MMDBType::ARRAY_T:
|
||||
$out = self::controlAndSize($type, count($value));
|
||||
foreach ($value as $v) {
|
||||
$out .= self::encode($v);
|
||||
}
|
||||
return $out;
|
||||
|
||||
default:
|
||||
throw new InvalidArgumentException("Unsupported MMDB type: $type");
|
||||
}
|
||||
@@ -191,7 +201,7 @@ final class MMDBEncoder
|
||||
* for a given MMDB type and payload size, per the spec's control
|
||||
* byte format.
|
||||
*/
|
||||
private static function controlAndSize(int $type, int $size): string
|
||||
public static function controlAndSize(int $type, int $size): string
|
||||
{
|
||||
$typeBits = $type <= 7 ? $type : 0; // 0 = "look at next byte for real type"
|
||||
$out = '';
|
||||
@@ -217,6 +227,157 @@ final class MMDBEncoder
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Deduplicating data section builder. Any value (string, number, or
|
||||
// whole map/array structure) that has already been written once is
|
||||
// replaced everywhere else with a 2-5 byte MMDB "pointer" record instead
|
||||
// of being re-encoded, which is where most of the size savings on
|
||||
// real-world data (repeated keys like "country", repeated country codes,
|
||||
// repeated sub-structures) comes from.
|
||||
// ---------------------------------------------------------------------
|
||||
final class MMDBDataSection
|
||||
{
|
||||
private $buffer = '';
|
||||
/** @var array<string,int> cache key => byte offset of its real (non-pointer) encoding */
|
||||
private $cache = [];
|
||||
|
||||
public function bytes(): string
|
||||
{
|
||||
return $this->buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure $value is present in the data section (writing it if this is
|
||||
* the first time it's been seen) and return the offset of its real
|
||||
* encoding. Used for top-level records referenced directly by tree
|
||||
* leaves, so this always returns a real offset, never a pointer.
|
||||
*/
|
||||
public function internTop($value): int
|
||||
{
|
||||
[$type, $raw, $cacheKey] = $this->classify($value);
|
||||
if ($cacheKey !== null && isset($this->cache[$cacheKey])) {
|
||||
return $this->cache[$cacheKey];
|
||||
}
|
||||
$offset = strlen($this->buffer);
|
||||
if ($cacheKey !== null) {
|
||||
$this->cache[$cacheKey] = $offset;
|
||||
}
|
||||
$this->appendEncoding($type, $raw);
|
||||
return $offset;
|
||||
}
|
||||
|
||||
/** Append bytes for a value nested inside a map/array: a pointer if
|
||||
* it's a repeat, otherwise its full encoding (registering it for
|
||||
* future reuse). */
|
||||
private function writeRef($value): void
|
||||
{
|
||||
[$type, $raw, $cacheKey] = $this->classify($value);
|
||||
if ($cacheKey !== null && isset($this->cache[$cacheKey])) {
|
||||
$this->buffer .= $this->pointerBytes($this->cache[$cacheKey]);
|
||||
return;
|
||||
}
|
||||
$offset = strlen($this->buffer);
|
||||
if ($cacheKey !== null) {
|
||||
$this->cache[$cacheKey] = $offset;
|
||||
}
|
||||
$this->appendEncoding($type, $raw);
|
||||
}
|
||||
|
||||
private function appendEncoding(int $type, $raw): void
|
||||
{
|
||||
if ($type === MMDBType::MAP) {
|
||||
$this->buffer .= MMDBEncoder::controlAndSize(MMDBType::MAP, count($raw));
|
||||
foreach ($raw as $k => $v) {
|
||||
$this->writeRef((string) $k);
|
||||
$this->writeRef($v);
|
||||
}
|
||||
return;
|
||||
}
|
||||
if ($type === MMDBType::ARRAY_T) {
|
||||
$this->buffer .= MMDBEncoder::controlAndSize(MMDBType::ARRAY_T, count($raw));
|
||||
foreach ($raw as $v) {
|
||||
$this->writeRef($v);
|
||||
}
|
||||
return;
|
||||
}
|
||||
$this->buffer .= MMDBEncoder::encodeScalar($type, $raw);
|
||||
}
|
||||
|
||||
/** @return array{0:int,1:mixed,2:?string} [type, raw value, cache key or null if not worth caching] */
|
||||
private function classify($value): array
|
||||
{
|
||||
if ($value instanceof MMDBValue) {
|
||||
$type = $value->type;
|
||||
$raw = $value->value;
|
||||
} elseif (is_bool($value)) {
|
||||
$type = MMDBType::BOOLEAN;
|
||||
$raw = $value;
|
||||
} elseif (is_int($value)) {
|
||||
if ($value >= 0 && $value <= 0xFFFFFFFF) {
|
||||
$type = MMDBType::UINT32;
|
||||
} elseif ($value < 0 && $value >= -2147483648) {
|
||||
$type = MMDBType::INT32;
|
||||
} else {
|
||||
$type = MMDBType::UINT64;
|
||||
}
|
||||
$raw = $value;
|
||||
} elseif (is_float($value)) {
|
||||
$type = MMDBType::DOUBLE;
|
||||
$raw = $value;
|
||||
} elseif (is_string($value)) {
|
||||
$type = MMDBType::STRING;
|
||||
$raw = $value;
|
||||
} elseif (is_array($value)) {
|
||||
$type = $this->isList($value) ? MMDBType::ARRAY_T : MMDBType::MAP;
|
||||
$raw = $value;
|
||||
} else {
|
||||
throw new InvalidArgumentException('Unsupported value type: ' . gettype($value));
|
||||
}
|
||||
|
||||
// A pointer costs 2-5 bytes; booleans already cost 1 byte with no
|
||||
// payload, so caching them can only ever waste space -- skip.
|
||||
$cacheKey = $type === MMDBType::BOOLEAN ? null : ($type . ':' . serialize($raw));
|
||||
|
||||
return [$type, $raw, $cacheKey];
|
||||
}
|
||||
|
||||
private function isList(array $a): bool
|
||||
{
|
||||
$i = 0;
|
||||
foreach ($a as $k => $_) {
|
||||
if ($k !== $i++) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/** Encode an MMDB pointer (type 1) record per the spec's 4 size classes. */
|
||||
private function pointerBytes(int $offset): string
|
||||
{
|
||||
if ($offset <= 0x7FF) { // 11-bit value, 1 extra byte
|
||||
$b0 = (MMDBType::POINTER << 5) | (0 << 3) | (($offset >> 8) & 0x07);
|
||||
return chr($b0) . chr($offset & 0xFF);
|
||||
}
|
||||
|
||||
$v = $offset - 2048;
|
||||
if ($v <= 0x7FFFF) { // 19-bit value, 2 extra bytes
|
||||
$b0 = (MMDBType::POINTER << 5) | (1 << 3) | (($v >> 16) & 0x07);
|
||||
return chr($b0) . chr(($v >> 8) & 0xFF) . chr($v & 0xFF);
|
||||
}
|
||||
|
||||
$v2 = $offset - 2048 - 524288;
|
||||
if ($v2 <= 0x7FFFFFF) { // 27-bit value, 3 extra bytes
|
||||
$b0 = (MMDBType::POINTER << 5) | (2 << 3) | (($v2 >> 24) & 0x07);
|
||||
return chr($b0) . chr(($v2 >> 16) & 0xFF) . chr(($v2 >> 8) & 0xFF) . chr($v2 & 0xFF);
|
||||
}
|
||||
|
||||
// Size class 3: full absolute 32-bit offset, 4 extra bytes.
|
||||
$b0 = (MMDBType::POINTER << 5) | (3 << 3);
|
||||
return chr($b0) . pack('N', $offset);
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------
|
||||
// Binary trie used to build the MMDB search tree
|
||||
// ---------------------------------------------------------------------
|
||||
@@ -266,6 +427,28 @@ final class MMDBTree
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Look up the data offset of whatever already-inserted network covers
|
||||
* this bit path (the "parent" network), without mutating the tree.
|
||||
* Returns null if no covering network has been inserted yet.
|
||||
*/
|
||||
public function lookupCoveringData(array $bits, int $prefixLen): ?int
|
||||
{
|
||||
$current = 0;
|
||||
for ($i = 0; $i < $prefixLen; $i++) {
|
||||
$bit = $bits[$i];
|
||||
$rec = $this->nodes[$current][$bit];
|
||||
if ($rec === null) {
|
||||
return null;
|
||||
}
|
||||
if (is_array($rec)) {
|
||||
return $rec['data'];
|
||||
}
|
||||
$current = $rec;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize the tree to bytes using the smallest record size (24, 28,
|
||||
* or 32 bits) that fits every record value.
|
||||
@@ -337,8 +520,9 @@ final class MMDBWriter
|
||||
private $ipVersion;
|
||||
private $totalDepth;
|
||||
private $tree;
|
||||
private $dataSection = '';
|
||||
private $dataCache = [];
|
||||
private $data;
|
||||
private $rawDataByOffset = [];
|
||||
private $pending = [];
|
||||
private $databaseType;
|
||||
private $languages;
|
||||
private $description;
|
||||
@@ -361,26 +545,101 @@ final class MMDBWriter
|
||||
$this->ipVersion = $ipVersion;
|
||||
$this->totalDepth = $ipVersion === 4 ? 32 : 128;
|
||||
$this->tree = new MMDBTree();
|
||||
$this->data = new MMDBDataSection();
|
||||
$this->databaseType = $databaseType;
|
||||
$this->languages = $languages;
|
||||
$this->description = $description;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue a network to be written. Records can be added in ANY order --
|
||||
* broader and narrower networks alike -- regardless of which one you
|
||||
* call this with first. At write() time, records are resolved from
|
||||
* broadest to narrowest so inheritance (see $inherit) always works
|
||||
* correctly no matter the call order.
|
||||
*
|
||||
* @param string $cidr e.g. "1.2.3.0/24", "2001:db8::/32", or a bare IP
|
||||
* @param mixed $data scalar/array/MMDBValue data to associate with the network
|
||||
* @param bool $inherit if true (default), and this network falls inside
|
||||
* a broader network also added to this writer,
|
||||
* fields from that broader record are merged in
|
||||
* first (this record's own fields still win on
|
||||
* conflicts).
|
||||
*/
|
||||
public function addRecord(string $cidr, $data): void
|
||||
public function addRecord(string $cidr, $data, bool $inherit = true): void
|
||||
{
|
||||
[$ip, $prefixLen] = $this->parseCidr($cidr);
|
||||
[$bits, $treePrefixLen] = $this->ipToTreeBits($ip, $prefixLen);
|
||||
$offset = $this->internData($data);
|
||||
$this->pending[] = [$bits, $treePrefixLen, $data, $inherit, count($this->pending)];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve all queued records (broadest network first, so inheritance
|
||||
* is correct regardless of the order addRecord() was called in) and
|
||||
* insert them into the tree.
|
||||
*/
|
||||
private function resolveRecords(): void
|
||||
{
|
||||
$records = $this->pending;
|
||||
usort($records, function ($a, $b) {
|
||||
return $a[1] <=> $b[1] ?: $a[4] <=> $b[4]; // prefix length asc, then original call order
|
||||
});
|
||||
|
||||
foreach ($records as [$bits, $treePrefixLen, $data, $inherit]) {
|
||||
if ($inherit) {
|
||||
$parentOffset = $this->tree->lookupCoveringData($bits, $treePrefixLen);
|
||||
if ($parentOffset !== null && isset($this->rawDataByOffset[$parentOffset])) {
|
||||
$data = $this->mergeWithParent($this->rawDataByOffset[$parentOffset], $data);
|
||||
}
|
||||
}
|
||||
|
||||
$offset = $this->data->internTop($data);
|
||||
$this->rawDataByOffset[$offset] = $data;
|
||||
$this->tree->insert($bits, $treePrefixLen, $offset);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Shallow-merge a child record over a parent record: parent fields are
|
||||
* inherited, and any key the child also sets overrides the parent's
|
||||
* value for that key. Non-map values (or MMDBValue-wrapped non-maps)
|
||||
* are left untouched -- the child value simply replaces the parent.
|
||||
*/
|
||||
private function mergeWithParent($parentData, $data)
|
||||
{
|
||||
$parentArr = $parentData instanceof MMDBValue && $parentData->type === MMDBType::MAP
|
||||
? $parentData->value
|
||||
: $parentData;
|
||||
$childArr = $data instanceof MMDBValue && $data->type === MMDBType::MAP
|
||||
? $data->value
|
||||
: $data;
|
||||
|
||||
if (!$this->isMap($parentArr) || !$this->isMap($childArr)) {
|
||||
return $data; // nothing sensible to merge; child value wins outright
|
||||
}
|
||||
|
||||
return array_replace($parentArr, $childArr);
|
||||
}
|
||||
|
||||
private function isMap($v): bool
|
||||
{
|
||||
if (!is_array($v) || $v === []) {
|
||||
return false;
|
||||
}
|
||||
$i = 0;
|
||||
foreach ($v as $k => $_) {
|
||||
if ($k !== $i++) {
|
||||
return true; // has at least one non-sequential/string key -> it's a map
|
||||
}
|
||||
}
|
||||
return false; // it's a plain list
|
||||
}
|
||||
|
||||
public function write(string $path): void
|
||||
{
|
||||
[$treeBytes, $recordSize] = $this->tree->serialize(strlen($this->dataSection));
|
||||
$this->resolveRecords();
|
||||
|
||||
[$treeBytes, $recordSize] = $this->tree->serialize(strlen($this->data->bytes()));
|
||||
|
||||
$metadata = [
|
||||
'node_count' => MMDBValue::uint32($this->tree->nodeCount()),
|
||||
@@ -399,25 +658,13 @@ final class MMDBWriter
|
||||
$separator = str_repeat("\x00", 16);
|
||||
$marker = "\xAB\xCD\xEFMaxMind.com";
|
||||
|
||||
$blob = $treeBytes . $separator . $this->dataSection . $marker . $metadataBytes;
|
||||
$blob = $treeBytes . $separator . $this->data->bytes() . $marker . $metadataBytes;
|
||||
|
||||
if (file_put_contents($path, $blob) === false) {
|
||||
throw new RuntimeException("Failed to write $path");
|
||||
}
|
||||
}
|
||||
|
||||
private function internData($data): int
|
||||
{
|
||||
$key = serialize($data);
|
||||
if (isset($this->dataCache[$key])) {
|
||||
return $this->dataCache[$key];
|
||||
}
|
||||
$offset = strlen($this->dataSection);
|
||||
$this->dataSection .= MMDBEncoder::encode($data);
|
||||
$this->dataCache[$key] = $offset;
|
||||
return $offset;
|
||||
}
|
||||
|
||||
private function parseCidr(string $cidr): array
|
||||
{
|
||||
if (strpos($cidr, '/') !== false) {
|
||||
395
lib/MMDB_ASN_Extractor.php
Normal file
395
lib/MMDB_ASN_Extractor.php
Normal file
@@ -0,0 +1,395 @@
|
||||
<?php
|
||||
|
||||
/**
|
||||
* GeoLite2ASNExtractor
|
||||
*
|
||||
* Pure-PHP reader for MaxMind's GeoLite2-ASN.mmdb that walks the entire
|
||||
* binary search tree and returns every network grouped by ASN, e.g.:
|
||||
*
|
||||
* [
|
||||
* "as1400" => ["1.0.0.0/32", "1.0.0.1/32", ...],
|
||||
* "as13335" => ["1.1.1.0/24", ...],
|
||||
* ]
|
||||
*
|
||||
* No Composer dependency required — implements the MaxMind DB binary
|
||||
* format (search tree + data section) directly per the public spec:
|
||||
* https://maxmind.github.io/MaxMind-DB/
|
||||
*
|
||||
* Usage:
|
||||
* $extractor = new GeoLite2ASNExtractor('/path/to/GeoLite2-ASN.mmdb');
|
||||
* $asnMap = $extractor->extract();
|
||||
*
|
||||
* Notes:
|
||||
* - GeoLite2-ASN.mmdb is normally built as an IPv6 tree with the IPv4
|
||||
* address space embedded under ::/96. This class detects that and
|
||||
* emits plain IPv4 CIDRs (e.g. "1.0.0.0/32"), skipping true
|
||||
* IPv6-only ranges. Set $includeIPv6 = true in extract() if you
|
||||
* want IPv6 ranges included too (returned as e.g. "2606:4700::/32").
|
||||
* - The whole file is loaded into memory once (file_get_contents).
|
||||
* GeoLite2-ASN.mmdb is tens of MB, which is fine for most setups,
|
||||
* but be aware if running under a memory-constrained environment.
|
||||
* - Walking the full tree is O(number of tree nodes), which can be a
|
||||
* few million recursive calls for the ASN database. This can take
|
||||
* anywhere from several seconds to a couple of minutes depending on
|
||||
* hardware. Consider raising max_execution_time for CLI/cron use.
|
||||
*/
|
||||
class GeoLite2ASNExtractor
|
||||
{
|
||||
private const METADATA_MARKER = "\xab\xcd\xefMaxMind.com";
|
||||
|
||||
private string $data;
|
||||
private array $metadata = [];
|
||||
|
||||
private int $nodeCount = 0;
|
||||
private int $recordSize = 0;
|
||||
private int $nodeByteSize = 0;
|
||||
private int $searchTreeSize = 0;
|
||||
private int $dataSectionStart = 0;
|
||||
private int $ipVersion = 6;
|
||||
|
||||
public function __construct(string $mmdbPath)
|
||||
{
|
||||
if (!is_readable($mmdbPath)) {
|
||||
throw new \RuntimeException("Cannot read file: {$mmdbPath}");
|
||||
}
|
||||
|
||||
$contents = file_get_contents($mmdbPath);
|
||||
if ($contents === false) {
|
||||
throw new \RuntimeException("Failed to read file: {$mmdbPath}");
|
||||
}
|
||||
|
||||
$this->data = $contents;
|
||||
$this->parseMetadata();
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional convenience helper: download an .mmdb file over HTTP(S)
|
||||
* and construct an extractor from it.
|
||||
*/
|
||||
public static function fromUrl(string $url, ?string $saveTo = null): self
|
||||
{
|
||||
$tmpPath = $saveTo ?? tempnam(sys_get_temp_dir(), 'mmdb_');
|
||||
|
||||
$ch = curl_init($url);
|
||||
$fh = fopen($tmpPath, 'wb');
|
||||
if ($fh === false) {
|
||||
throw new \RuntimeException("Cannot open temp file for writing: {$tmpPath}");
|
||||
}
|
||||
|
||||
curl_setopt_array($ch, [
|
||||
CURLOPT_FILE => $fh,
|
||||
CURLOPT_FOLLOWLOCATION => true,
|
||||
CURLOPT_MAXREDIRS => 5,
|
||||
CURLOPT_TIMEOUT => 300,
|
||||
CURLOPT_FAILONERROR => true,
|
||||
CURLOPT_USERAGENT => 'GeoLite2ASNExtractor/1.0',
|
||||
]);
|
||||
|
||||
$ok = curl_exec($ch);
|
||||
$err = curl_error($ch);
|
||||
curl_close($ch);
|
||||
fclose($fh);
|
||||
|
||||
if ($ok === false) {
|
||||
if ($saveTo === null) {
|
||||
@unlink($tmpPath);
|
||||
}
|
||||
throw new \RuntimeException("Download failed: {$err}");
|
||||
}
|
||||
|
||||
return new self($tmpPath);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk the entire tree and return ["as<N>" => ["a.b.c.d/x", ...], ...]
|
||||
*/
|
||||
public function extract(bool $includeIPv6 = false): array
|
||||
{
|
||||
$result = [];
|
||||
$this->walk(0, '', $result, $includeIPv6);
|
||||
return $result;
|
||||
}
|
||||
|
||||
public function getMetadata(): array
|
||||
{
|
||||
return $this->metadata;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Search tree traversal
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* @param string $bits '0'/'1' characters representing the path taken so far
|
||||
*/
|
||||
private function walk(int $recordValue, string $bits, array &$result, bool $includeIPv6): void
|
||||
{
|
||||
if ($recordValue === $this->nodeCount) {
|
||||
// Empty branch, no data assigned here.
|
||||
return;
|
||||
}
|
||||
|
||||
if ($recordValue > $this->nodeCount) {
|
||||
// Leaf: record points into the data section.
|
||||
//
|
||||
// Per the MaxMind DB spec, the offset within the data section is
|
||||
// (record_value - node_count - 16), and the data section itself
|
||||
// starts at (search_tree_size + 16) — the two 16s cancel out, so
|
||||
// the correct absolute file offset is search_tree_size plus
|
||||
// (record_value - node_count). Using data_section_start here
|
||||
// instead would shift every leaf read by 16 bytes.
|
||||
$absoluteOffset = $this->searchTreeSize + ($recordValue - $this->nodeCount);
|
||||
[$record, ] = $this->decodeData($absoluteOffset);
|
||||
$this->addResult($bits, $record, $result, $includeIPv6);
|
||||
return;
|
||||
}
|
||||
|
||||
// Otherwise it's an index of another node — recurse both children.
|
||||
[$left, $right] = $this->readNode($recordValue);
|
||||
$this->walk($left, $bits . '0', $result, $includeIPv6);
|
||||
$this->walk($right, $bits . '1', $result, $includeIPv6);
|
||||
}
|
||||
|
||||
private function readNode(int $nodeNumber): array
|
||||
{
|
||||
$offset = $nodeNumber * $this->nodeByteSize;
|
||||
$bytes = substr($this->data, $offset, $this->nodeByteSize);
|
||||
|
||||
if ($this->recordSize === 24) {
|
||||
$left = (ord($bytes[0]) << 16) | (ord($bytes[1]) << 8) | ord($bytes[2]);
|
||||
$right = (ord($bytes[3]) << 16) | (ord($bytes[4]) << 8) | ord($bytes[5]);
|
||||
} elseif ($this->recordSize === 28) {
|
||||
$middle = ord($bytes[3]);
|
||||
$left = (ord($bytes[0]) << 16) | (ord($bytes[1]) << 8) | ord($bytes[2]);
|
||||
$left |= ($middle >> 4) << 24;
|
||||
$right = (($middle & 0x0f) << 24) | (ord($bytes[4]) << 16) | (ord($bytes[5]) << 8) | ord($bytes[6]);
|
||||
} elseif ($this->recordSize === 32) {
|
||||
$left = (ord($bytes[0]) << 24) | (ord($bytes[1]) << 16) | (ord($bytes[2]) << 8) | ord($bytes[3]);
|
||||
$right = (ord($bytes[4]) << 24) | (ord($bytes[5]) << 16) | (ord($bytes[6]) << 8) | ord($bytes[7]);
|
||||
} else {
|
||||
throw new \RuntimeException("Unsupported record_size: {$this->recordSize}");
|
||||
}
|
||||
|
||||
return [$left, $right];
|
||||
}
|
||||
|
||||
private function addResult(string $bits, $record, array &$result, bool $includeIPv6): void
|
||||
{
|
||||
if (!is_array($record) || !isset($record['autonomous_system_number'])) {
|
||||
return; // no ASN data at this leaf
|
||||
}
|
||||
|
||||
$asn = 'as' . $record['autonomous_system_number'];
|
||||
$prefixLen = strlen($bits);
|
||||
|
||||
if ($this->ipVersion === 6) {
|
||||
if ($prefixLen >= 96 && substr($bits, 0, 96) === str_repeat('0', 96)) {
|
||||
// Embedded IPv4 space (::/96).
|
||||
$ipv4Bits = substr($bits, 96);
|
||||
$cidr = $this->bitsToIPv4($ipv4Bits) . '/' . strlen($ipv4Bits);
|
||||
$result[$asn][] = $cidr;
|
||||
} elseif ($includeIPv6) {
|
||||
$cidr = $this->bitsToIPv6($bits) . '/' . $prefixLen;
|
||||
$result[$asn][] = $cidr;
|
||||
}
|
||||
// else: real IPv6 range but caller doesn't want IPv6 — skip.
|
||||
} else {
|
||||
// Pure IPv4 tree (older / rare db builds).
|
||||
$cidr = $this->bitsToIPv4($bits) . '/' . $prefixLen;
|
||||
$result[$asn][] = $cidr;
|
||||
}
|
||||
}
|
||||
|
||||
private function bitsToIPv4(string $bits): string
|
||||
{
|
||||
$bits = str_pad($bits, 32, '0'); // pad host bits with 0 -> network address
|
||||
$octets = [];
|
||||
for ($i = 0; $i < 4; $i++) {
|
||||
$octets[] = bindec(substr($bits, $i * 8, 8));
|
||||
}
|
||||
return implode('.', $octets);
|
||||
}
|
||||
|
||||
private function bitsToIPv6(string $bits): string
|
||||
{
|
||||
$bits = str_pad($bits, 128, '0');
|
||||
$groups = [];
|
||||
for ($i = 0; $i < 8; $i++) {
|
||||
$groups[] = dechex(bindec(substr($bits, $i * 16, 16)));
|
||||
}
|
||||
$expanded = implode(':', $groups);
|
||||
return inet_ntop(inet_pton($expanded)) ?: $expanded;
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
// Metadata / data section decoding (MaxMind DB data format)
|
||||
// ------------------------------------------------------------------
|
||||
|
||||
private function parseMetadata(): void
|
||||
{
|
||||
$pos = strrpos($this->data, self::METADATA_MARKER);
|
||||
if ($pos === false) {
|
||||
throw new \RuntimeException('MaxMind DB metadata marker not found — not a valid .mmdb file');
|
||||
}
|
||||
|
||||
$offset = $pos + strlen(self::METADATA_MARKER);
|
||||
[$metadata, ] = $this->decodeData($offset);
|
||||
|
||||
if (!is_array($metadata) || !isset($metadata['node_count'], $metadata['record_size'], $metadata['ip_version'])) {
|
||||
throw new \RuntimeException('Malformed MaxMind DB metadata');
|
||||
}
|
||||
|
||||
$this->metadata = $metadata;
|
||||
$this->nodeCount = (int) $metadata['node_count'];
|
||||
$this->recordSize = (int) $metadata['record_size'];
|
||||
$this->ipVersion = (int) $metadata['ip_version'];
|
||||
$this->nodeByteSize = (int) (($this->recordSize * 2) / 8);
|
||||
$this->searchTreeSize = $this->nodeCount * $this->nodeByteSize;
|
||||
$this->dataSectionStart = $this->searchTreeSize + 16; // 16-byte all-zero separator
|
||||
}
|
||||
|
||||
/**
|
||||
* Decodes one MaxMind DB data item starting at $offset.
|
||||
* Returns [decodedValue, offsetAfterThisItem].
|
||||
*/
|
||||
private function decodeData(int $offset): array
|
||||
{
|
||||
$ctrl = ord($this->data[$offset]);
|
||||
$offset++;
|
||||
|
||||
$type = $ctrl >> 5; // top 3 bits
|
||||
$size = $ctrl & 0x1f; // bottom 5 bits
|
||||
|
||||
if ($type === 0) {
|
||||
// Extended type: real type = next byte + 7
|
||||
$type = 7 + ord($this->data[$offset]);
|
||||
$offset++;
|
||||
}
|
||||
|
||||
if ($type === 1) {
|
||||
return $this->decodePointer($ctrl, $offset);
|
||||
}
|
||||
|
||||
if ($type !== 14) { // not boolean — booleans store their value in $size directly
|
||||
if ($size === 29) {
|
||||
$size = 29 + ord($this->data[$offset]);
|
||||
$offset += 1;
|
||||
} elseif ($size === 30) {
|
||||
$size = 285 + $this->readUint($offset, 2);
|
||||
$offset += 2;
|
||||
} elseif ($size === 31) {
|
||||
$size = 65821 + $this->readUint($offset, 3);
|
||||
$offset += 3;
|
||||
}
|
||||
}
|
||||
|
||||
switch ($type) {
|
||||
case 2: // utf8_string
|
||||
case 4: // bytes
|
||||
$value = substr($this->data, $offset, $size);
|
||||
$offset += $size;
|
||||
return [$value, $offset];
|
||||
|
||||
case 3: // double
|
||||
$bytes = substr($this->data, $offset, 8);
|
||||
$offset += 8;
|
||||
$u = unpack('E', $bytes);
|
||||
return [$u[1], $offset];
|
||||
|
||||
case 5: // uint16
|
||||
case 6: // uint32
|
||||
case 9: // uint64 (may lose precision above 2^63 — not expected for ASN fields)
|
||||
$value = $this->readUint($offset, $size);
|
||||
$offset += $size;
|
||||
return [$value, $offset];
|
||||
|
||||
case 7: // map
|
||||
$map = [];
|
||||
for ($i = 0; $i < $size; $i++) {
|
||||
[$key, $offset] = $this->decodeData($offset);
|
||||
[$val, $offset] = $this->decodeData($offset);
|
||||
$map[$key] = $val;
|
||||
}
|
||||
return [$map, $offset];
|
||||
|
||||
case 8: // int32
|
||||
$value = $this->readUint($offset, $size);
|
||||
if ($size > 0 && ($value & (1 << (8 * $size - 1)))) {
|
||||
$value -= (1 << (8 * $size));
|
||||
}
|
||||
$offset += $size;
|
||||
return [$value, $offset];
|
||||
|
||||
case 10: // uint128 — returned as a hex string (not used by ASN records)
|
||||
$hex = bin2hex(substr($this->data, $offset, $size));
|
||||
$offset += $size;
|
||||
return [$hex, $offset];
|
||||
|
||||
case 11: // array
|
||||
$arr = [];
|
||||
for ($i = 0; $i < $size; $i++) {
|
||||
[$val, $offset] = $this->decodeData($offset);
|
||||
$arr[] = $val;
|
||||
}
|
||||
return [$arr, $offset];
|
||||
|
||||
case 13: // end marker
|
||||
return [null, $offset];
|
||||
|
||||
case 14: // boolean — value is $size itself (0 or 1)
|
||||
return [$size === 1, $offset];
|
||||
|
||||
case 15: // float
|
||||
$bytes = substr($this->data, $offset, 4);
|
||||
$offset += 4;
|
||||
$u = unpack('G', $bytes);
|
||||
return [$u[1], $offset];
|
||||
|
||||
default:
|
||||
throw new \RuntimeException("Unsupported MaxMind DB data type: {$type} at offset {$offset}");
|
||||
}
|
||||
}
|
||||
|
||||
private function decodePointer(int $ctrl, int $offset): array
|
||||
{
|
||||
$ptrSize = ($ctrl >> 3) & 0x03;
|
||||
$valueHigh = $ctrl & 0x07;
|
||||
|
||||
switch ($ptrSize) {
|
||||
case 0:
|
||||
$pointer = ($valueHigh << 8) | ord($this->data[$offset]);
|
||||
$offset += 1;
|
||||
$base = 0;
|
||||
break;
|
||||
case 1:
|
||||
$pointer = ($valueHigh << 16) | $this->readUint($offset, 2);
|
||||
$offset += 2;
|
||||
$base = 2048;
|
||||
break;
|
||||
case 2:
|
||||
$pointer = ($valueHigh << 24) | $this->readUint($offset, 3);
|
||||
$offset += 3;
|
||||
$base = 526336;
|
||||
break;
|
||||
default: // 3
|
||||
$pointer = $this->readUint($offset, 4);
|
||||
$offset += 4;
|
||||
$base = 0;
|
||||
break;
|
||||
}
|
||||
|
||||
$target = $this->dataSectionStart + $pointer + $base;
|
||||
[$value, ] = $this->decodeData($target);
|
||||
|
||||
return [$value, $offset];
|
||||
}
|
||||
|
||||
private function readUint(int $offset, int $numBytes): int
|
||||
{
|
||||
$value = 0;
|
||||
for ($i = 0; $i < $numBytes; $i++) {
|
||||
$value = ($value << 8) | ord($this->data[$offset + $i]);
|
||||
}
|
||||
return $value;
|
||||
}
|
||||
}
|
||||
628
lib/fuckhtml.php
Normal file
628
lib/fuckhtml.php
Normal file
@@ -0,0 +1,628 @@
|
||||
<?php
|
||||
class fuckhtml{
|
||||
|
||||
public function __construct($html = null, $isfile = false){
|
||||
|
||||
if($html !== null){
|
||||
|
||||
$this->load($html, $isfile);
|
||||
}
|
||||
}
|
||||
|
||||
public function load($html, $isfile = false){
|
||||
|
||||
if(is_array($html)){
|
||||
|
||||
if(!array_key_exists("innerHTML", $html)){
|
||||
|
||||
throw new Exception("(load) Supplied array doesn't contain an innerHTML index");
|
||||
}
|
||||
$html = $html["innerHTML"];
|
||||
}
|
||||
|
||||
if($isfile){
|
||||
|
||||
$handle = fopen($html, "r");
|
||||
$fetch = fread($handle, filesize($html));
|
||||
fclose($handle);
|
||||
|
||||
$this->html = $fetch;
|
||||
}else{
|
||||
|
||||
$this->html = $html;
|
||||
}
|
||||
|
||||
$this->strlen = strlen($this->html);
|
||||
}
|
||||
|
||||
public function getloadedhtml(){
|
||||
|
||||
return $this->html;
|
||||
}
|
||||
|
||||
public function getElementsByTagName(string $tagname){
|
||||
|
||||
$out = [];
|
||||
|
||||
/*
|
||||
Scrape start of the tag. Example
|
||||
<div class="mydiv"> ...
|
||||
*/
|
||||
|
||||
if($tagname == "*"){
|
||||
|
||||
$tagname = '[A-Za-z0-9._-]+';
|
||||
}else{
|
||||
|
||||
$tagname = preg_quote(strtolower($tagname));
|
||||
}
|
||||
|
||||
preg_match_all(
|
||||
'/<\s*(' . $tagname . ')(\s(?:[^>\'"]*|"[^"]*"|\'[^\']*\')+)?\s*>/i',
|
||||
/* '/<\s*(' . $tagname . ')(\s[\S\s]*?)?>/i', */
|
||||
$this->html,
|
||||
$starting_tags,
|
||||
PREG_OFFSET_CAPTURE
|
||||
);
|
||||
|
||||
for($i=0; $i<count($starting_tags[0]); $i++){
|
||||
|
||||
/*
|
||||
Parse attributes
|
||||
*/
|
||||
$attributes = [];
|
||||
|
||||
preg_match_all(
|
||||
'/([^\/\s\\=]+)(?:\s*=\s*("[^"]*"|\'[^\']*\'|[^\s]*))?/i',
|
||||
$starting_tags[2][$i][0],
|
||||
$regex_attributes
|
||||
);
|
||||
|
||||
for($k=0; $k<count($regex_attributes[0]); $k++){
|
||||
|
||||
if(trim($regex_attributes[2][$k]) == ""){
|
||||
|
||||
$attributes[$regex_attributes[1][$k]] =
|
||||
"true";
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
$attributes[strtolower($regex_attributes[1][$k])] =
|
||||
trim($regex_attributes[2][$k], "'\" \n\r\t\v\x00");
|
||||
}
|
||||
|
||||
$out[] = [
|
||||
"tagName" => strtolower($starting_tags[1][$i][0]),
|
||||
"startPos" => $starting_tags[0][$i][1],
|
||||
"endPos" => 0,
|
||||
"startTag" => $starting_tags[0][$i][0],
|
||||
"attributes" => $attributes,
|
||||
"innerHTML" => null
|
||||
];
|
||||
}
|
||||
|
||||
/*
|
||||
Get innerHTML
|
||||
*/
|
||||
// get closing tag positions
|
||||
preg_match_all(
|
||||
'/<\s*\/\s*(' . $tagname . ')\s*>/i',
|
||||
$this->html,
|
||||
$regex_closing_tags,
|
||||
PREG_OFFSET_CAPTURE
|
||||
);
|
||||
|
||||
// merge opening and closing tags together
|
||||
for($i=0; $i<count($regex_closing_tags[1]); $i++){
|
||||
|
||||
$out[] = [
|
||||
"tagName" => strtolower($regex_closing_tags[1][$i][0]),
|
||||
"endTag" => $regex_closing_tags[0][$i][0],
|
||||
"startPos" => $regex_closing_tags[0][$i][1]
|
||||
];
|
||||
}
|
||||
|
||||
usort(
|
||||
$out,
|
||||
function($a, $b){
|
||||
|
||||
return $a["startPos"] > $b["startPos"];
|
||||
}
|
||||
);
|
||||
|
||||
// compute the indent level for each element
|
||||
$level = [];
|
||||
$count = count($out);
|
||||
|
||||
for($i=0; $i<$count; $i++){
|
||||
|
||||
if(!isset($level[$out[$i]["tagName"]])){
|
||||
|
||||
$level[$out[$i]["tagName"]] = 0;
|
||||
}
|
||||
|
||||
if(isset($out[$i]["startTag"])){
|
||||
|
||||
// encountered starting tag
|
||||
$level[$out[$i]["tagName"]]++;
|
||||
$out[$i]["level"] = $level[$out[$i]["tagName"]];
|
||||
}else{
|
||||
|
||||
// encountered closing tag
|
||||
$out[$i]["level"] = $level[$out[$i]["tagName"]];
|
||||
$level[$out[$i]["tagName"]]--;
|
||||
}
|
||||
}
|
||||
|
||||
// if the indent level is the same for a div,
|
||||
// we encountered _THE_ closing tag
|
||||
for($i=0; $i<$count; $i++){
|
||||
|
||||
if(!isset($out[$i]["startTag"])){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
for($k=$i; $k<$count; $k++){
|
||||
|
||||
if(
|
||||
isset($out[$k]["endTag"]) &&
|
||||
$out[$i]["tagName"] == $out[$k]["tagName"] &&
|
||||
$out[$i]["level"]
|
||||
=== $out[$k]["level"]
|
||||
){
|
||||
|
||||
$startlen = strlen($out[$i]["startTag"]);
|
||||
$endlen = strlen($out[$k]["endTag"]);
|
||||
|
||||
$out[$i]["endPos"] = $out[$k]["startPos"] + $endlen;
|
||||
|
||||
$out[$i]["innerHTML"] =
|
||||
substr(
|
||||
$this->html,
|
||||
$out[$i]["startPos"] + $startlen,
|
||||
$out[$k]["startPos"] - ($out[$i]["startPos"] + $startlen)
|
||||
);
|
||||
|
||||
$out[$i]["outerHTML"] =
|
||||
substr(
|
||||
$this->html,
|
||||
$out[$i]["startPos"],
|
||||
$out[$k]["startPos"] - $out[$i]["startPos"] + $endlen
|
||||
);
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// filter out ending divs
|
||||
for($i=0; $i<$count; $i++){
|
||||
|
||||
if(isset($out[$i]["endTag"])){
|
||||
|
||||
unset($out[$i]);
|
||||
}
|
||||
|
||||
unset($out[$i]["startTag"]);
|
||||
}
|
||||
|
||||
return array_values($out);
|
||||
}
|
||||
|
||||
public function getElementsByAttributeName(string $name, $collection = null){
|
||||
|
||||
if($collection === null){
|
||||
|
||||
$collection = $this->getElementsByTagName("*");
|
||||
}elseif(is_string($collection)){
|
||||
|
||||
$collection = $this->getElementsByTagName($collection);
|
||||
}
|
||||
|
||||
$return = [];
|
||||
foreach($collection as $elem){
|
||||
|
||||
foreach($elem["attributes"] as $attrib_name => $attrib_value){
|
||||
|
||||
if($attrib_name == $name){
|
||||
|
||||
$return[] = $elem;
|
||||
continue 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $return;
|
||||
}
|
||||
|
||||
public function getElementsByFuzzyAttributeValue(string $name, string $value, $collection = null){
|
||||
|
||||
$elems = $this->getElementsByAttributeName($name, $collection);
|
||||
|
||||
$value =
|
||||
explode(
|
||||
" ",
|
||||
trim(
|
||||
preg_replace(
|
||||
'/\s+/',
|
||||
" ",
|
||||
$value
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
$return = [];
|
||||
|
||||
foreach($elems as $elem){
|
||||
|
||||
foreach($elem["attributes"] as $attrib_name => $attrib_value){
|
||||
|
||||
$attrib_value =
|
||||
explode(
|
||||
" ",
|
||||
trim(
|
||||
preg_replace(
|
||||
'/\s+/',
|
||||
" ",
|
||||
$attrib_value
|
||||
)
|
||||
)
|
||||
);
|
||||
|
||||
$ac = count($attrib_value);
|
||||
$nc = count($value);
|
||||
$cr = 0;
|
||||
|
||||
for($i=0; $i<$nc; $i++){
|
||||
|
||||
for($k=0; $k<$ac; $k++){
|
||||
|
||||
if($value[$i] == $attrib_value[$k]){
|
||||
|
||||
$cr++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if($cr === $nc){
|
||||
|
||||
$return[] = $elem;
|
||||
continue 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $return;
|
||||
}
|
||||
|
||||
public function getElementsByAttributeValue(string $name, string $value, $collection = null){
|
||||
|
||||
$elems = $this->getElementsByAttributeName($name, $collection);
|
||||
|
||||
$return = [];
|
||||
|
||||
foreach($elems as $elem){
|
||||
|
||||
foreach($elem["attributes"] as $attrib_name => $attrib_value){
|
||||
|
||||
if($attrib_value == $value){
|
||||
|
||||
$return[] = $elem;
|
||||
continue 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return $return;
|
||||
}
|
||||
|
||||
public function getElementById(string $idname, $collection = null){
|
||||
|
||||
$id = $this->getElementsByAttributeValue("id", $idname, $collection);
|
||||
|
||||
if(count($id) !== 0){
|
||||
|
||||
return $id[0];
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
public function getElementsByClassName(string $classname, $collection = null){
|
||||
|
||||
return $this->getElementsByFuzzyAttributeValue("class", $classname, $collection);
|
||||
}
|
||||
|
||||
public function getTextContent($html, $whitespace = false, $trim = true){
|
||||
|
||||
if(is_array($html)){
|
||||
|
||||
if(!array_key_exists("innerHTML", $html)){
|
||||
|
||||
throw new Exception("(getTextContent) Supplied array doesn't contain an innerHTML index");
|
||||
}
|
||||
|
||||
$html = $html["innerHTML"];
|
||||
}
|
||||
|
||||
$html = preg_split('/\n|<\/?br>/i', $html);
|
||||
|
||||
$out = "";
|
||||
for($i=0; $i<count($html); $i++){
|
||||
|
||||
$tmp =
|
||||
html_entity_decode(
|
||||
strip_tags(
|
||||
$html[$i]
|
||||
),
|
||||
ENT_QUOTES | ENT_XML1, "UTF-8"
|
||||
);
|
||||
|
||||
if($trim){
|
||||
|
||||
$tmp = trim($tmp);
|
||||
}
|
||||
|
||||
$out .= $tmp;
|
||||
|
||||
if($whitespace === true){
|
||||
|
||||
$out .= "\n";
|
||||
}else{
|
||||
|
||||
$out .= " ";
|
||||
}
|
||||
}
|
||||
|
||||
if($trim){
|
||||
|
||||
return trim($out);
|
||||
}
|
||||
|
||||
return $out;
|
||||
}
|
||||
|
||||
public function parseJsObject(string $json){
|
||||
|
||||
$bracket = false;
|
||||
$is_close_bracket = false;
|
||||
$escape = false;
|
||||
$lastchar = false;
|
||||
$json_out = null;
|
||||
$last_char = null;
|
||||
|
||||
$keyword_check = null;
|
||||
|
||||
for($i=0; $i<strlen($json); $i++){
|
||||
|
||||
switch($json[$i]){
|
||||
|
||||
case "\"":
|
||||
case "'":
|
||||
if($escape === true){
|
||||
|
||||
break;
|
||||
}
|
||||
|
||||
if($json[$i] == $bracket){
|
||||
|
||||
$bracket = false;
|
||||
$is_close_bracket = true;
|
||||
|
||||
}else{
|
||||
|
||||
if($bracket === false){
|
||||
|
||||
$bracket = $json[$i];
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
default:
|
||||
$is_close_bracket = false;
|
||||
break;
|
||||
}
|
||||
|
||||
if(
|
||||
$json[$i] == "\\" &&
|
||||
!(
|
||||
$lastchar !== false &&
|
||||
$lastchar . $json[$i] == "\\\\"
|
||||
)
|
||||
){
|
||||
|
||||
$escape = true;
|
||||
}else{
|
||||
|
||||
$escape = false;
|
||||
}
|
||||
|
||||
if(
|
||||
$bracket === false &&
|
||||
$is_close_bracket === false
|
||||
){
|
||||
|
||||
// do keyword check
|
||||
$keyword_check .= $json[$i];
|
||||
|
||||
if(in_array($json[$i], [":", "{"])){
|
||||
|
||||
$keyword_check = substr($keyword_check, 0, -1);
|
||||
|
||||
if(
|
||||
preg_match(
|
||||
'/function|array|return/i',
|
||||
$keyword_check
|
||||
)
|
||||
){
|
||||
|
||||
$json_out =
|
||||
preg_replace(
|
||||
'/[{"]*' . preg_quote($keyword_check, "/") . '$/',
|
||||
"",
|
||||
$json_out
|
||||
);
|
||||
}
|
||||
|
||||
$keyword_check = null;
|
||||
}
|
||||
|
||||
// here we know we're not iterating over a quoted string
|
||||
switch($json[$i]){
|
||||
|
||||
case "[":
|
||||
case "{":
|
||||
$json_out .= $json[$i];
|
||||
break;
|
||||
|
||||
case "]":
|
||||
case "}":
|
||||
case ",":
|
||||
case ":":
|
||||
if(!in_array($last_char, ["[", "{", "}", "]", "\""])){
|
||||
|
||||
$json_out .= "\"";
|
||||
}
|
||||
|
||||
$json_out .= $json[$i];
|
||||
break;
|
||||
|
||||
default:
|
||||
if(in_array($last_char, ["{", "[", ",", ":"])){
|
||||
|
||||
$json_out .= "\"";
|
||||
}
|
||||
|
||||
$json_out .= $json[$i];
|
||||
break;
|
||||
}
|
||||
}else{
|
||||
|
||||
$json_out .= $json[$i];
|
||||
}
|
||||
|
||||
$last_char = $json[$i];
|
||||
}
|
||||
|
||||
return json_decode($json_out, true);
|
||||
}
|
||||
|
||||
public function parseJsString($string){
|
||||
|
||||
return
|
||||
preg_replace_callback(
|
||||
'/\\\u[A-Fa-f0-9]{4}|\\\x[A-Fa-f0-9]{2}|\\\n|\\\r/',
|
||||
function($match){
|
||||
|
||||
switch($match[0][1]){
|
||||
|
||||
case "u":
|
||||
return json_decode('"' . $match[0] . '"');
|
||||
break;
|
||||
|
||||
case "x":
|
||||
return mb_convert_encoding(
|
||||
stripcslashes($match[0]),
|
||||
"utf-8",
|
||||
"windows-1252"
|
||||
);
|
||||
break;
|
||||
|
||||
default:
|
||||
return " ";
|
||||
break;
|
||||
}
|
||||
},
|
||||
$string
|
||||
);
|
||||
}
|
||||
|
||||
public function extract_json($json){
|
||||
|
||||
$len = strlen($json);
|
||||
$array_level = 0;
|
||||
$object_level = 0;
|
||||
$in_quote = null;
|
||||
$start = null;
|
||||
|
||||
for($i=0; $i<$len; $i++){
|
||||
|
||||
switch($json[$i]){
|
||||
|
||||
case "\"":
|
||||
case "'":
|
||||
// count preceding backslashes
|
||||
$bsCount = 0;
|
||||
$j = $i - 1;
|
||||
|
||||
while($j >= 0 && $json[$j] === "\\"){
|
||||
$bsCount++;
|
||||
$j--;
|
||||
}
|
||||
|
||||
// quote is NOT escaped if even number of backslashes
|
||||
if($bsCount % 2 === 0){
|
||||
if($in_quote === null){
|
||||
// open quote
|
||||
$in_quote = $json[$i];
|
||||
}elseif($in_quote === $json[$i]){
|
||||
// close quote
|
||||
$in_quote = null;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "[":
|
||||
if($in_quote === null){
|
||||
|
||||
$array_level++;
|
||||
if($start === null){
|
||||
|
||||
$start = $i;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "]":
|
||||
if($in_quote === null){
|
||||
|
||||
$array_level--;
|
||||
}
|
||||
break;
|
||||
|
||||
case "{":
|
||||
if($in_quote === null){
|
||||
|
||||
$object_level++;
|
||||
if($start === null){
|
||||
|
||||
$start = $i;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
case "}":
|
||||
if($in_quote === null){
|
||||
|
||||
$object_level--;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
if(
|
||||
$array_level === 0 &&
|
||||
$object_level === 0 &&
|
||||
$start !== null
|
||||
){
|
||||
|
||||
return substr($json, $start, $i - $start + 1);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// fallback
|
||||
return "[]";
|
||||
}
|
||||
}
|
||||
216
mmdb.php
216
mmdb.php
@@ -6,9 +6,15 @@ class mmdb_creator{
|
||||
|
||||
public function __construct(){
|
||||
|
||||
$this->ip_list = [];
|
||||
|
||||
include "lib/fuckhtml.php";
|
||||
$this->fuckhtml = new fuckhtml();
|
||||
|
||||
//
|
||||
// Import database
|
||||
// Download data
|
||||
//
|
||||
/*
|
||||
if(!file_exists("data")){ mkdir("data"); }
|
||||
chdir("data");
|
||||
if(!file_exists("blocklist-ipsets")){
|
||||
@@ -23,6 +29,9 @@ class mmdb_creator{
|
||||
chdir("..");
|
||||
}
|
||||
|
||||
echo "Downloading MaxMind's GeoLite2-ASN database...\n";
|
||||
$this->dl("https://git.io/GeoLite2-ASN.mmdb", "GeoLite2-ASN.mmdb");
|
||||
|
||||
echo "Downloading Tunnelbear list\n";
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
|
||||
|
||||
@@ -37,24 +46,95 @@ class mmdb_creator{
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
|
||||
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
|
||||
|
||||
echo "Downloading hosting lists... (";
|
||||
if(!file_exists("x4bnet-hosting")){ mkdir("x4bnet-hosting"); }
|
||||
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt", "x4bnet-hosting/ipv4.txt"); echo "1,";
|
||||
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt", "x4bnet-hosting/ipv6.txt"); echo "2)\n";
|
||||
|
||||
echo "Downloading CDN list...\n";
|
||||
$this->dl("https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst", "cdn.lst");
|
||||
|
||||
echo "Downloading Tor exit node list...\n";
|
||||
$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
|
||||
|
||||
chdir("..");
|
||||
echo "Done! Creating MMDB database...\n";
|
||||
chdir("..");*/
|
||||
|
||||
//
|
||||
// Construct ASN blocklist
|
||||
//
|
||||
echo "Extracting ASNs from GeoLite database...\n";
|
||||
|
||||
require "lib/MMDB_ASN_Extractor.php";
|
||||
|
||||
$asn_extractor = new GeoLite2ASNExtractor("data/GeoLite2-ASN.mmdb");
|
||||
$mmdb_asns = $asn_extractor->extract();
|
||||
|
||||
|
||||
//
|
||||
// Scrape ASN classifications from BGP.tools
|
||||
//
|
||||
$good =
|
||||
array_unique(
|
||||
array_merge(
|
||||
$this->parse_bgptools("dsl"),
|
||||
$this->parse_bgptools("mobile")
|
||||
)
|
||||
);
|
||||
|
||||
$bad =
|
||||
array_unique(
|
||||
array_merge(
|
||||
$this->parse_bgptools("cdn"),
|
||||
$this->parse_bgptools("vpsh"),
|
||||
$this->parse_bgptools("vpn")
|
||||
)
|
||||
);
|
||||
|
||||
echo "Processing hosting ASNs...\n";
|
||||
foreach($bad as $bad_asn){
|
||||
|
||||
// if a bad ASN is found to be a DSL/mobile provider, ignore
|
||||
if(in_array($bad_asn, $good)){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!isset($mmdb_asns[$bad_asn])){
|
||||
|
||||
// no IP range available for that ASN
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Hosting: $bad_asn (" . count($mmdb_asns[$bad_asn]) . " ranges)\n";
|
||||
|
||||
foreach($mmdb_asns[$bad_asn] as $ip_range){
|
||||
|
||||
$this->ip_list[$ip_range]["hosting"] = true;
|
||||
}
|
||||
}
|
||||
|
||||
echo "Processing residential ASNs...\n";
|
||||
foreach($good as $good_asn){
|
||||
|
||||
// if a residential ASN is found to be a CDN/VPSH/VPN provider, ignore
|
||||
if(in_array($good_asn, $bad)){
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if(!isset($mmdb_asns[$good_asn])){
|
||||
|
||||
// no IP range available for that ASN
|
||||
continue;
|
||||
}
|
||||
|
||||
echo "Residential: $good_asn (" . count($mmdb_asns[$good_asn]) . " ranges)\n";
|
||||
|
||||
foreach($mmdb_asns[$good_asn] as $ip_range){
|
||||
|
||||
$this->ip_list[$ip_range]["residential"] = true;
|
||||
}
|
||||
}
|
||||
|
||||
echo "Clearing memory...\n";
|
||||
unset($good);
|
||||
unset($bad);
|
||||
|
||||
//
|
||||
// Construct database
|
||||
//
|
||||
$this->ip_list = [];
|
||||
|
||||
// import miscelaneous lists
|
||||
$this->import_firehol_lists(
|
||||
@@ -66,9 +146,6 @@ class mmdb_creator{
|
||||
],
|
||||
"tor" => [
|
||||
"tor-exit-list.txt"
|
||||
],
|
||||
"hosting" => [
|
||||
"cdn.lst" // mansourjabin's
|
||||
]
|
||||
],
|
||||
""
|
||||
@@ -85,16 +162,6 @@ class mmdb_creator{
|
||||
"x4bnet-vpn/"
|
||||
);
|
||||
|
||||
$this->import_firehol_lists(
|
||||
[
|
||||
"hosting" => [
|
||||
"ipv4.txt",
|
||||
"ipv6.txt",
|
||||
]
|
||||
],
|
||||
"x4bnet-hosting/"
|
||||
);
|
||||
|
||||
// import firehol lists
|
||||
$this->import_firehol_lists(
|
||||
[
|
||||
@@ -121,13 +188,13 @@ class mmdb_creator{
|
||||
"blocklist-ipsets/"
|
||||
);
|
||||
|
||||
require __DIR__ . '/MMDBWriter.php';
|
||||
require "lib/MMDBWriter.php";
|
||||
|
||||
$w = new MMDBWriter(
|
||||
6, // accepts IPv4 and IPv6 networks
|
||||
'GeoIP-Custom', // database_type
|
||||
['en'], // languages
|
||||
['en' => 'Cloudfish']
|
||||
"GeoIP-Custom", // database_type
|
||||
["en"], // languages
|
||||
["en" => "Cloudfish"]
|
||||
);
|
||||
|
||||
echo "Generating mmdb file\n";
|
||||
@@ -139,12 +206,14 @@ class mmdb_creator{
|
||||
//"city" => "Ashburn",
|
||||
...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
|
||||
...(isset($data["tor"]) ? ["is_tor" => true] : []),
|
||||
...(isset($data["hosting"]) ? ["is_hosting" => true] : [])
|
||||
...(isset($data["hosting"]) ? ["is_hosting" => true] : []),
|
||||
...(isset($data["residential"]) ? ["is_residential" => true] : [])
|
||||
//"asn" => MMDBValue::uint32(15169),
|
||||
]);
|
||||
}
|
||||
|
||||
$w->write(__DIR__ . '/output.mmdb');
|
||||
echo "Saving database...\n";
|
||||
$w->write(__DIR__ . "/output.mmdb");
|
||||
|
||||
echo "done\n";
|
||||
}
|
||||
@@ -182,4 +251,89 @@ class mmdb_creator{
|
||||
$data = file_get_contents($url);
|
||||
file_put_contents($path, $data);
|
||||
}
|
||||
|
||||
public function curl($url){
|
||||
|
||||
$curlproc = curl_init();
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_URL, $url);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_ENCODING, ""); // default encoding
|
||||
curl_setopt($curlproc, CURLOPT_HTTPHEADER, [
|
||||
"User-Agent: Mozilla/5.0 (X11; Linux x86_64; rv:153.0) Gecko/20100101 Firefox/153.0",
|
||||
"Accept: text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language: en-US,en;q=0.9",
|
||||
"Accept-Encoding: gzip, deflate, br",
|
||||
"DNT: 1",
|
||||
"Connection: keep-alive",
|
||||
"Upgrade-Insecure-Requests: 1",
|
||||
"Sec-Fetch-Dest: document",
|
||||
"Sec-Fetch-Mode: navigate",
|
||||
"Sec-Fetch-Site: none",
|
||||
"Sec-Fetch-User: ?1"
|
||||
]);
|
||||
|
||||
curl_setopt($curlproc, CURLOPT_RETURNTRANSFER, true);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYHOST, 2);
|
||||
curl_setopt($curlproc, CURLOPT_SSL_VERIFYPEER, true);
|
||||
curl_setopt($curlproc, CURLOPT_CONNECTTIMEOUT, 30);
|
||||
curl_setopt($curlproc, CURLOPT_TIMEOUT, 30);
|
||||
|
||||
$data = curl_exec($curlproc);
|
||||
|
||||
if(curl_errno($curlproc)){
|
||||
|
||||
throw new Exception(curl_error($curlproc));
|
||||
}
|
||||
|
||||
curl_close($curlproc);
|
||||
return $data;
|
||||
}
|
||||
|
||||
public function parse_bgptools($tag){
|
||||
|
||||
$page = $this->curl("https://bgp.tools/tags/{$tag}");
|
||||
|
||||
$asns = [];
|
||||
|
||||
$this->fuckhtml->load($page);
|
||||
|
||||
$table =
|
||||
$this->fuckhtml
|
||||
->getElementById("upstreamTable", "table");
|
||||
|
||||
if($table === false){
|
||||
|
||||
throw new Exception("Failed to grep table element on bgptool's {$tag} page");
|
||||
}
|
||||
|
||||
$this->fuckhtml->load($table);
|
||||
|
||||
$trs =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("tr");
|
||||
|
||||
foreach($trs as $tr){
|
||||
|
||||
$this->fuckhtml->load($tr);
|
||||
|
||||
$tds =
|
||||
$this->fuckhtml
|
||||
->getElementsByTagName("td");
|
||||
|
||||
if(!isset($tds[1])){ continue; }
|
||||
|
||||
$asns[] =
|
||||
strtolower(
|
||||
$this->fuckhtml
|
||||
->getTextContent(
|
||||
$tds[1]
|
||||
)
|
||||
);
|
||||
}
|
||||
|
||||
echo "Scraped bgp.tools/tags/{$tag} (got " . number_format(count($asns)) . " ASNs)\n";
|
||||
|
||||
return $asns;
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user