i exist and i must consume
This commit is contained in:
4
.gitignore
vendored
Normal file
4
.gitignore
vendored
Normal file
@@ -0,0 +1,4 @@
|
|||||||
|
output.mmdb
|
||||||
|
data
|
||||||
|
data/*
|
||||||
|
data/
|
||||||
459
MMDBWriter.php
Normal file
459
MMDBWriter.php
Normal file
@@ -0,0 +1,459 @@
|
|||||||
|
<?php
|
||||||
|
/**
|
||||||
|
* MMDBWriter - a pure-PHP library for generating MaxMind DB (.mmdb) files.
|
||||||
|
*
|
||||||
|
* Implements the MaxMind DB binary format from scratch:
|
||||||
|
* https://maxmind.github.io/MaxMind-DB/
|
||||||
|
*
|
||||||
|
* Supports IPv4 and IPv6 (dual-stack) databases, arbitrary nested
|
||||||
|
* map/array data structures, and all MMDB scalar types.
|
||||||
|
*
|
||||||
|
* Usage:
|
||||||
|
* $w = new MMDBWriter(6, 'GeoIP-Custom', ['en']);
|
||||||
|
* $w->addRecord('1.2.3.0/24', ['country' => 'US', 'city' => 'Ashburn']);
|
||||||
|
* $w->addRecord('2001:db8::/32', ['country' => 'DE']);
|
||||||
|
* $w->write('/path/to/out.mmdb');
|
||||||
|
*/
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// Explicit value typing (use when auto-detection of PHP types isn't
|
||||||
|
// precise enough, e.g. you need a uint16 instead of the default uint32).
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
final class MMDBType
|
||||||
|
{
|
||||||
|
const POINTER = 1;
|
||||||
|
const STRING = 2;
|
||||||
|
const DOUBLE = 3;
|
||||||
|
const BYTES = 4;
|
||||||
|
const UINT16 = 5;
|
||||||
|
const UINT32 = 6;
|
||||||
|
const MAP = 7;
|
||||||
|
const INT32 = 8;
|
||||||
|
const UINT64 = 9;
|
||||||
|
const UINT128 = 10;
|
||||||
|
const ARRAY_T = 11;
|
||||||
|
const BOOLEAN = 14;
|
||||||
|
const FLOAT = 15;
|
||||||
|
}
|
||||||
|
|
||||||
|
final class MMDBValue
|
||||||
|
{
|
||||||
|
public $type;
|
||||||
|
public $value;
|
||||||
|
|
||||||
|
private function __construct(int $type, $value)
|
||||||
|
{
|
||||||
|
$this->type = $type;
|
||||||
|
$this->value = $value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public static function string(string $v): self { return new self(MMDBType::STRING, $v); }
|
||||||
|
public static function bytes(string $v): self { return new self(MMDBType::BYTES, $v); }
|
||||||
|
public static function double(float $v): self { return new self(MMDBType::DOUBLE, $v); }
|
||||||
|
public static function float(float $v): self { return new self(MMDBType::FLOAT, $v); }
|
||||||
|
public static function uint16(int $v): self { return new self(MMDBType::UINT16, $v); }
|
||||||
|
public static function uint32(int $v): self { return new self(MMDBType::UINT32, $v); }
|
||||||
|
public static function uint64(int $v): self { return new self(MMDBType::UINT64, $v); }
|
||||||
|
// $v may be a PHP int (<= 64 bit) or a hex string like "0x1ff...".
|
||||||
|
public static function uint128($v): self { return new self(MMDBType::UINT128, $v); }
|
||||||
|
public static function int32(int $v): self { return new self(MMDBType::INT32, $v); }
|
||||||
|
public static function bool(bool $v): self { return new self(MMDBType::BOOLEAN, $v); }
|
||||||
|
public static function map(array $v): self { return new self(MMDBType::MAP, $v); }
|
||||||
|
public static function arr(array $v): self { return new self(MMDBType::ARRAY_T, $v); }
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// Binary encoder for the MaxMind DB "data format"
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
final class MMDBEncoder
|
||||||
|
{
|
||||||
|
/** Encode any supported PHP value (or MMDBValue wrapper) to raw bytes. */
|
||||||
|
public static function encode($value): string
|
||||||
|
{
|
||||||
|
if ($value instanceof MMDBValue) {
|
||||||
|
return self::encodeTyped($value->type, $value->value);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (is_bool($value)) {
|
||||||
|
return self::encodeTyped(MMDBType::BOOLEAN, $value);
|
||||||
|
}
|
||||||
|
if (is_int($value)) {
|
||||||
|
if ($value >= 0 && $value <= 0xFFFFFFFF) {
|
||||||
|
return self::encodeTyped(MMDBType::UINT32, $value);
|
||||||
|
}
|
||||||
|
if ($value < 0 && $value >= -2147483648) {
|
||||||
|
return self::encodeTyped(MMDBType::INT32, $value);
|
||||||
|
}
|
||||||
|
return self::encodeTyped(MMDBType::UINT64, $value);
|
||||||
|
}
|
||||||
|
if (is_float($value)) {
|
||||||
|
return self::encodeTyped(MMDBType::DOUBLE, $value);
|
||||||
|
}
|
||||||
|
if (is_string($value)) {
|
||||||
|
return self::encodeTyped(MMDBType::STRING, $value);
|
||||||
|
}
|
||||||
|
if (is_array($value)) {
|
||||||
|
if (self::isList($value)) {
|
||||||
|
return self::encodeTyped(MMDBType::ARRAY_T, $value);
|
||||||
|
}
|
||||||
|
return self::encodeTyped(MMDBType::MAP, $value);
|
||||||
|
}
|
||||||
|
|
||||||
|
throw new InvalidArgumentException('Unsupported value type: ' . gettype($value));
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function isList(array $a): bool
|
||||||
|
{
|
||||||
|
$i = 0;
|
||||||
|
foreach ($a as $k => $_) {
|
||||||
|
if ($k !== $i++) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function encodeTyped(int $type, $value): string
|
||||||
|
{
|
||||||
|
switch ($type) {
|
||||||
|
case MMDBType::STRING:
|
||||||
|
$payload = $value;
|
||||||
|
return self::controlAndSize($type, strlen($payload)) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::BYTES:
|
||||||
|
$payload = $value;
|
||||||
|
return self::controlAndSize($type, strlen($payload)) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::DOUBLE:
|
||||||
|
$payload = pack('E', $value); // 8-byte big-endian double
|
||||||
|
return self::controlAndSize($type, 8) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::FLOAT:
|
||||||
|
$payload = pack('G', $value); // 4-byte big-endian float
|
||||||
|
return self::controlAndSize($type, 4) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::UINT16:
|
||||||
|
$payload = pack('n', $value & 0xFFFF);
|
||||||
|
return self::controlAndSize($type, 2) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::UINT32:
|
||||||
|
$payload = pack('N', $value & 0xFFFFFFFF);
|
||||||
|
return self::controlAndSize($type, 4) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::INT32:
|
||||||
|
$payload = pack('N', $value & 0xFFFFFFFF);
|
||||||
|
return self::controlAndSize($type, 4) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::UINT64:
|
||||||
|
$payload = pack('J', $value); // 8-byte big-endian uint64
|
||||||
|
return self::controlAndSize($type, 8) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::UINT128:
|
||||||
|
$payload = self::to16Bytes($value);
|
||||||
|
return self::controlAndSize($type, 16) . $payload;
|
||||||
|
|
||||||
|
case MMDBType::BOOLEAN:
|
||||||
|
// Boolean has no payload; the value lives in the size field.
|
||||||
|
return self::controlAndSize($type, $value ? 1 : 0);
|
||||||
|
|
||||||
|
case MMDBType::MAP:
|
||||||
|
$out = self::controlAndSize($type, count($value));
|
||||||
|
foreach ($value as $k => $v) {
|
||||||
|
$out .= self::encodeTyped(MMDBType::STRING, (string) $k);
|
||||||
|
$out .= self::encode($v);
|
||||||
|
}
|
||||||
|
return $out;
|
||||||
|
|
||||||
|
case MMDBType::ARRAY_T:
|
||||||
|
$out = self::controlAndSize($type, count($value));
|
||||||
|
foreach ($value as $v) {
|
||||||
|
$out .= self::encode($v);
|
||||||
|
}
|
||||||
|
return $out;
|
||||||
|
|
||||||
|
default:
|
||||||
|
throw new InvalidArgumentException("Unsupported MMDB type: $type");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private static function to16Bytes($value): string
|
||||||
|
{
|
||||||
|
if (is_string($value) && preg_match('/^0x[0-9a-fA-F]+$/', $value)) {
|
||||||
|
$hex = str_pad(substr($value, 2), 32, '0', STR_PAD_LEFT);
|
||||||
|
return hex2bin($hex);
|
||||||
|
}
|
||||||
|
// Treat as a (<=64-bit) PHP int, zero-padded to 128 bits.
|
||||||
|
return str_repeat("\x00", 8) . pack('J', (int) $value);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Build the control byte (+ extra size bytes, + extended-type byte)
|
||||||
|
* for a given MMDB type and payload size, per the spec's control
|
||||||
|
* byte format.
|
||||||
|
*/
|
||||||
|
private static function controlAndSize(int $type, int $size): string
|
||||||
|
{
|
||||||
|
$typeBits = $type <= 7 ? $type : 0; // 0 = "look at next byte for real type"
|
||||||
|
$out = '';
|
||||||
|
|
||||||
|
if ($size < 29) {
|
||||||
|
$out .= chr(($typeBits << 5) | $size);
|
||||||
|
} elseif ($size < 285) {
|
||||||
|
$out .= chr(($typeBits << 5) | 29);
|
||||||
|
$out .= chr($size - 29);
|
||||||
|
} elseif ($size < 65821) {
|
||||||
|
$out .= chr(($typeBits << 5) | 30);
|
||||||
|
$out .= substr(pack('N', $size - 285), 2, 2); // 2-byte BE
|
||||||
|
} else {
|
||||||
|
$out .= chr(($typeBits << 5) | 31);
|
||||||
|
$out .= substr(pack('N', $size - 65821), 1, 3); // 3-byte BE
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($type > 7) {
|
||||||
|
$out .= chr($type - 7); // extended type byte
|
||||||
|
}
|
||||||
|
|
||||||
|
return $out;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// Binary trie used to build the MMDB search tree
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
final class MMDBTree
|
||||||
|
{
|
||||||
|
/** @var array<int, array{0:mixed,1:mixed}> node index => [left, right] */
|
||||||
|
private $nodes = [[null, null]];
|
||||||
|
|
||||||
|
public function nodeCount(): int
|
||||||
|
{
|
||||||
|
return count($this->nodes);
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int[] $bits 0/1 array, the path from the root
|
||||||
|
* @param int $prefixLen how many bits of $bits to actually use
|
||||||
|
* @param int $dataOffset byte offset of this record's value in the data section
|
||||||
|
*/
|
||||||
|
public function insert(array $bits, int $prefixLen, int $dataOffset): void
|
||||||
|
{
|
||||||
|
$current = 0;
|
||||||
|
for ($i = 0; $i < $prefixLen; $i++) {
|
||||||
|
$bit = $bits[$i];
|
||||||
|
$isLast = ($i === $prefixLen - 1);
|
||||||
|
$rec = $this->nodes[$current][$bit];
|
||||||
|
|
||||||
|
if ($isLast) {
|
||||||
|
$this->nodes[$current][$bit] = ['data' => $dataOffset];
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($rec === null) {
|
||||||
|
$newIdx = count($this->nodes);
|
||||||
|
$this->nodes[] = [null, null];
|
||||||
|
$this->nodes[$current][$bit] = $newIdx;
|
||||||
|
$current = $newIdx;
|
||||||
|
} elseif (is_array($rec)) {
|
||||||
|
// A less-specific network already terminates here; split it
|
||||||
|
// so both children inherit its data, then keep descending.
|
||||||
|
$newIdx = count($this->nodes);
|
||||||
|
$this->nodes[] = [$rec, $rec];
|
||||||
|
$this->nodes[$current][$bit] = $newIdx;
|
||||||
|
$current = $newIdx;
|
||||||
|
} else {
|
||||||
|
$current = $rec;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Serialize the tree to bytes using the smallest record size (24, 28,
|
||||||
|
* or 32 bits) that fits every record value.
|
||||||
|
*
|
||||||
|
* @return array{0:string,1:int} [tree bytes, record size used]
|
||||||
|
*/
|
||||||
|
public function serialize(int $dataSectionLength): array
|
||||||
|
{
|
||||||
|
$nodeCount = count($this->nodes);
|
||||||
|
$maxDataPointer = $dataSectionLength + $nodeCount + 16;
|
||||||
|
$maxValue = max($nodeCount, $maxDataPointer);
|
||||||
|
|
||||||
|
if ($maxValue <= 0xFFFFFF) {
|
||||||
|
$recordSize = 24;
|
||||||
|
} elseif ($maxValue <= 0xFFFFFFF) {
|
||||||
|
$recordSize = 28;
|
||||||
|
} elseif ($maxValue <= 0xFFFFFFFF) {
|
||||||
|
$recordSize = 32;
|
||||||
|
} else {
|
||||||
|
throw new RuntimeException('Database too large for a 32-bit record size.');
|
||||||
|
}
|
||||||
|
|
||||||
|
$out = '';
|
||||||
|
foreach ($this->nodes as $node) {
|
||||||
|
$left = $this->recordValue($node[0], $nodeCount);
|
||||||
|
$right = $this->recordValue($node[1], $nodeCount);
|
||||||
|
$out .= $this->packPair($left, $right, $recordSize);
|
||||||
|
}
|
||||||
|
|
||||||
|
return [$out, $recordSize];
|
||||||
|
}
|
||||||
|
|
||||||
|
private function recordValue($rec, int $nodeCount): int
|
||||||
|
{
|
||||||
|
if ($rec === null) {
|
||||||
|
return $nodeCount; // "not found" sentinel
|
||||||
|
}
|
||||||
|
if (is_array($rec)) {
|
||||||
|
return $rec['data'] + $nodeCount + 16;
|
||||||
|
}
|
||||||
|
return $rec; // pointer to another node
|
||||||
|
}
|
||||||
|
|
||||||
|
private function packPair(int $left, int $right, int $recordSize): string
|
||||||
|
{
|
||||||
|
if ($recordSize === 24) {
|
||||||
|
return substr(pack('N', $left), 1, 3) . substr(pack('N', $right), 1, 3);
|
||||||
|
}
|
||||||
|
if ($recordSize === 32) {
|
||||||
|
return pack('N', $left) . pack('N', $right);
|
||||||
|
}
|
||||||
|
// 28-bit: 7 bytes total, per the MMDB spec's packing scheme.
|
||||||
|
$b0 = ($left >> 20) & 0xFF;
|
||||||
|
$b1 = ($left >> 12) & 0xFF;
|
||||||
|
$b2 = ($left >> 4) & 0xFF;
|
||||||
|
$b3 = (($left & 0xF) << 4) | (($right >> 24) & 0xF);
|
||||||
|
$b4 = ($right >> 16) & 0xFF;
|
||||||
|
$b5 = ($right >> 8) & 0xFF;
|
||||||
|
$b6 = $right & 0xFF;
|
||||||
|
return chr($b0) . chr($b1) . chr($b2) . chr($b3) . chr($b4) . chr($b5) . chr($b6);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
// Top-level writer: ties the tree, data section and metadata together
|
||||||
|
// ---------------------------------------------------------------------
|
||||||
|
final class MMDBWriter
|
||||||
|
{
|
||||||
|
private $ipVersion;
|
||||||
|
private $totalDepth;
|
||||||
|
private $tree;
|
||||||
|
private $dataSection = '';
|
||||||
|
private $dataCache = [];
|
||||||
|
private $databaseType;
|
||||||
|
private $languages;
|
||||||
|
private $description;
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param int $ipVersion 4 (IPv4-only tree) or 6 (dual-stack tree)
|
||||||
|
* @param string $databaseType free-form name, e.g. "GeoIP-Custom"
|
||||||
|
* @param string[] $languages language codes present in $description
|
||||||
|
* @param array $description language code => human description
|
||||||
|
*/
|
||||||
|
public function __construct(
|
||||||
|
int $ipVersion = 6,
|
||||||
|
string $databaseType = 'Custom',
|
||||||
|
array $languages = ['en'],
|
||||||
|
array $description = ['en' => 'Custom MMDB database']
|
||||||
|
) {
|
||||||
|
if ($ipVersion !== 4 && $ipVersion !== 6) {
|
||||||
|
throw new InvalidArgumentException('ipVersion must be 4 or 6');
|
||||||
|
}
|
||||||
|
$this->ipVersion = $ipVersion;
|
||||||
|
$this->totalDepth = $ipVersion === 4 ? 32 : 128;
|
||||||
|
$this->tree = new MMDBTree();
|
||||||
|
$this->databaseType = $databaseType;
|
||||||
|
$this->languages = $languages;
|
||||||
|
$this->description = $description;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* @param string $cidr e.g. "1.2.3.0/24", "2001:db8::/32", or a bare IP
|
||||||
|
* @param mixed $data scalar/array/MMDBValue data to associate with the network
|
||||||
|
*/
|
||||||
|
public function addRecord(string $cidr, $data): void
|
||||||
|
{
|
||||||
|
[$ip, $prefixLen] = $this->parseCidr($cidr);
|
||||||
|
[$bits, $treePrefixLen] = $this->ipToTreeBits($ip, $prefixLen);
|
||||||
|
$offset = $this->internData($data);
|
||||||
|
$this->tree->insert($bits, $treePrefixLen, $offset);
|
||||||
|
}
|
||||||
|
|
||||||
|
public function write(string $path): void
|
||||||
|
{
|
||||||
|
[$treeBytes, $recordSize] = $this->tree->serialize(strlen($this->dataSection));
|
||||||
|
|
||||||
|
$metadata = [
|
||||||
|
'node_count' => MMDBValue::uint32($this->tree->nodeCount()),
|
||||||
|
'record_size' => MMDBValue::uint16($recordSize),
|
||||||
|
'ip_version' => MMDBValue::uint16($this->ipVersion),
|
||||||
|
'database_type' => MMDBValue::string($this->databaseType),
|
||||||
|
'languages' => array_values($this->languages),
|
||||||
|
'binary_format_major_version' => MMDBValue::uint16(2),
|
||||||
|
'binary_format_minor_version' => MMDBValue::uint16(0),
|
||||||
|
'build_epoch' => MMDBValue::uint64(time()),
|
||||||
|
'description' => $this->description,
|
||||||
|
];
|
||||||
|
|
||||||
|
$metadataBytes = MMDBEncoder::encode(MMDBValue::map($metadata));
|
||||||
|
|
||||||
|
$separator = str_repeat("\x00", 16);
|
||||||
|
$marker = "\xAB\xCD\xEFMaxMind.com";
|
||||||
|
|
||||||
|
$blob = $treeBytes . $separator . $this->dataSection . $marker . $metadataBytes;
|
||||||
|
|
||||||
|
if (file_put_contents($path, $blob) === false) {
|
||||||
|
throw new RuntimeException("Failed to write $path");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
private function internData($data): int
|
||||||
|
{
|
||||||
|
$key = serialize($data);
|
||||||
|
if (isset($this->dataCache[$key])) {
|
||||||
|
return $this->dataCache[$key];
|
||||||
|
}
|
||||||
|
$offset = strlen($this->dataSection);
|
||||||
|
$this->dataSection .= MMDBEncoder::encode($data);
|
||||||
|
$this->dataCache[$key] = $offset;
|
||||||
|
return $offset;
|
||||||
|
}
|
||||||
|
|
||||||
|
private function parseCidr(string $cidr): array
|
||||||
|
{
|
||||||
|
if (strpos($cidr, '/') !== false) {
|
||||||
|
[$ip, $prefix] = explode('/', $cidr, 2);
|
||||||
|
return [$ip, (int) $prefix];
|
||||||
|
}
|
||||||
|
$isV4 = strpos($cidr, ':') === false;
|
||||||
|
return [$cidr, $isV4 ? 32 : 128];
|
||||||
|
}
|
||||||
|
|
||||||
|
private function ipToTreeBits(string $ip, int $prefixLen): array
|
||||||
|
{
|
||||||
|
$packed = @inet_pton($ip);
|
||||||
|
if ($packed === false) {
|
||||||
|
throw new InvalidArgumentException("Invalid IP address: $ip");
|
||||||
|
}
|
||||||
|
$isV4 = strlen($packed) === 4;
|
||||||
|
|
||||||
|
if ($isV4 && $this->totalDepth === 128) {
|
||||||
|
// Embed IPv4 networks under ::/96, as MaxMind's own dual-stack
|
||||||
|
// databases do.
|
||||||
|
$prefix = array_fill(0, 96, 0);
|
||||||
|
$treePrefixLen = 96 + $prefixLen;
|
||||||
|
} else {
|
||||||
|
$prefix = [];
|
||||||
|
$treePrefixLen = $prefixLen;
|
||||||
|
}
|
||||||
|
|
||||||
|
$bits = $prefix;
|
||||||
|
foreach (str_split($packed) as $byte) {
|
||||||
|
$b = ord($byte);
|
||||||
|
for ($i = 7; $i >= 0; $i--) {
|
||||||
|
$bits[] = ($b >> $i) & 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return [$bits, $treePrefixLen];
|
||||||
|
}
|
||||||
|
}
|
||||||
69
README.md
Normal file
69
README.md
Normal file
@@ -0,0 +1,69 @@
|
|||||||
|
# Cloudfish
|
||||||
|
Cloudfish (aims to be) a selfhosted Cloudflare replacement.
|
||||||
|
|
||||||
|
Right now, it only features an IP abuse MMDB database builder. You give it an IP and it will expose 3 optional values: `is_proxy`, `is_hosting` and `is_tor`.
|
||||||
|
|
||||||
|
To use the database, you can use this PHP code:
|
||||||
|
```php
|
||||||
|
<?php
|
||||||
|
require "geoip2.phar";
|
||||||
|
use MaxMind\Db\Reader as MmdbReader;
|
||||||
|
|
||||||
|
$cf = new MmdbReader("cloudfish.mmdb");
|
||||||
|
$cf_lookup = $reader_abuse->get("45.80.201.66");
|
||||||
|
|
||||||
|
print_r($cf_lookup); // ["is_hosting" => true]
|
||||||
|
```
|
||||||
|
|
||||||
|
**Warning**: Any field may be missing. Fields that are set are always set to `true`.
|
||||||
|
|
||||||
|
# Data sources
|
||||||
|
|
||||||
|
## Proxy detection ([firehol](https://iplists.firehol.org/))
|
||||||
|
- [firehol_anonymous](https://iplists.firehol.org/?ipset=firehol_anonymous)
|
||||||
|
- [spamhaus_drop](https://iplists.firehol.org/?ipset=spamhaus_drop)
|
||||||
|
- [dshield_30d](https://iplists.firehol.org/?ipset=dshield_30d)
|
||||||
|
- [greensnow](https://iplists.firehol.org/?ipset=greensnow)
|
||||||
|
- [blocklist_de](https://iplists.firehol.org/?ipset=blocklist_de)
|
||||||
|
- [bruteforceblocker](https://iplists.firehol.org/?ipset=bruteforceblocker)
|
||||||
|
- [ciarmy](https://iplists.firehol.org/?ipset=ciarmy)
|
||||||
|
- [myip](https://iplists.firehol.org/?ipset=myip)
|
||||||
|
- [vxvault](https://iplists.firehol.org/?ipset=vxvault)
|
||||||
|
- [blocklist_net_ua](https://iplists.firehol.org/?ipset=blocklist_net_ua)
|
||||||
|
- [botscout_30d](https://iplists.firehol.org/?ipset=botscout_30d)
|
||||||
|
- [cybercrime](https://iplists.firehol.org/?ipset=cybercrime)
|
||||||
|
- [iblocklist_hijacked](https://iplists.firehol.org/?ipset=iblocklist_hijacked)
|
||||||
|
- [iblocklist_spyware](https://iplists.firehol.org/?ipset=iblocklist_spyware)
|
||||||
|
- [iblocklist_webexploit](https://iplists.firehol.org/?ipset=iblocklist_webexploit)
|
||||||
|
|
||||||
|
## VPN providers (inserted in db as "is_proxy")
|
||||||
|
- [Tunnelbear](https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt) ([source](https://github.com/tn3w/TunnelBear-IPs))
|
||||||
|
- [ProtonVPN](https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/refs/heads/master/protonvpn_ips.txt) ([source](https://github.com/tn3w/ProtonVPN-IPs))
|
||||||
|
- [Windscribe](https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt) ([source](https://github.com/tn3w/Windscribe-IPs))
|
||||||
|
- pia/proton/apple/mullvad [ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv4.txt), [ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/vpn/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/))
|
||||||
|
|
||||||
|
## Hosting detection
|
||||||
|
- [datacenter-ipv4](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt), [datacenter-ipv6](https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt) ([source](https://github.com/X4BNet/lists_vpn/))
|
||||||
|
- [CDN list](https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst) ([source](https://github.com/mansourjabin/cdn-ip-database))
|
||||||
|
|
||||||
|
## Tor detection
|
||||||
|
- [tor exit node list](https://openinternet.io/tor/tor-exit-list.txt) ([source](https://openinternet.io))
|
||||||
|
- [dm_tor](https://iplists.firehol.org/?ipset=dm_tor)
|
||||||
|
|
||||||
|
# Recommendation
|
||||||
|
I recommend these additional MMDB databases to complement Cloudfish.
|
||||||
|
|
||||||
|
## ASN detection
|
||||||
|
- [GeoLite2-ASN](https://git.io/GeoLite2-ASN.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb))
|
||||||
|
|
||||||
|
## Country/city detection
|
||||||
|
- [GeoLite2-City](https://git.io/GeoLite2-City.mmdb) ([source](https://github.com/P3TERX/GeoLite.mmdb))
|
||||||
|
|
||||||
|
# Try it
|
||||||
|
I wrote a simple IP lookup script. It's available at `ip.lolcat.ca`, `ip4.lolcat.ca` and `ip6.lolcat.ca`.
|
||||||
|
|
||||||
|
### Disclaimer
|
||||||
|
I transpiled MMDBWriter.php from it's original Go implementation using Claude. I'll make my own when I have more time, no idea if the filesize could be reduced.
|
||||||
|
|
||||||
|
# License
|
||||||
|
AGPLv3, make sure to credit all blocklists used, they all have their own fuckass licenses.
|
||||||
185
mmdb.php
Normal file
185
mmdb.php
Normal file
@@ -0,0 +1,185 @@
|
|||||||
|
<?php
|
||||||
|
|
||||||
|
$s = new mmdb_creator();
|
||||||
|
|
||||||
|
class mmdb_creator{
|
||||||
|
|
||||||
|
public function __construct(){
|
||||||
|
|
||||||
|
//
|
||||||
|
// Import database
|
||||||
|
//
|
||||||
|
if(!file_exists("data")){ mkdir("data"); }
|
||||||
|
chdir("data");
|
||||||
|
if(!file_exists("blocklist-ipsets")){
|
||||||
|
|
||||||
|
echo "Downloading firehol lists\n";
|
||||||
|
shell_exec("git clone https://github.com/firehol/blocklist-ipsets");
|
||||||
|
}else{
|
||||||
|
|
||||||
|
echo "Updating firehol lists\n";
|
||||||
|
chdir("blocklist-ipsets");
|
||||||
|
echo shell_exec("git pull");
|
||||||
|
chdir("..");
|
||||||
|
}
|
||||||
|
|
||||||
|
echo "Downloading Tunnelbear list\n";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/tn3w/TunnelBear-IPs/refs/heads/master/tunnelbear_ips.txt", "tunnelbear_ips.txt");
|
||||||
|
|
||||||
|
echo "Downloading ProtonVPN list\n";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/tn3w/ProtonVPN-IPs/refs/heads/master/protonvpn_ips.txt", "protonvpn_ips.txt");
|
||||||
|
|
||||||
|
echo "Downloading Windscribe list\n";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "windscribe_ips.txt");
|
||||||
|
|
||||||
|
echo "Downloading pia/proton/apple/mullvad lists... (";
|
||||||
|
if(!file_exists("x4bnet-vpn")){ mkdir("x4bnet-vpn"); }
|
||||||
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv4.txt"); echo "1,";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/tn3w/Windscribe-IPs/refs/heads/master/windscribe_ips.txt", "x4bnet-vpn/ipv6.txt"); echo "2)\n";
|
||||||
|
|
||||||
|
echo "Downloading hosting lists... (";
|
||||||
|
if(!file_exists("x4bnet-hosting")){ mkdir("x4bnet-hosting"); }
|
||||||
|
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv4.txt", "x4bnet-hosting/ipv4.txt"); echo "1,";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/X4BNet/lists_vpn/refs/heads/main/output/datacenter/ipv6.txt", "x4bnet-hosting/ipv6.txt"); echo "2)\n";
|
||||||
|
|
||||||
|
echo "Downloading CDN list...\n";
|
||||||
|
$this->dl("https://raw.githubusercontent.com/mansourjabin/cdn-ip-database/refs/heads/main/data/cdn.lst", "cdn.lst");
|
||||||
|
|
||||||
|
echo "Downloading Tor exit node list...\n";
|
||||||
|
$this->dl("https://openinternet.io/tor/tor-exit-list.txt", "tor-exit-list.txt");
|
||||||
|
|
||||||
|
chdir("..");
|
||||||
|
echo "Done! Creating MMDB database...\n";
|
||||||
|
|
||||||
|
//
|
||||||
|
// Construct database
|
||||||
|
//
|
||||||
|
$this->ip_list = [];
|
||||||
|
|
||||||
|
// import miscelaneous lists
|
||||||
|
$this->import_firehol_lists(
|
||||||
|
[
|
||||||
|
"proxy" => [ // tn3w's
|
||||||
|
"tunnelbear_ips.txt",
|
||||||
|
"protonvpn_ips.txt",
|
||||||
|
"windscribe_ips.txt"
|
||||||
|
],
|
||||||
|
"tor" => [
|
||||||
|
"tor-exit-list.txt"
|
||||||
|
],
|
||||||
|
"hosting" => [
|
||||||
|
"cdn.lst" // mansourjabin's
|
||||||
|
]
|
||||||
|
],
|
||||||
|
""
|
||||||
|
);
|
||||||
|
|
||||||
|
// import X4BNet's lists
|
||||||
|
$this->import_firehol_lists(
|
||||||
|
[
|
||||||
|
"proxy" => [
|
||||||
|
"ipv4.txt",
|
||||||
|
"ipv6.txt",
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"x4bnet-vpn/"
|
||||||
|
);
|
||||||
|
|
||||||
|
$this->import_firehol_lists(
|
||||||
|
[
|
||||||
|
"hosting" => [
|
||||||
|
"ipv4.txt",
|
||||||
|
"ipv6.txt",
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"x4bnet-hosting/"
|
||||||
|
);
|
||||||
|
|
||||||
|
// import firehol lists
|
||||||
|
$this->import_firehol_lists(
|
||||||
|
[
|
||||||
|
"proxy" => [
|
||||||
|
"firehol_anonymous.netset",
|
||||||
|
"spamhaus_drop.netset",
|
||||||
|
"dshield_30d.netset",
|
||||||
|
"greensnow.ipset",
|
||||||
|
"blocklist_de.ipset",
|
||||||
|
"bruteforceblocker.ipset",
|
||||||
|
"ciarmy.ipset",
|
||||||
|
"stopforumspam_90d.ipset",
|
||||||
|
"myip.ipset",
|
||||||
|
"vxvault.ipset",
|
||||||
|
"blocklist_net_ua.ipset",
|
||||||
|
"botscout_30d.ipset",
|
||||||
|
"cybercure.ipset",
|
||||||
|
"cybercrime.ipset"
|
||||||
|
],
|
||||||
|
"tor" => [ // + tor, the other tor list misses ipv4s with ipv6 addresses
|
||||||
|
"dm_tor.ipset"
|
||||||
|
]
|
||||||
|
],
|
||||||
|
"blocklist-ipsets/"
|
||||||
|
);
|
||||||
|
|
||||||
|
require __DIR__ . '/MMDBWriter.php';
|
||||||
|
|
||||||
|
$w = new MMDBWriter(
|
||||||
|
6, // accepts IPv4 and IPv6 networks
|
||||||
|
'GeoIP-Custom', // database_type
|
||||||
|
['en'], // languages
|
||||||
|
['en' => 'Cloudfish']
|
||||||
|
);
|
||||||
|
|
||||||
|
echo "Generating mmdb file\n";
|
||||||
|
|
||||||
|
foreach($this->ip_list as $ip => $data){
|
||||||
|
|
||||||
|
$w->addRecord($ip, [
|
||||||
|
//"country" => "US",
|
||||||
|
//"city" => "Ashburn",
|
||||||
|
...(isset($data["proxy"]) ? ["is_proxy" => true] : []),
|
||||||
|
...(isset($data["tor"]) ? ["is_tor" => true] : []),
|
||||||
|
...(isset($data["hosting"]) ? ["is_hosting" => true] : [])
|
||||||
|
//"asn" => MMDBValue::uint32(15169),
|
||||||
|
]);
|
||||||
|
}
|
||||||
|
|
||||||
|
$w->write(__DIR__ . '/output.mmdb');
|
||||||
|
|
||||||
|
echo "done\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
public function import_firehol_lists($targets_assoc, $path = "blocklist-ipsets/"){
|
||||||
|
|
||||||
|
foreach($targets_assoc as $key => $targets){
|
||||||
|
|
||||||
|
foreach($targets as $target){
|
||||||
|
|
||||||
|
$path_use = "data/{$path}{$target}";
|
||||||
|
echo "Reading {$path_use}\n";
|
||||||
|
|
||||||
|
$lines = explode("\n", file_get_contents($path_use));
|
||||||
|
|
||||||
|
foreach($lines as $line){
|
||||||
|
|
||||||
|
$line = trim($line);
|
||||||
|
if(
|
||||||
|
strlen($line) === 0 ||
|
||||||
|
$line[0] == "#"
|
||||||
|
){
|
||||||
|
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
$this->ip_list[$line][$key] = true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
public function dl($url, $path){
|
||||||
|
|
||||||
|
$data = file_get_contents($url);
|
||||||
|
file_put_contents($path, $data);
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user