add vpn-ip-lookups (fix #9)

Author: superstes

src/riskdb/builder/config.py

@@ -16,6 +16,12 @@
 REPORT_COOLDOWN = 10  # sec
 REPORT_DAYS = 30  # sliding window
 TOR_EXIT_NODE_LIST = 'https://check.torproject.org/torbulkexitlist'
+VPN_URLS = {
+    'apple': 'https://mask-api.icloud.com/egress-ip-ranges.csv',
+    'mullvad': 'https://api.mullvad.net/app/v1/relays',
+    'pia': 'https://raw.githubusercontent.com/Lars-/PIA-servers/refs/heads/master/export.csv',
+}
+DOWNLOAD_TIMEOUT = 5  # do not wait forever if something changed (p.e. firewall blocks download)
 
 PTR_LOOKUP_THREADS = 50
 PTR_CACHE_DAYS = 30

src/riskdb/builder/enrich_data.py

@@ -12,12 +12,14 @@
 from oxl_utils.ps import wait_for_threads
 from dns.resolver import Resolver, NoAnswer, NXDOMAIN, LifetimeTimeout, NoNameservers
 from dns.exception import SyntaxError as DNSSyntaxError
+from netaddr.ip import IPNetwork, cidr_merge
+from netaddr.core import AddrFormatError, AddrConversionError
 
 from riskdb.builder.util import log
 from riskdb.config import KIND_FILES
 from riskdb.builder.load_reports import FileLoader
 from riskdb.builder.config import CACHE_FILE_PTR, ASN_JSON_FILE, TOR_EXIT_NODE_LIST, PTR_LOOKUP_THREADS, \
-    PTR_CACHE_DAYS, PTR_STATUS_COUNT, PTR_NAMESERVERS, PTR_MAX_QUERY_RETRIES
+    PTR_CACHE_DAYS, PTR_STATUS_COUNT, PTR_NAMESERVERS, PTR_MAX_QUERY_RETRIES, VPN_URLS, DOWNLOAD_TIMEOUT
 
 now = int(time())
 ptr_cache_lock = Lock()
@@ -35,6 +37,35 @@ def load_lookup_list_asn() -> dict:
         return json_loads(f.read())
 
 
+def load_vpn_ips() -> list[IPNetwork]:
+    vpn_file = '/tmp/vpns.txt'
+    os_shell(
+        f"wget -q --connect-timeout={DOWNLOAD_TIMEOUT} -O {vpn_file}.tmp {VPN_URLS['apple']} &&"
+        f"tail -n +2 {vpn_file}.tmp | cut -d ',' -f1 | sort | uniq > {vpn_file}"
+    )
+    os_shell(
+        f"wget -q --connect-timeout={DOWNLOAD_TIMEOUT} -O {vpn_file}.tmp {VPN_URLS['mullvad']} &&"
+        f"jq -r '.[][] | try .[] | try .ipv4_addr_in' < {vpn_file}.tmp | sort | uniq >> {vpn_file} &&"
+        f"jq -r '.[][] | try .[] | try .ipv6_addr_in' < {vpn_file}.tmp | sort | uniq >> {vpn_file}"
+    )
+    os_shell(
+        f"wget -q --connect-timeout={DOWNLOAD_TIMEOUT} -O {vpn_file}.tmp {VPN_URLS['pia']} &&"
+        f"tail -n +2 {vpn_file}.tmp | cut -d ',' -f1 | sort | uniq >> {vpn_file}"
+    )
+
+    # NOTE: using netaddr to allow for subnet-merging where possible
+    nets = []
+    with open(vpn_file, 'r', encoding='utf-8') as f:
+        for net in f.readlines():
+            try:
+                nets.append(IPNetwork(net.strip()))
+
+            except (AddrFormatError, AddrConversionError, ValueError):
+                continue
+
+    return cidr_merge(nets)
+
+
 def load_lookup_lists() -> dict:
     lookup_lists = {}
     tor_exit_node_file = '/tmp/tor_exit_nodes.txt'
@@ -51,6 +82,7 @@ def load_lookup_lists() -> dict:
             except AddressValueError:
                 continue
 
+    lookup_lists['vpns'] = load_vpn_ips()
     lookup_lists['asn'] = load_lookup_list_asn()
 
     # creation of these files has yet to be automated

src/riskdb/builder/kind/crawler.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 17858
 18403
 32934

src/riskdb/builder/kind/education.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 29484
 680
 137

src/riskdb/builder/kind/hosting.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 559
 834
 917

src/riskdb/builder/kind/isp.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 209
 701
 852

src/riskdb/builder/kind/scanner.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 174
 6939
 10439

src/riskdb/builder/kind/vpn.txt

@@ -1,3 +1,4 @@
+# for dynamic/full lists refer to: https://github.com/O-X-L/risk-db-lists/tree/main/asn
 62371
 141039
 147049

src/riskdb/builder/obj/ip.py

@@ -1,3 +1,5 @@
+from netaddr import IPAddress as NetaddrIPAddress
+
 from riskdb.config import RISK_CATEGORIES
 
 from riskdb.builder.obj.asn import ASN
@@ -84,6 +86,11 @@ def _init_kind(self, lookup_lists: dict) -> list[str]:
         if self.ip in lookup_lists['tor']:
             k.append('tor')
 
+        ip_obj = NetaddrIPAddress(self.ip)
+        for net in lookup_lists['vpns']:
+            if ip_obj in net:
+                k.append('vpn')
+
         if self.ptr is None:
             return k