add asn-csv-lists

Author: superstes

src/riskdb/builder/enrich_data.py

@@ -29,6 +29,12 @@
 dns_resolver.nameservers = PTR_NAMESERVERS
 
 
+def load_lookup_list_asn() -> dict:
+    # source: https://github.com/O-X-L/geoip-asn
+    with open(ASN_JSON_FILE, 'r', encoding='utf-8') as f:
+        return json_loads(f.read())
+
+
 def load_lookup_lists() -> dict:
     lookup_lists = {}
     tor_exit_node_file = '/tmp/tor_exit_nodes.txt'
@@ -45,9 +51,7 @@ def load_lookup_lists() -> dict:
             except AddressValueError:
                 continue
 
-    # source: https://github.com/O-X-L/geoip-asn
-    with open(ASN_JSON_FILE, 'r', encoding='utf-8') as f:
-        lookup_lists['asn'] = json_loads(f.read())
+    lookup_lists['asn'] = load_lookup_list_asn()
 
     # creation of these files has yet to be automated
     for k, v in KIND_FILES.items():

src/riskdb/lister/kind.py

@@ -6,14 +6,14 @@
 from datetime import datetime, timedelta
 
 from riskdb.builder.util import log
-from riskdb.lister.util import write_list
+from riskdb.builder.obj.ip import IP_KINDS
+from riskdb.builder.obj.asn import ASN_KINDS
 from riskdb.builder.build import build_objects
+from riskdb.lister.config import LIST_START_DATE
 from riskdb.builder.load_reports import FileLoader
-from riskdb.builder.enrich_data import load_lookup_lists, get_ptrs_from_cache
-from riskdb.builder.obj.asn import ASN_KINDS
 from riskdb.builder.obj.network import KIND_IP_INHERITANCE
-from riskdb.builder.obj.ip import IP_KINDS
-from riskdb.lister.config import LIST_START_DATE
+from riskdb.lister.util import write_list, get_asn_organisation
+from riskdb.builder.enrich_data import load_lookup_lists, get_ptrs_from_cache
 
 END_DATE = datetime.now()
 
@@ -111,3 +111,11 @@ def list_kind(tmp_dir: Path):
             l = list(l)
             l.sort()
             write_list(d=t, file=f'kind_{k}.txt', lines=l, tmp_dir=tmp_dir)
+
+            if t == 'asn':
+                csv = ['ASN,Organization']
+                csv.extend([
+                    f"{k},{get_asn_organisation(lookup_lists['asn'], k)}"
+                    for k in l
+                ])
+                write_list(d=t, file=f'kind_{k}.csv', lines=csv, tmp_dir=tmp_dir)

src/riskdb/lister/most_reported.py

@@ -7,9 +7,10 @@
 
 from riskdb.config import NET_SIZE
 from riskdb.builder.util import log
-from riskdb.lister.util import write_list
 from riskdb.builder.load_reports import FileLoader
 from riskdb.lister.config import LIST_STATUS_COUNT
+from riskdb.builder.enrich_data import load_lookup_list_asn
+from riskdb.lister.util import write_list, get_asn_organisation
 from riskdb.builder.config import ASN_MMDB_FILE_IP4, ASN_MMDB_FILE_IP6
 
 TOP_N = {
@@ -36,6 +37,8 @@ def list_most_reported(tmp_dir: Path):
 
     log('Building Most-Reported Lists')
 
+    asn_metadata = load_lookup_list_asn()
+
     ir = 0
     with mmdb_database(ASN_MMDB_FILE_IP4) as asn_db_ip4, mmdb_database(ASN_MMDB_FILE_IP6) as asn_db_ip6:
         for r in FileLoader(sliding_window=False):
@@ -117,7 +120,27 @@ def list_most_reported(tmp_dir: Path):
                 a = '_ips_6'
 
             write_list(d=t, file=f'top_{top_n}{a}.txt', lines=l[:top_n], tmp_dir=tmp_dir)
+
+            if t == 'asn':
+                csv = ['ASN,Report count,Organization']
+                csv.extend([
+                    f"{k},{v},{get_asn_organisation(asn_metadata, k)}"
+                    for k, v in dict(r[:top_n]).items()
+                ])
+
+            elif t == 'net':
+                csv = ['Network,Report count']
+
+            else:
+                csv = ['IP,Report count']
+
+            if t != 'asn':
+                csv.extend([
+                    f"{k},{v}"
+                    for k, v in dict(r[:top_n]).items()
+                ])
+
             write_list(
                 d=t, file=f'top_{top_n}{a}.csv', tmp_dir=tmp_dir,
-                lines=[f'{k},{v}' for k, v in dict(r[:top_n]).items()],
+                lines=csv,
             )

src/riskdb/lister/util.py

@@ -13,3 +13,19 @@ def write_list(d: str, file: str, lines: list[str], tmp_dir: Path):
 
             except TypeError:
                 f.write('\n'.join([str(l) for l in lines]))
+
+
+def get_asn_organisation(asn_metadata: dict, asn: int, csv: bool = True) -> str:
+    asn = str(asn)
+    if asn not in asn_metadata:
+        return ''
+
+    m = asn_metadata[asn]
+    if 'info' not in m:
+        return ''
+
+    org = m['info'].get('name', m['organization'].get('name', ''))
+    if csv:
+        org = org.replace(',', '')
+
+    return org