add user-agent and ja4 as report-fields

Author: superstes

src/riskdb/api/main.py

@@ -10,7 +10,6 @@
 from socket import gethostname
 from sys import path as sys_path
 from re import sub as regex_replace
-from re import compile as regex_compile
 from json import dumps as json_dumps
 from json import loads as json_loads
 from ipaddress import ip_address, ip_network
@@ -23,7 +22,7 @@
 from oxl_utils.valid.net import valid_public_ip, valid_asn, get_ipv
 
 from riskdb.config import BUILD_DIR, KIND_FILES, REPORT_DIR, RISK_CATEGORIES, NET_SIZE, USER_TOKENS, \
-    EXCLUDE_NETS_IP4, EXCLUDE_NETS_IP6
+    EXCLUDE_NETS_IP4, EXCLUDE_NETS_IP6, JA4_REGEX
 
 app = Flask('risk-db')
 RISKY_DB_FILE = {
@@ -35,7 +34,6 @@
     4: BUILD_DIR / 'risk_net4_med.json',
     6: BUILD_DIR / 'risk_net6_med.json',
 }
-JA4_REGEX = regex_compile(r'^[tqd](13|12|11|10|s3|s2|00)[di][a-f0-9]{4}[a-z0-9]{2}_[a-f0-9]{12}_[a-f0-9]{12}$')
 
 report_lock = Lock()
 

src/riskdb/builder/obj/report.py

@@ -1,5 +1,6 @@
 # pylint: disable=R0801,R0902
 
+from riskdb.config import JA4_REGEX
 from riskdb.builder.obj.reporter import Reporter, ANONYMOUS
 
 
@@ -10,9 +11,12 @@ def __init__(self, raw: dict, reporters: list[Reporter]):
         self.ip_anonymized = raw['an'] == 1 if 'an' in raw else False
         self.category = raw['cat']
         self.comment = raw['cmt']
+        self.by_ip = raw['by']
         self.user_agent = raw.get('ua', '')
         self.fingerprint_ja4 = raw.get('ja4', '')
-        self.by_ip = raw['by']
+
+        if JA4_REGEX.match(self.fingerprint_ja4) is None:
+            self.fingerprint_ja4 = ''
 
         self.reporter = self._init_reporter(token=raw['token'], reporters=reporters)
         self.legitimacy = self._init_legitimacy()
@@ -31,7 +35,7 @@ def _init_legitimacy(self) -> int:
             l += 5
             l += self.reporter.reputation
 
-        if len(self.comment) > 5:
+        if len(self.comment) > 5 or len(self.user_agent) > 5 or self.fingerprint_ja4 != '':
             l += 1
 
         # todo: extend logic

src/riskdb/config.py

@@ -1,6 +1,7 @@
 from os import environ
 from pathlib import Path
 from ipaddress import ip_network
+from re import compile as regex_compile
 
 MODE_TEST = environ.get('RISKDB_TEST', '0')
 
@@ -36,3 +37,4 @@
     '2001:4860:4860::/64'  # google dns
 ]
 EXCLUDE_NETS_IP6 = [ip_network(n) for n in _EXCLUDE_NETS_IP6]
+JA4_REGEX = regex_compile(r'^[tqd](13|12|11|10|s3|s2|00)[di][a-f0-9]{4}[a-z0-9]{2}_[a-f0-9]{12}_[a-f0-9]{12}$')