Improve mlflow detection

.github/workflows/unit-tests.yml

@@ -37,6 +37,7 @@ jobs:
           - test_whitelist.py
           - test_arp.py
           - test_blocking.py
+          - test_unblocker.py
           - test_flow_handler.py
           - test_horizontal_portscans.py
           - test_http_analyzer.py

.secrets.baseline

@@ -149,14 +149,14 @@
         "filename": "config/slips.yaml",
         "hashed_secret": "4cac50cee3ad8e462728e711eac3e670753d5016",
         "is_verified": false,
-        "line_number": 224
+        "line_number": 226
       },
       {
         "type": "Secret Keyword",
         "filename": "config/slips.yaml",
         "hashed_secret": "d033e22ae348aeb5660fc2140aec35850c4da997",
         "is_verified": false,
-        "line_number": 394
+        "line_number": 396
       }
     ],
     "dataset/test14-malicious-zeek-dir/http.log": [
@@ -7192,5 +7192,5 @@
       }
     ]
   },
-  "generated_at": "2025-02-13T22:47:52Z"
+  "generated_at": "2025-05-10T13:18:46Z"
 }

config/slips.yaml

@@ -26,7 +26,6 @@ parameters:
   # For 5 min
   # time_window_width : 300
   # For 1 hour
-  # time_window_width : 3600
   time_window_width: 3600
   # For 1 day
   # time_window_width = 86400
@@ -106,13 +105,12 @@ parameters:
   deletePrevdb: true
 
   # Set the label for all the flows that are being read.
-  # For now only normal and malware directly. No option for setting labels
-  # with a filter
+  # For now only Benign and Malicious (Capitalized)
   # The purpose is to be used in the training of ML models and to output
   # flows with labels for other tools.
-  # label: malicious
-  # label: unknown
-  label: normal
+  # label: Malicious
+  # label: Benign
+  label: Benign
   # If Zeek files are rotated or not to avoid running out of disk.
   # Zeek rotation is enabled by default when using an interface,
   # which means Slips will delete all Zeek log files after 1 day
@@ -214,7 +212,10 @@ flowmldetection:
   # training the models, to test in unknown data.
   # You should have trained at least once with 'Normal' data and once with
   # 'Malicious' data in order for the test to work.
-  mode: test
+  mode: train
+  # creates an extra log file called training.log/testing.log in the
+  # ouptput dir with performance metrics depending on the mode.
+  create_performance_metrics_log_files: True
 
 #############################
 virustotal:

docs/architecture.md

@@ -12,7 +12,7 @@ Slips is heavily based on the Zeek monitoring tool as input tool for packets fro
 Figure 1 shows how the data is analyzed by Slips.
 As we can see, Slips internally uses <a href="https://zeek.org/">Zeek</a>, an
 open source network security monitoring tool. Slips divides flows into profiles and
-each profile into a timewindows.
+each profile into a timewindows, timewindows are numbered from 1 to infinity.
 Slips runs detection modules on each flow and stores all evidence,
 alerts and features in an appropriate profile structure.
 All profile info, performed detections, profiles and timewindows' data,

install/requirements.txt

@@ -1,4 +1,4 @@
-maxminddb==2.6.3
+maxminddb==2.7.0
 numpy==1.26.4
 watchdog==5.0.0
 redis==5.2.1
@@ -10,7 +10,7 @@ stix2==3.0.1
 certifi==2025.4.26
 tensorflow==2.16.1
 Keras
-validators==0.34.0
+validators==0.35.0
 ipwhois==1.2.0
 matplotlib==3.10.1
 scikit_learn
@@ -29,12 +29,12 @@ pytest-dependency==0.6.0
 whois==1.20240129.2
 flask
 tldextract==5.3.0
-termcolor==3.0.1
+termcolor==3.1.0
 yappi==1.6.10
 pytest-sugar==1.0.0
 aid_hash
 black==24.10.0
-ruff==0.11.7
+ruff==0.11.8
 pre-commit==4.0.1
 coverage==7.8.0
 netifaces==0.11.0

managers/host_ip_manager.py

@@ -1,7 +1,7 @@
 # SPDX-FileCopyrightText: 2021 Sebastian Garcia <[email protected]>
 # SPDX-License-Identifier: GPL-2.0-only
-import socket
 import time
+import netifaces
 from typing import (
     Set,
     Optional,
@@ -16,29 +16,19 @@ def __init__(self, main):
 
     def get_host_ip(self) -> Optional[str]:
         """
-        tries to determine the machine's IP address by creating a UDP
-        connection to cloudflare
-        returns ipv4 or ipv6 of the current computer
+        tries to determine the machine's IP
         """
-        for address_family in (socket.AF_INET, socket.AF_INET6):
-            try:
-                s = socket.socket(address_family, socket.SOCK_DGRAM)
+        interfaces = netifaces.interfaces()
 
-                test_address = (
-                    ("1.1.1.1", 80)
-                    if address_family == socket.AF_INET
-                    else ("2606:4700:4700::1111", 80)
-                )
-
-                s.connect(test_address)
-                ipaddr_check = s.getsockname()[0]
-                s.close()
-                return ipaddr_check
-            except socket.error:
+        for iface in interfaces:
+            addrs = netifaces.ifaddresses(iface)
+            # check for IPv4 address
+            if netifaces.AF_INET not in addrs:
                 continue
-
-        # neither ipv4 nor ipv6 worked
-        return None
+            for addr in addrs[netifaces.AF_INET]:
+                ip = addr.get("addr")
+                if ip and not ip.startswith("127."):
+                    return ip
 
     def store_host_ip(self) -> Optional[str]:
         """

managers/metadata_manager.py

@@ -77,7 +77,7 @@ def set_analysis_end_date(self, end_date):
         if not self.enable_metadata:
             return
 
-        end_date = utils.convert_format(end_date, utils.alerts_format)
+        end_date = utils.convert_ts_format(end_date, utils.alerts_format)
         self.main.db.set_input_metadata({"analysis_end": end_date})
 
         # add slips end date in the metadata dir

managers/process_manager.py

@@ -242,92 +242,120 @@ def is_abstract_module(self, obj) -> bool:
 
     def get_modules(self):
         """
-        Get modules from the 'modules' folder.
+        get modules to load from the modules/ dir and ignore the ones in
+        the disable param in the config file.
+        Starts the blocking module only if --clearblocking in given
+        and returns a list of modules to load in the correct order if
+        applicable.
         """
-        # This plugins import will automatically load the modules
-        # and put them in the __modules__ variable
         plugins = {}
         failed_to_load_modules = 0
 
+        for module_name in self._discover_module_names():
+            if not self._should_load_module(module_name):
+                continue
+
+            module = self._import_module(module_name)
+            if not module:
+                failed_to_load_modules += 1
+                continue
+
+            plugins = self._load_valid_classes_from_module(module, plugins)
+
+        plugins = self._reorder_modules(plugins)
+        return plugins, failed_to_load_modules
+
+    def _reorder_modules(self, plugins):
+        plugins = self._prioritize_blocking_module(plugins)
+        plugins = self._start_cyst_module_last(plugins)
+        return plugins
+
+    def _discover_module_names(self):
+        """
+        walk recursively through all modules and packages found in modules/
+        """
         # __path__ is the current path of this python program
         look_for_modules_in = modules.__path__
         prefix = f"{modules.__name__}."
-        # Walk recursively through all modules and packages found on the .
-        # folder.
+
         for loader, module_name, ispkg in pkgutil.walk_packages(
             look_for_modules_in, prefix
         ):
-            # If current item is a package, skip.
             if ispkg:
-                continue
+                continue  # skip if current item is a package
+
+            dir_name, file_name = module_name.split(".")[1:3]
 
             # to avoid loading everything in the dir,
             # only load modules that have the same name as the dir name
-            dir_name = module_name.split(".")[1]
-            file_name = module_name.split(".")[2]
-            if dir_name != file_name:
-                continue
-
-            if self.bootstrap_p2p:  # if bootstrapping the p2p network
-                if not self.is_bootstrapping_module(
-                    module_name
-                ):  # keep only the bootstrapping-necessary modules
-                    continue
-            else:  # if not bootstrappig mode
-                if self.is_ignored_module(
-                    module_name
-                ):  # ignore blacklisted modules
-                    continue
-
-            # Try to import the module, otherwise skip.
-            try:
-                # "level specifies whether to use absolute or relative imports.
-                # The default is -1 which
-                # indicates both absolute and relative imports will
-                # be attempted.
-                # 0 means only perform absolute imports.
-                # Positive values for level indicate the number of parent
-                # directories to search relative to the directory of the
-                # module calling __import__()."
-                module = importlib.import_module(module_name)
-            except ImportError as e:
-                print(
-                    f"Something wrong happened while "
-                    f"importing the module {module_name}: {e}"
-                )
-                print(traceback.format_exc())
-                failed_to_load_modules += 1
-                continue
-
-            # Walk through all members of currently imported modules.
-            for member_name, member_object in inspect.getmembers(module):
-                # Check if current member is a class.
-                if inspect.isclass(member_object) and (
-                    issubclass(member_object, IModule)
-                    and not self.is_abstract_module(member_object)
-                ):
-                    plugins[member_object.name] = dict(
-                        obj=member_object,
-                        description=member_object.description,
-                    )
+            if dir_name == file_name:
+                yield module_name
+
+    def _should_load_module(self, module_name):
+        # filter modules based on bootstrapping or blacklist conditions
+        if self.bootstrap_p2p:
+            if not self.is_bootstrapping_module(module_name):
+                return False  # keep only the bootstrapping-necessary modules
+        else:
+            if self.is_ignored_module(module_name):
+                return False  # ignore blacklisted modules
+        return True
 
-        # Change the order of the blocking module(load it first)
+    def _import_module(self, module_name):
+        # try to import the module, otherwise return None
+        try:
+            # "level" specifies how importlib should resolve the module
+            return importlib.import_module(module_name)
+        except ImportError as e:
+            print(
+                f"Something wrong happened while importing the module"
+                f" {module_name}: {e}"
+            )
+            print(traceback.format_exc())
+            return None
+
+    def _load_valid_classes_from_module(self, module, plugins):
+        # walk through all members of the given module
+        for member_name, member_object in inspect.getmembers(module):
+            if inspect.isclass(member_object):
+                if issubclass(
+                    member_object, IModule
+                ) and not self.is_abstract_module(member_object):
+                    plugins[member_object.name] = {
+                        "obj": member_object,
+                        "description": member_object.description,
+                    }
+        return plugins
+
+    def _prioritize_blocking_module(self, plugins):
+        # change the order of the blocking module (load it first)
         # so it can receive msgs sent from other modules
-        if "Blocking" in plugins:
-            plugins = OrderedDict(plugins)
-            # last=False to move to the beginning of the dict
-            plugins.move_to_end("Blocking", last=False)
-
+        if "Blocking" not in plugins:
+            return plugins
+
+        ordered = OrderedDict(plugins)
+        ordered.move_to_end(
+            "Blocking", last=False
+        )  # last=False to move to the beginning of the dict
+        plugins.clear()
+        plugins.update(ordered)
+        return plugins
+
+    def _start_cyst_module_last(self, plugins):
         # when cyst starts first, as soon as slips connects to cyst,
         # cyst sends slips the flows,
         # but the inputprocess didn't even start yet so the flows are lost
-        # to fix this, change the order of the CYST module(load it last)
-        if "cyst" in plugins:
-            plugins = OrderedDict(plugins)
-            # last=False to move to the beginning of the dict
-            plugins.move_to_end("cyst", last=True)
-
-        return plugins, failed_to_load_modules
+        # to fix this, change the order of the CYST module (load it last)
+        if "cyst" not in plugins:
+            return plugins
+
+        ordered = OrderedDict(plugins)
+        ordered.move_to_end(
+            "cyst", last=True
+        )  # last=True to move to the end of the dict
+        plugins.clear()
+        plugins.update(ordered)
+        return plugins
 
     def print_disabled_modules(self):
         print("-" * 27)
@@ -515,7 +543,7 @@ def get_analysis_time(self) -> Tuple[str, str]:
         returns analysis_time in minutes and slips end_time as a date
         """
         start_time = self.main.db.get_slips_start_time()
-        end_time = utils.convert_format(datetime.now(), "unixtimestamp")
+        end_time = utils.convert_ts_format(datetime.now(), "unixtimestamp")
         return (
             utils.get_time_diff(start_time, end_time, return_type="minutes"),
             end_time,