monitors loader: load only updated monitors

Author: GabrielSalla

src/components/monitors_loader/monitors_loader.py

@@ -238,43 +238,33 @@ async def _get_monitors_to_load(
     return monitors, code_modules
 
 
-async def _load_monitors() -> None:
+async def _load_monitors(last_load_time: datetime | None) -> None:
     """Load all enabled monitors from the database and add them to the registry. If any of the
     monitor's modules fails to load, the monitor will not be added to the registry"""
     registry.monitors_ready.clear()
 
-    loaded_monitors = await Monitor.get_all(Monitor.enabled.is_(True))
-    monitors_ids = [monitor.id for monitor in loaded_monitors]
-
-    code_modules = await CodeModule.get_all(CodeModule.monitor_id.in_(monitors_ids))
-    code_modules_map = {code_module.monitor_id: code_module for code_module in code_modules}
-
-    _logger.info(f"Monitors found: {len(loaded_monitors)}")
+    monitors, code_modules = await _get_monitors_to_load(last_load_time)
+    _logger.info(f"Monitors to load: {len(code_modules)}")
 
     # To load the monitors safely, first create all the files and then import them
     # Loading right after writing the files can result in an error where the Monitor module is not
     # found
     monitors_paths = {}
-    for monitor in loaded_monitors:
+    for code_module in code_modules:
         with catch_exceptions(_logger):
-            code_module = code_modules_map.get(monitor.id)
-            if code_module is None:
-                await monitor.set_enabled(False)
-                _logger.warning(f"Monitor '{monitor.name}' has no code module, it will be disabled")
-                continue
+            monitor = monitors[code_module.monitor_id]
 
             monitors_paths[monitor.id] = module_loader.create_module_files(
                 module_name=monitor.name,
                 module_code=code_module.code,
                 additional_files=code_module.additional_files,
             )
 
-    for monitor in loaded_monitors:
+    for code_module in code_modules:
         with catch_exceptions(_logger):
-            monitor_path = monitors_paths.get(monitor.id)
-            if monitor_path is None:
-                continue
+            monitor = monitors[code_module.monitor_id]
 
+            monitor_path = monitors_paths[monitor.id]
             monitor_module = cast(MonitorModule, module_loader.load_module_from_file(monitor_path))
             _configure_monitor(monitor_module)
 
@@ -286,11 +276,12 @@ async def _load_monitors() -> None:
 
 async def _run() -> None:
     """Monitors loading loop, loading them recurrently. Stops automatically when the app stops"""
-    last_load_time: datetime
+    last_load_time: datetime | None = None
 
     while app.running():
         with catch_exceptions(_logger):
-            await _load_monitors()
+            await _disable_monitors_without_code_modules()
+            await _load_monitors(last_load_time)
             last_load_time = now()
 
             # The sleep task will start seconds earlier to try to load all monitors before the

tests/components/monitors_loader/test_monitors_loader.py

@@ -664,13 +664,35 @@ async def test_load_monitors(clear_database):
         "(9999457, 'def get_value(): return 12');"
     )
 
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     assert len(registry._monitors) == 2
     assert isinstance(registry._monitors[9999123]["module"], ModuleType)
     assert isinstance(registry._monitors[9999456]["module"], ModuleType)
 
 
+async def test_load_monitors_only_updated(clear_database):
+    """'_load_monitors' should load all enabled monitors that were updated after the provided
+    timestamp"""
+    await databases.execute_application(
+        'insert into "Monitors"(id, name, enabled) values'
+        "(9999123, 'monitor_1', true),"
+        "(9999456, 'internal.monitor_2', true),"
+        "(9999457, 'disabled_monitor', false);"
+    )
+    await databases.execute_application(
+        'insert into "CodeModules"(monitor_id, code, registered_at) values'
+        "(9999123, 'def get_value(): return 10', '2025-01-10 00:00'),"
+        "(9999456, 'def get_value(): return 11', '2025-01-20 00:00'),"
+        "(9999457, 'def get_value(): return 12', '2025-01-30 00:00');"
+    )
+
+    await monitors_loader._load_monitors(datetime(2025, 1, 15, tzinfo=timezone.utc))
+
+    assert len(registry._monitors) == 1
+    assert isinstance(registry._monitors[9999456]["module"], ModuleType)
+
+
 async def test_load_monitors_monitors_ready_flag(monkeypatch, clear_database):
     """'_load_monitors' should clear and set the registry's 'monitors_ready' while loading the
     monitors"""
@@ -695,7 +717,7 @@ async def slow_get_all(self, *args, **kwargs):
     assert registry.monitors_ready.is_set()
     assert registry.monitors_pending.is_set()
 
-    load_monitors_task = asyncio.create_task(monitors_loader._load_monitors())
+    load_monitors_task = asyncio.create_task(monitors_loader._load_monitors(None))
 
     await asyncio.sleep(0.1)
     assert not registry.monitors_ready.is_set()
@@ -720,7 +742,7 @@ async def slow_get_all(self, *args, **kwargs):
         "insert into \"Monitors\"(id, name, enabled) values (9999123, 'monitor_1', true);"
     )
 
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     monitor = await Monitor.get_by_id(9999123)
     assert monitor is not None
@@ -745,7 +767,7 @@ async def test_load_monitors_error(caplog, clear_database):
         "(9999456, 'def get_value(): return 10');"
     )
 
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     assert len(registry._monitors) == 1
     assert isinstance(registry._monitors[9999456]["module"], ModuleType)