monitors loader: load only updated monitors

Author: GabrielSalla

src/components/monitors_loader/monitors_loader.py

@@ -243,43 +243,33 @@ async def _get_monitors_to_load(
     return monitors, code_modules
 
 
-async def _load_monitors() -> None:
+async def _load_monitors(last_load_time: datetime | None) -> None:
     """Load all enabled monitors from the database and add them to the registry. If any of the
     monitor's modules fails to load, the monitor will not be added to the registry"""
     registry.monitors_ready.clear()
 
-    loaded_monitors = await Monitor.get_all(Monitor.enabled.is_(True))
-    monitors_ids = [monitor.id for monitor in loaded_monitors]
-
-    code_modules = await CodeModule.get_all(CodeModule.monitor_id.in_(monitors_ids))
-    code_modules_map = {code_module.monitor_id: code_module for code_module in code_modules}
-
-    _logger.info(f"Monitors found: {len(loaded_monitors)}")
+    monitors, code_modules = await _get_monitors_to_load(last_load_time)
+    _logger.info(f"Monitors to load: {len(code_modules)}")
 
     # To load the monitors safely, first create all the files and then import them
     # Loading right after writing the files can result in an error where the Monitor module is not
     # found
     monitors_paths = {}
-    for monitor in loaded_monitors:
+    for code_module in code_modules:
         with catch_exceptions(_logger):
-            code_module = code_modules_map.get(monitor.id)
-            if code_module is None:
-                await monitor.set_enabled(False)
-                _logger.warning(f"Monitor '{monitor.name}' has no code module, it will be disabled")
-                continue
+            monitor = monitors[code_module.monitor_id]
 
             monitors_paths[monitor.id] = module_loader.create_module_files(
                 module_name=monitor.name,
                 module_code=code_module.code,
                 additional_files=code_module.additional_files,
             )
 
-    for monitor in loaded_monitors:
+    for code_module in code_modules:
         with catch_exceptions(_logger):
-            monitor_path = monitors_paths.get(monitor.id)
-            if monitor_path is None:
-                continue
+            monitor = monitors[code_module.monitor_id]
 
+            monitor_path = monitors_paths[monitor.id]
             monitor_module = cast(MonitorModule, module_loader.load_module_from_file(monitor_path))
             _configure_monitor(monitor_module)
 
@@ -291,11 +281,12 @@ async def _load_monitors() -> None:
 
 async def _run() -> None:
     """Monitors loading loop, loading them recurrently. Stops automatically when the app stops"""
-    last_load_time: datetime
+    last_load_time: datetime | None = None
 
     while app.running():
         with catch_exceptions(_logger):
-            await _load_monitors()
+            await _disable_monitors_without_code_modules()
+            await _load_monitors(last_load_time)
             last_load_time = now()
 
             # The sleep task will start seconds earlier to try to load all monitors before the

tests/components/controller/test_controller.py

@@ -338,7 +338,7 @@ async def test_run(monkeypatch, clear_queue, clear_database):
     assert len(queue_items) == 0
 
     # Load the monitors and wait for a while
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
     await asyncio.sleep(0.2)
 
     # Stop the app and wait for the controller task
@@ -399,7 +399,7 @@ async def test_run_monitors_not_ready(caplog, monkeypatch, mocker):
 
     # Run the controller for a while then stop it
     await monitors_loader._register_monitors()
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
     registry.monitors_ready.clear()
 
     controller_task = asyncio.create_task(controller.run())
@@ -419,7 +419,7 @@ async def test_run_monitors_not_registered(caplog, monkeypatch, mocker):
 
     # Run the controller for a while then stop it
     await monitors_loader._register_monitors()
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     controller_task = asyncio.create_task(controller.run())
     await asyncio.sleep(0.2)
@@ -442,7 +442,7 @@ def error(*args):
     # Run the controller for a while then stop it
     await monitors_loader._register_monitors()
     controller_task = asyncio.create_task(controller.run())
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
     await asyncio.sleep(0.3)
 
     assert_message_in_log(caplog, "ValueError: Not able to get the monitors")

tests/components/monitors_loader/test_monitors_loader.py

@@ -680,13 +680,41 @@ async def test_load_monitors(clear_database):
         "(9999457, 'def get_value(): return 12');"
     )
 
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     assert len(registry._monitors) == 2
     assert isinstance(registry._monitors[9999123]["module"], ModuleType)
     assert isinstance(registry._monitors[9999456]["module"], ModuleType)
 
 
+async def test_load_monitors_only_updated(clear_database):
+    """'_load_monitors' should load all enabled monitors that were updated after the provided
+    timestamp"""
+    await databases.execute_application(
+        'insert into "Monitors"(id, name, enabled) values'
+        "(9999123, 'monitor_1', true),"
+        "(9999456, 'internal.monitor_2', true),"
+        "(9999457, 'disabled_monitor', false);"
+    )
+    await databases.execute_application(
+        'insert into "CodeModules"(monitor_id, code, registered_at) values'
+        "(9999123, 'def get_value(): return 10', '2025-01-10 00:00'),"
+        "(9999456, 'def get_value(): return 11', '2025-01-20 00:00'),"
+        "(9999457, 'def get_value(): return 12', '2025-01-30 00:00');"
+    )
+
+    registry.add_monitor(9999123, "monitor_1", ModuleType(name="MockMonitorModule"))
+    await monitors_loader._load_monitors(datetime(2025, 1, 15, tzinfo=timezone.utc))
+
+    assert len(registry._monitors) == 2
+    assert isinstance(registry._monitors[9999123]["module"], ModuleType)
+    assert isinstance(registry._monitors[9999456]["module"], ModuleType)
+    # Only the monitor 9999456 was updated after the timestamp, so only it should be reloaded
+    with pytest.raises(AttributeError):
+        registry._monitors[9999123]["module"].get_value()
+    assert registry._monitors[9999456]["module"].get_value() == 11
+
+
 async def test_load_monitors_monitors_ready_flag(monkeypatch, clear_database):
     """'_load_monitors' should clear and set the registry's 'monitors_ready' while loading the
     monitors"""
@@ -711,7 +739,7 @@ async def slow_get_all(self, *args, **kwargs):
     assert registry.monitors_ready.is_set()
     assert registry.monitors_pending.is_set()
 
-    load_monitors_task = asyncio.create_task(monitors_loader._load_monitors())
+    load_monitors_task = asyncio.create_task(monitors_loader._load_monitors(None))
 
     await asyncio.sleep(0.1)
     assert not registry.monitors_ready.is_set()
@@ -722,30 +750,6 @@ async def slow_get_all(self, *args, **kwargs):
     assert not registry.monitors_pending.is_set()
 
 
-async def test_load_monitors_monitor_without_code_module(caplog, monkeypatch, clear_database):
-    """'_load_monitors' should disable the monitor if it doesn't have a code module"""
-    monitor_get_all = Monitor.get_all
-
-    async def slow_get_all(self, *args, **kwargs):
-        await asyncio.sleep(0.2)
-        return await monitor_get_all(self, *args, **kwargs)
-
-    monkeypatch.setattr(Monitor, "get_all", slow_get_all)
-
-    await databases.execute_application(
-        "insert into \"Monitors\"(id, name, enabled) values (9999123, 'monitor_1', true);"
-    )
-
-    await monitors_loader._load_monitors()
-
-    monitor = await Monitor.get_by_id(9999123)
-    assert monitor is not None
-    assert not monitor.enabled
-
-    assert len(registry._monitors) == 0
-    assert_message_in_log(caplog, "Monitor 'monitor_1' has no code module, it will be disabled")
-
-
 async def test_load_monitors_error(caplog, clear_database):
     """'_load_monitors' should load all the monitors from the database and add them to the
     registry, even if an error occurs while loading any of them. Monitors with errors will not be
@@ -761,7 +765,7 @@ async def test_load_monitors_error(caplog, clear_database):
         "(9999456, 'def get_value(): return 10');"
     )
 
-    await monitors_loader._load_monitors()
+    await monitors_loader._load_monitors(None)
 
     assert len(registry._monitors) == 1
     assert isinstance(registry._monitors[9999456]["module"], ModuleType)