Isolate plugins dependencies

Dockerfile

@@ -11,7 +11,7 @@ RUN python3 -m venv $VIRTUAL_ENV \
 
 COPY . /app/
 
-RUN poetry install --no-root --only main \
+RUN poetry install --no-root --only $(python ./tools/get_plugins_list.py) \
     && poetry cache clear --no-interaction --all .
 
 

docker-compose-dev.yaml

@@ -26,6 +26,7 @@ services:
     ports:
       - 8000:8000
     environment:
+      SENTINELA_PLUGINS: slack
       SAMPLE_SLACK_CHANNEL: C07NCL94SDT
       SAMPLE_SLACK_MENTION: U07NFGGMB98
       SLACK_WEBSOCKET_ENABLED: true

docker-compose-local.yaml

@@ -21,6 +21,7 @@ services:
       retries: 3
       start_period: 2s
     environment:
+      SENTINELA_PLUGINS: slack
       SAMPLE_SLACK_CHANNEL: C07NCL94SDT
       SAMPLE_SLACK_MENTION: U07NFGGMB98
       SLACK_WEBSOCKET_ENABLED: true

docs/how_to_run.md

@@ -6,6 +6,10 @@ The application will try to load the configs file through the path defined in th
 
 THe monitors path is also defined in the `configs.yaml` file. By default, it's set to the `sample_monitors` folder, but it can be changed to another folder if desired. The `configs.yaml` file also have other configurations that can be adjusted.
 
+To enable a plugin, set the environment variable `SENTINELA_PLUGINS` with the name of the desired plugin. When enabling multiple plugins, separate them with commas.
+- To enable the Slack plugin, the environment variable should be set as `SENTINELA_PLUGINS=slack`.
+- To enable multiple plugins, the environment variable should be set as `SENTINELA_PLUGINS=plugin_1,plugin_2`.
+
 For the secrets, the application expects them to be set as environment variables.
 - `DATABASE_APPLICATION`: The database DSN that will be used to connect to the application database. This database will not be accessible through the databases interface for the monitors.
 - Every variable that starts with `DATABASE`, besides the application database, will have a connection pool instantiated, that can be used in the monitors to query data from them.
@@ -59,11 +63,22 @@ make run-local
 This will start the database and the application.
 
 ## Production deployment
-In production deployment, it's recommended that the controller and executors to be deployed in different containers or pods (in case of a kubernetes deployment). With this method a SQS queue is required to allow the controller communicate with the executors.
+### Building the image
+The [Dockerfile](../Dockerfile) is a starting point for building the application image. This file implements the logic to install all the enabled plugins dependencies correctly.
+
+```shell
+# Install the dependencies for the application and enabled plugins
+poetry install --no-root --only $(python ./tools/get_plugins_list.py)
+```
+
+### Deploying the application
+In production deployment, it's recommended that the controller and executors to be deployed in different containers or pods (in case of a Kubernetes deployment). With this method a SQS queue is required to allow the controller communicate with the executors.
+
+The files provided in the [Kubernetes template](../resources/kubernetes_template) directory can be used as a reference for a Kubernetes deployment.
 
-The deployment should set the necessary environment variables for all the instances for them to work properly.
+All services must have the environment variables set as specified in the [Configs and secrets](#configs-and-secrets) section.
 
-Controllers and executors can be started passing them as parameters.
+Controllers and executors can be started specifying them as parameters.
 ```shell
 # Controller
 python3 src/main.py controller

docs/plugins.md

@@ -79,3 +79,7 @@ async def stop(): ...
 ## Built-in plugins
 Sentinela comes with some built-in plugins that can be used to extend the application's functionality.
 - [Slack](./plugin_slack.md)
+
+To enable a plugin, set the environment variable `SENTINELA_PLUGINS` with the name of the desired plugin. When enabling multiple plugins, separate them with commas.
+- To enable the Slack plugin, the environment variable should be set as `SENTINELA_PLUGINS=slack`.
+- To enable multiple plugins, the environment variable should be set as `SENTINELA_PLUGINS=plugin_1,plugin_2`.

pyproject.toml

@@ -22,7 +22,6 @@ pydantic = "^2.10.4"
 python = "^3.12"
 pytz = "*"
 pyyaml = "^6.0.2"
-slack-bolt = "^1.22.0"
 sqlalchemy = "^2.0.36"
 tabulate = "^0.9.0"
 uvloop = "^0.21.0"
@@ -44,6 +43,9 @@ types-pytz = "*"
 types-pymysql = "*"
 types-croniter = "*"
 
+[tool.poetry.group.plugin-slack.dependencies]
+slack-bolt = "^1.22.0"
+
 [tool.ruff]
 include = ["src/**/*.py", "tests/**/*.py", "internal_monitors/**/*.py", "sample_monitors/**/*.py", "tools/**/*.py"]
 lint.extend-select = ["E", "F", "Q", "I", "RET"]

resources/kubernetes_template/controller.yaml

@@ -24,6 +24,8 @@ spec:
         env:
         - name: CONFIGS_FILE
           value: configs_sqs.yaml
+        - name: SENTINELA_PLUGINS
+          value: "slack"
         - name: SAMPLE_SLACK_CHANNEL
           value: C07NCL94SDT
         - name: SAMPLE_SLACK_MENTION

resources/kubernetes_template/executor.yaml

@@ -24,6 +24,8 @@ spec:
         env:
         - name: CONFIGS_FILE
           value: configs_sqs.yaml
+        - name: SENTINELA_PLUGINS
+          value: "slack"
         - name: SAMPLE_SLACK_CHANNEL
           value: C07NCL94SDT
         - name: SAMPLE_SLACK_MENTION

src/plugins/plugins_loader.py

@@ -10,7 +10,6 @@
 
 def load_plugins(path: str | None = None) -> dict[str, ModuleType]:
     """Load all plugins from the plugins directory"""
-
     if path is None:
         plugins_directory = Path(__file__).parent.relative_to(Path.cwd())
     else:
@@ -25,11 +24,17 @@ def load_plugins(path: str | None = None) -> dict[str, ModuleType]:
         if os.path.isdir(plugins_directory / item):
             plugins_names.append(item)
 
-    # Load all plugins
+    # Load all enabled plugins
+    enabled_plugins = os.getenv("SENTINELA_PLUGINS", "").split(",")
+
     plugins_relative_path = plugins_directory.relative_to("src")
     plugins_import_path = plugins_relative_path.as_posix().replace("/", ".")
     plugins = {}
     for plugin_name in plugins_names:
+        # Skip not enabled plugins
+        if plugin_name not in enabled_plugins:
+            continue
+
         try:
             plugin = importlib.import_module(f"{plugins_import_path}.{plugin_name}")
             plugins[plugin_name] = plugin

tests/plugins/test_plugins_loader.py

@@ -6,7 +6,7 @@
 
 import plugins as plugins
 import plugins.plugins_loader as plugins_loader
-from tests.test_utils import assert_message_in_log
+from tests.test_utils import assert_message_in_log, assert_message_not_in_log
 
 
 @pytest.fixture(scope="function")
@@ -28,27 +28,81 @@ def plugins_directory(temp_dir: Path) -> Path:
     return plugins_dir
 
 
-def test_load_plugins(plugins_directory):
-    """'load_plugins' should load all plugins from the directory"""
+@pytest.mark.parametrize(
+    "enabled_plugins",
+    [
+        ["plugin1", "plugin2", "plugin3"],
+        ["plugin1", "plugin2"],
+        ["plugin1", "plugin3"],
+        ["plugin2", "plugin3"],
+        ["plugin1"],
+        ["plugin2"],
+        ["plugin3"],
+        [],
+    ],
+)
+def test_load_plugins(monkeypatch, plugins_directory, enabled_plugins):
+    """'load_plugins' should load all plugins from the directory, loading only the enabled
+    plugins"""
+    monkeypatch.setenv("SENTINELA_PLUGINS", ",".join(enabled_plugins))
+
     loaded_plugins = plugins_loader.load_plugins(str(plugins_directory))
-    assert loaded_plugins["plugin1"].a == 10
-    assert loaded_plugins["plugin2"].a == 20
-    assert loaded_plugins["plugin3"].a == 30
+
+    if "plugin1" in enabled_plugins:
+        assert loaded_plugins["plugin1"].a == 10
+    else:
+        assert "plugin1" not in loaded_plugins
+    if "plugin2" in enabled_plugins:
+        assert loaded_plugins["plugin2"].a == 20
+    else:
+        assert "plugin2" not in loaded_plugins
+    if "plugin3" in enabled_plugins:
+        assert loaded_plugins["plugin3"].a == 30
+    else:
+        assert "plugin3" not in loaded_plugins
+
     assert "__pycache__" not in loaded_plugins
 
 
-def test_load_plugins_with_error(caplog, plugins_directory):
+@pytest.mark.parametrize(
+    "enabled_plugins",
+    [
+        ["plugin1", "plugin2", "plugin3"],
+        ["plugin1", "plugin2"],
+        ["plugin1", "plugin3"],
+        ["plugin2", "plugin3"],
+        ["plugin1"],
+        ["plugin2"],
+        ["plugin3"],
+        [],
+    ],
+)
+def test_load_plugins_with_error(caplog, monkeypatch, plugins_directory, enabled_plugins):
     """'load_plugins' should catch exceptions and log error messages when a plugin cannot be
     loaded"""
+    monkeypatch.setenv("SENTINELA_PLUGINS", ",".join(enabled_plugins))
+
     with open(plugins_directory / "plugin1" / "__init__.py", "w") as file:
         file.write("raise Exception('Some import error')")
 
     loaded_plugins = plugins_loader.load_plugins(str(plugins_directory))
 
-    assert_message_in_log(caplog, "Error loading plugin 'plugin1'")
-    assert_message_in_log(caplog, "Exception: Some import error")
-    assert loaded_plugins["plugin2"].a == 20
-    assert loaded_plugins["plugin3"].a == 30
+    if "plugin1" in enabled_plugins:
+        assert "plugin1" not in loaded_plugins
+        assert_message_in_log(caplog, "Error loading plugin 'plugin1'")
+        assert_message_in_log(caplog, "Exception: Some import error")
+    else:
+        assert "plugin1" not in loaded_plugins
+        assert_message_not_in_log(caplog, "Error loading plugin 'plugin1'")
+        assert_message_not_in_log(caplog, "Exception: Some import error")
+    if "plugin2" in enabled_plugins:
+        assert loaded_plugins["plugin2"].a == 20
+    else:
+        assert "plugin2" not in loaded_plugins
+    if "plugin3" in enabled_plugins:
+        assert loaded_plugins["plugin3"].a == 30
+    else:
+        assert "plugin3" not in loaded_plugins
 
 
 def test_load_plugins_default_path(mocker):

tools/get_plugins_list.py

@@ -0,0 +1,12 @@
+"""Tool to get the list of plugins from the environment variable to install the required
+dependencies"""
+
+import os
+
+plugins = os.environ.get("SENTINELA_PLUGINS")
+
+if not plugins:
+    print("main")
+else:
+    plugins_list = [f"plugin-{plugin.strip()}" for plugin in plugins.split(",")]
+    print("main," + ",".join(plugins_list))