diff --git a/Dockerfile b/Dockerfile index 1a06045b..e4a6d88f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,7 +11,7 @@ RUN python3 -m venv $VIRTUAL_ENV \ COPY . /app/ -RUN poetry install --no-root --only main \ +RUN poetry install --no-root --only $(python ./tools/get_plugins_list.py) \ && poetry cache clear --no-interaction --all . diff --git a/docker-compose-dev.yaml b/docker-compose-dev.yaml index a6481c4b..89561766 100644 --- a/docker-compose-dev.yaml +++ b/docker-compose-dev.yaml @@ -26,6 +26,7 @@ services: ports: - 8000:8000 environment: + SENTINELA_PLUGINS: slack SAMPLE_SLACK_CHANNEL: C07NCL94SDT SAMPLE_SLACK_MENTION: U07NFGGMB98 SLACK_WEBSOCKET_ENABLED: true diff --git a/docker-compose-local.yaml b/docker-compose-local.yaml index 21714d73..98096d4e 100644 --- a/docker-compose-local.yaml +++ b/docker-compose-local.yaml @@ -21,6 +21,7 @@ services: retries: 3 start_period: 2s environment: + SENTINELA_PLUGINS: slack SAMPLE_SLACK_CHANNEL: C07NCL94SDT SAMPLE_SLACK_MENTION: U07NFGGMB98 SLACK_WEBSOCKET_ENABLED: true diff --git a/docs/how_to_run.md b/docs/how_to_run.md index 5300674e..94c4ecfc 100644 --- a/docs/how_to_run.md +++ b/docs/how_to_run.md @@ -6,6 +6,10 @@ The application will try to load the configs file through the path defined in th THe monitors path is also defined in the `configs.yaml` file. By default, it's set to the `sample_monitors` folder, but it can be changed to another folder if desired. The `configs.yaml` file also have other configurations that can be adjusted. +To enable a plugin, set the environment variable `SENTINELA_PLUGINS` with the name of the desired plugin. When enabling multiple plugins, separate them with commas. +- To enable the Slack plugin, the environment variable should be set as `SENTINELA_PLUGINS=slack`. +- To enable multiple plugins, the environment variable should be set as `SENTINELA_PLUGINS=plugin_1,plugin_2`. + For the secrets, the application expects them to be set as environment variables. - `DATABASE_APPLICATION`: The database DSN that will be used to connect to the application database. This database will not be accessible through the databases interface for the monitors. - Every variable that starts with `DATABASE`, besides the application database, will have a connection pool instantiated, that can be used in the monitors to query data from them. @@ -59,11 +63,22 @@ make run-local This will start the database and the application. ## Production deployment -In production deployment, it's recommended that the controller and executors to be deployed in different containers or pods (in case of a kubernetes deployment). With this method a SQS queue is required to allow the controller communicate with the executors. +### Building the image +The [Dockerfile](../Dockerfile) is a starting point for building the application image. This file implements the logic to install all the enabled plugins dependencies correctly. + +```shell +# Install the dependencies for the application and enabled plugins +poetry install --no-root --only $(python ./tools/get_plugins_list.py) +``` + +### Deploying the application +In production deployment, it's recommended that the controller and executors to be deployed in different containers or pods (in case of a Kubernetes deployment). With this method a SQS queue is required to allow the controller communicate with the executors. + +The files provided in the [Kubernetes template](../resources/kubernetes_template) directory can be used as a reference for a Kubernetes deployment. -The deployment should set the necessary environment variables for all the instances for them to work properly. +All services must have the environment variables set as specified in the [Configs and secrets](#configs-and-secrets) section. -Controllers and executors can be started passing them as parameters. +Controllers and executors can be started specifying them as parameters. ```shell # Controller python3 src/main.py controller diff --git a/docs/plugins.md b/docs/plugins.md index c44a2d6c..d8b4dba1 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -79,3 +79,7 @@ async def stop(): ... ## Built-in plugins Sentinela comes with some built-in plugins that can be used to extend the application's functionality. - [Slack](./plugin_slack.md) + +To enable a plugin, set the environment variable `SENTINELA_PLUGINS` with the name of the desired plugin. When enabling multiple plugins, separate them with commas. +- To enable the Slack plugin, the environment variable should be set as `SENTINELA_PLUGINS=slack`. +- To enable multiple plugins, the environment variable should be set as `SENTINELA_PLUGINS=plugin_1,plugin_2`. diff --git a/poetry.lock b/poetry.lock index abe35d7a..aa4ff7d5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1492,7 +1492,7 @@ version = "1.22.0" description = "The Bolt Framework for Python" optional = false python-versions = ">=3.6" -groups = ["main"] +groups = ["plugin-slack"] files = [ {file = "slack_bolt-1.22.0-py2.py3-none-any.whl", hash = "sha256:349097136a586617e5fb71f40f58a30fa847f664c598577f67a01f99faa1a9eb"}, {file = "slack_bolt-1.22.0.tar.gz", hash = "sha256:b9c66d088fe3ec8bdd0494278eb500fe58092c2941de86d6822d00f4b3c7c88b"}, @@ -1507,7 +1507,7 @@ version = "3.34.0" description = "The Slack API Platform SDK for Python" optional = false python-versions = ">=3.6" -groups = ["main"] +groups = ["plugin-slack"] files = [ {file = "slack_sdk-3.34.0-py2.py3-none-any.whl", hash = "sha256:c61f57f310d85be83466db5a98ab6ae3bb2e5587437b54fa0daa8fae6a0feffa"}, {file = "slack_sdk-3.34.0.tar.gz", hash = "sha256:ff61db7012160eed742285ea91f11c72b7a38a6500a7f6c5335662b4bc6b853d"}, @@ -1971,4 +1971,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.12" -content-hash = "d9ca1798dfde1a2515ee7f59a4631127e2964dac08a1a5c9acbef2741342ac81" +content-hash = "4601479a1a2e932f2c3438d29df174d4c8d149abc733603b0360f0f91bbefd2a" diff --git a/pyproject.toml b/pyproject.toml index 70cd3bdf..d4aba03c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,6 @@ pydantic = "^2.10.4" python = "^3.12" pytz = "*" pyyaml = "^6.0.2" -slack-bolt = "^1.22.0" sqlalchemy = "^2.0.36" tabulate = "^0.9.0" uvloop = "^0.21.0" @@ -44,6 +43,9 @@ types-pytz = "*" types-pymysql = "*" types-croniter = "*" +[tool.poetry.group.plugin-slack.dependencies] +slack-bolt = "^1.22.0" + [tool.ruff] include = ["src/**/*.py", "tests/**/*.py", "internal_monitors/**/*.py", "sample_monitors/**/*.py", "tools/**/*.py"] lint.extend-select = ["E", "F", "Q", "I", "RET"] diff --git a/resources/kubernetes_template/controller.yaml b/resources/kubernetes_template/controller.yaml index 522c46e0..e122b706 100644 --- a/resources/kubernetes_template/controller.yaml +++ b/resources/kubernetes_template/controller.yaml @@ -24,6 +24,8 @@ spec: env: - name: CONFIGS_FILE value: configs_sqs.yaml + - name: SENTINELA_PLUGINS + value: "slack" - name: SAMPLE_SLACK_CHANNEL value: C07NCL94SDT - name: SAMPLE_SLACK_MENTION diff --git a/resources/kubernetes_template/executor.yaml b/resources/kubernetes_template/executor.yaml index 6c0d1c43..55283b95 100644 --- a/resources/kubernetes_template/executor.yaml +++ b/resources/kubernetes_template/executor.yaml @@ -24,6 +24,8 @@ spec: env: - name: CONFIGS_FILE value: configs_sqs.yaml + - name: SENTINELA_PLUGINS + value: "slack" - name: SAMPLE_SLACK_CHANNEL value: C07NCL94SDT - name: SAMPLE_SLACK_MENTION diff --git a/src/plugins/plugins_loader.py b/src/plugins/plugins_loader.py index 46f248c7..76164530 100644 --- a/src/plugins/plugins_loader.py +++ b/src/plugins/plugins_loader.py @@ -10,7 +10,6 @@ def load_plugins(path: str | None = None) -> dict[str, ModuleType]: """Load all plugins from the plugins directory""" - if path is None: plugins_directory = Path(__file__).parent.relative_to(Path.cwd()) else: @@ -25,11 +24,17 @@ def load_plugins(path: str | None = None) -> dict[str, ModuleType]: if os.path.isdir(plugins_directory / item): plugins_names.append(item) - # Load all plugins + # Load all enabled plugins + enabled_plugins = os.getenv("SENTINELA_PLUGINS", "").split(",") + plugins_relative_path = plugins_directory.relative_to("src") plugins_import_path = plugins_relative_path.as_posix().replace("/", ".") plugins = {} for plugin_name in plugins_names: + # Skip not enabled plugins + if plugin_name not in enabled_plugins: + continue + try: plugin = importlib.import_module(f"{plugins_import_path}.{plugin_name}") plugins[plugin_name] = plugin diff --git a/tests/plugins/test_plugins_loader.py b/tests/plugins/test_plugins_loader.py index a03c7b87..9605f040 100644 --- a/tests/plugins/test_plugins_loader.py +++ b/tests/plugins/test_plugins_loader.py @@ -6,7 +6,7 @@ import plugins as plugins import plugins.plugins_loader as plugins_loader -from tests.test_utils import assert_message_in_log +from tests.test_utils import assert_message_in_log, assert_message_not_in_log @pytest.fixture(scope="function") @@ -28,27 +28,81 @@ def plugins_directory(temp_dir: Path) -> Path: return plugins_dir -def test_load_plugins(plugins_directory): - """'load_plugins' should load all plugins from the directory""" +@pytest.mark.parametrize( + "enabled_plugins", + [ + ["plugin1", "plugin2", "plugin3"], + ["plugin1", "plugin2"], + ["plugin1", "plugin3"], + ["plugin2", "plugin3"], + ["plugin1"], + ["plugin2"], + ["plugin3"], + [], + ], +) +def test_load_plugins(monkeypatch, plugins_directory, enabled_plugins): + """'load_plugins' should load all plugins from the directory, loading only the enabled + plugins""" + monkeypatch.setenv("SENTINELA_PLUGINS", ",".join(enabled_plugins)) + loaded_plugins = plugins_loader.load_plugins(str(plugins_directory)) - assert loaded_plugins["plugin1"].a == 10 - assert loaded_plugins["plugin2"].a == 20 - assert loaded_plugins["plugin3"].a == 30 + + if "plugin1" in enabled_plugins: + assert loaded_plugins["plugin1"].a == 10 + else: + assert "plugin1" not in loaded_plugins + if "plugin2" in enabled_plugins: + assert loaded_plugins["plugin2"].a == 20 + else: + assert "plugin2" not in loaded_plugins + if "plugin3" in enabled_plugins: + assert loaded_plugins["plugin3"].a == 30 + else: + assert "plugin3" not in loaded_plugins + assert "__pycache__" not in loaded_plugins -def test_load_plugins_with_error(caplog, plugins_directory): +@pytest.mark.parametrize( + "enabled_plugins", + [ + ["plugin1", "plugin2", "plugin3"], + ["plugin1", "plugin2"], + ["plugin1", "plugin3"], + ["plugin2", "plugin3"], + ["plugin1"], + ["plugin2"], + ["plugin3"], + [], + ], +) +def test_load_plugins_with_error(caplog, monkeypatch, plugins_directory, enabled_plugins): """'load_plugins' should catch exceptions and log error messages when a plugin cannot be loaded""" + monkeypatch.setenv("SENTINELA_PLUGINS", ",".join(enabled_plugins)) + with open(plugins_directory / "plugin1" / "__init__.py", "w") as file: file.write("raise Exception('Some import error')") loaded_plugins = plugins_loader.load_plugins(str(plugins_directory)) - assert_message_in_log(caplog, "Error loading plugin 'plugin1'") - assert_message_in_log(caplog, "Exception: Some import error") - assert loaded_plugins["plugin2"].a == 20 - assert loaded_plugins["plugin3"].a == 30 + if "plugin1" in enabled_plugins: + assert "plugin1" not in loaded_plugins + assert_message_in_log(caplog, "Error loading plugin 'plugin1'") + assert_message_in_log(caplog, "Exception: Some import error") + else: + assert "plugin1" not in loaded_plugins + assert_message_not_in_log(caplog, "Error loading plugin 'plugin1'") + assert_message_not_in_log(caplog, "Exception: Some import error") + if "plugin2" in enabled_plugins: + assert loaded_plugins["plugin2"].a == 20 + else: + assert "plugin2" not in loaded_plugins + if "plugin3" in enabled_plugins: + assert loaded_plugins["plugin3"].a == 30 + else: + assert "plugin3" not in loaded_plugins def test_load_plugins_default_path(mocker): diff --git a/tools/get_plugins_list.py b/tools/get_plugins_list.py new file mode 100644 index 00000000..f01c1540 --- /dev/null +++ b/tools/get_plugins_list.py @@ -0,0 +1,12 @@ +"""Tool to get the list of plugins from the environment variable to install the required +dependencies""" + +import os + +plugins = os.environ.get("SENTINELA_PLUGINS") + +if not plugins: + print("main") +else: + plugins_list = [f"plugin-{plugin.strip()}" for plugin in plugins.split(",")] + print("main," + ",".join(plugins_list))