Handle normalizer configs as well when cloning settings between organizations (1.19) (#4598)

Co-authored-by: Donny Peeters <46660228+Donnype@users.noreply.github.com>
Co-authored-by: Jan Klopper <janklopper+underdark@gmail.com>

Author: 3 people

boefjes/boefjes/dependencies/plugins.py

@@ -86,17 +86,29 @@ def get_all_settings(self, organisation_id: str, plugin_id: str):
         return self.config_storage.get_all_settings(organisation_id, plugin_id)
 
     def clone_settings_to_organisation(self, from_organisation: str, to_organisation: str):
-        # One requirement is that only boefjes enabled in the from_organisation end up being enabled for the target.
+        # One requirement is that only boefjes enabled in the from_organisation end up being enabled for the target,
+        # and only normalizers disabled in the from_organisation end up being disabled.
         for plugin_id in self.config_storage.get_enabled_boefjes(to_organisation):
             self.set_enabled_by_id(plugin_id, to_organisation, enabled=False)
 
-        for plugin in self.get_all(from_organisation):
+        for plugin_id in self.config_storage.get_enabled_normalizers(to_organisation):
+            self.set_enabled_by_id(plugin_id, to_organisation, enabled=True)
+
+        # Clone all settings from the from_organisation to the to_organisation
+        for plugin in self._get_all_without_enabled().values():
+            if not isinstance(plugin, Boefje):  # Only boefjes have settings
+                continue
             if all_settings := self.get_all_settings(from_organisation, plugin.id):
                 self.upsert_settings(all_settings, to_organisation, plugin.id)
 
+        # Enable the same boefjes
         for plugin_id in self.config_storage.get_enabled_boefjes(from_organisation):
             self.set_enabled_by_id(plugin_id, to_organisation, enabled=True)
 
+        # Disable the same normalizers
+        for plugin_id in self.config_storage.get_disabled_normalizers(from_organisation):
+            self.set_enabled_by_id(plugin_id, to_organisation, enabled=False)
+
     def upsert_settings(self, settings: dict, organisation_id: str, plugin_id: str):
         self._assert_settings_match_schema(settings, plugin_id, organisation_id)
         self._put_boefje(plugin_id)

boefjes/boefjes/sql/config_storage.py

@@ -169,6 +169,31 @@ def get_enabled_normalizers(self, organisation_id: str) -> list[str]:
 
         return [plugin[0] for plugin in enabled_normalizers.all()]
 
+    def get_disabled_boefjes(self, organisation_id: str) -> list[str]:
+        enabled_boefjes = (
+            self.session.query(BoefjeInDB.plugin_id)
+            .join(BoefjeConfigInDB)
+            .filter(BoefjeConfigInDB.boefje_id == BoefjeInDB.id)
+            .join(OrganisationInDB)
+            .filter(BoefjeConfigInDB.organisation_pk == OrganisationInDB.pk)
+            .filter(OrganisationInDB.id == organisation_id)
+            .filter(BoefjeConfigInDB.enabled == False)  # noqa: E712
+        )
+        return [x[0] for x in enabled_boefjes.all()]
+
+    def get_disabled_normalizers(self, organisation_id: str) -> list[str]:
+        enabled_normalizers = (
+            self.session.query(NormalizerInDB.plugin_id)
+            .join(NormalizerConfigInDB)
+            .filter(NormalizerConfigInDB.normalizer_id == NormalizerInDB.id)
+            .join(OrganisationInDB)
+            .filter(NormalizerConfigInDB.organisation_pk == OrganisationInDB.pk)
+            .filter(OrganisationInDB.id == organisation_id)
+            .filter(NormalizerConfigInDB.enabled == False)  # noqa: E712
+        )
+
+        return [plugin[0] for plugin in enabled_normalizers.all()]
+
     def get_states_for_organisation(self, organisation_id: str) -> dict[str, bool]:
         enabled_boefjes = (
             self.session.query(BoefjeInDB.plugin_id, BoefjeConfigInDB.enabled)

boefjes/boefjes/storage/interfaces.py

@@ -157,6 +157,12 @@ def get_enabled_boefjes(self, organisation_id: str) -> list[str]:
     def get_enabled_normalizers(self, organisation_id: str) -> list[str]:
         raise NotImplementedError
 
+    def get_disabled_boefjes(self, organisation_id: str) -> list[str]:
+        raise NotImplementedError
+
+    def get_disabled_normalizers(self, organisation_id: str) -> list[str]:
+        raise NotImplementedError
+
     def get_states_for_organisation(self, organisation_id: str) -> dict[str, bool]:
         raise NotImplementedError
 

boefjes/boefjes/storage/memory.py

@@ -131,6 +131,20 @@ def get_enabled_normalizers(self, organisation_id: str) -> list[str]:
             if enabled and "norm" in plugin_id
         ]
 
+    def get_disabled_boefjes(self, organisation_id: str) -> list[str]:
+        return [
+            plugin_id
+            for plugin_id, enabled in self._enabled.get(organisation_id, {}).items()
+            if not enabled and "norm" not in plugin_id
+        ]
+
+    def get_disabled_normalizers(self, organisation_id: str) -> list[str]:
+        return [
+            plugin_id
+            for plugin_id, enabled in self._enabled.get(organisation_id, {}).items()
+            if not enabled and "norm" in plugin_id
+        ]
+
     def get_states_for_organisation(self, organisation_id: str) -> dict[str, bool]:
         return self._enabled.get(organisation_id, {})
 

boefjes/tests/integration/test_api.py

@@ -316,6 +316,7 @@ def test_clone_settings_and_config_api_shows_both(test_client, organisation):
         json={"test_key": "test value", "test_key_2": "test value 2"},
     )
     test_client.patch(f"/v1/organisations/{organisation.id}/plugins/{plug}", json={"enabled": True})
+    test_client.patch(f"/v1/organisations/{organisation.id}/plugins/kat_dns_normalize", json={"enabled": False})
 
     assert test_client.get(f"/v1/organisations/{organisation.id}/{plug}/settings").json() == {
         "test_key": "test value",
@@ -352,6 +353,10 @@ def test_clone_settings_and_config_api_shows_both(test_client, organisation):
     response = test_client.get(f"/v1/organisations/{new_org_id}/plugins/nmap")
     assert response.json()["enabled"] is False
 
+    # And the originally disabled normalizer got disabled
+    response = test_client.get(f"/v1/organisations/{new_org_id}/plugins/kat_dns_normalize")
+    assert response.json()["enabled"] is False
+
     # Assert we can fetch the settings with the new configs API
     expected = [
         {

mula/scheduler/utils/cron.py

@@ -8,4 +8,4 @@ def next_run(expression: str, start_time: datetime | None = None) -> datetime:
         start_time = datetime.now(timezone.utc)
 
     cron = croniter(expression, start_time)
-    return cron.get_next(datetime)  # type: ignore
+    return cron.get_next(datetime)