From 119041fb7c085a8592e66ae25f76b2bb88286956 Mon Sep 17 00:00:00 2001 From: Anton Kremenetsky Date: Fri, 20 Jun 2025 17:22:14 +0300 Subject: [PATCH] Secret manager: Passwords Added the initial part of Secret Manager to manager passwords. The first implementation is supported only plain passwords but a particular `constructor` interface allow to implement more strict solutions in the future. The secret manager follows the universal agent way so actual password generation and saving is on the driver side. Single driver `PasswordCapabilityDriver' has been added in the first implementation that uses the PG database as a storage for passwords. An example, of adding a password: Request: ```curl curl --location 'http://10.20.0.2:11010/v1/secret/passwords/' \ --header 'Content-Type: application/json' \ --header 'Authorization: Bearer ' \ --data '{ "name": "my-password", "project_id": "c27f59fe-d3d3-4625-804c-5c9fd3bb3142", "method": "AUTO_HEX", "constructor": { "kind": "plain" } }' ``` Response: ```curl { "uuid": "985dad4b-161c-432c-8d9f-be78763df758", "created_at": "2025-06-25T20:39:50.095203Z", "updated_at": "2025-06-25T20:39:56.335798Z", "project_id": "c27f59fe-d3d3-4625-804c-5c9fd3bb3142", "name": "my-password", "description": "", "method": "AUTO_HEX", "status": "ACTIVE", "constructor": { "kind": "plain" }, "value": "755eca35ece5bd921852c72996d1a790" } ``` There are two supported method for generation: AUTO_HEX, AUTO_URL_SAFE Other improvements: - Significant fixes for the configuraiton service, integration with SDK. - Fixes for Element Manager for integration with universal agent. - A tiny fix for DNS for tests. Signed-off-by: Anton Kremenetsky --- .../genesis_universal_agent.conf | 5 +- .../universal/{driver.py => drivers/core.py} | 0 .../universal/drivers/secret/__init__.py | 0 .../drivers/secret/backend/__init__.py | 0 .../universal/drivers/secret/backend/db.py | 121 +++++++++ .../universal/drivers/secret/dm/__init__.py | 0 .../universal/drivers/secret/dm/models.py | 61 +++++ .../universal/drivers/secret/password.py | 44 ++++ genesis_core/config/constants.py | 2 + genesis_core/config/dm/models.py | 72 +----- genesis_core/config/service.py | 78 +++--- genesis_core/elements/dm/models.py | 4 + genesis_core/gservice/service.py | 5 +- genesis_core/secret/__init__.py | 0 genesis_core/secret/constants.py | 37 +++ genesis_core/secret/dm/__init__.py | 0 genesis_core/secret/dm/models.py | 107 ++++++++ genesis_core/secret/service.py | 232 ++++++++++++++++++ genesis_core/status_api/api/routes.py | 2 +- genesis_core/tests/functional/conftest.py | 34 +++ .../functional/restapi/secret/__init__.py | 0 .../restapi/secret/test_passwords.py | 208 ++++++++++++++++ .../tests/functional/service/test_config.py | 6 +- .../tests/functional/service/test_secrets.py | 221 +++++++++++++++++ genesis_core/user_api/api/routes.py | 2 + genesis_core/user_api/secret/__init__.py | 0 genesis_core/user_api/secret/api/__init__.py | 0 .../user_api/secret/api/controllers.py | 47 ++++ genesis_core/user_api/secret/api/routes.py | 34 +++ migrations/0018-add-elements-76bca4.py | 25 +- migrations/0019-init-dns-40a307.py | 2 +- migrations/0020-init-secret-a643b1.py | 98 ++++++++ requirements.txt | 2 +- setup.cfg | 3 +- 34 files changed, 1339 insertions(+), 113 deletions(-) rename genesis_core/agent/universal/{driver.py => drivers/core.py} (100%) create mode 100644 genesis_core/agent/universal/drivers/secret/__init__.py create mode 100644 genesis_core/agent/universal/drivers/secret/backend/__init__.py create mode 100644 genesis_core/agent/universal/drivers/secret/backend/db.py create mode 100644 genesis_core/agent/universal/drivers/secret/dm/__init__.py create mode 100644 genesis_core/agent/universal/drivers/secret/dm/models.py create mode 100644 genesis_core/agent/universal/drivers/secret/password.py create mode 100644 genesis_core/secret/__init__.py create mode 100644 genesis_core/secret/constants.py create mode 100644 genesis_core/secret/dm/__init__.py create mode 100644 genesis_core/secret/dm/models.py create mode 100644 genesis_core/secret/service.py create mode 100644 genesis_core/tests/functional/restapi/secret/__init__.py create mode 100644 genesis_core/tests/functional/restapi/secret/test_passwords.py create mode 100644 genesis_core/tests/functional/service/test_secrets.py create mode 100644 genesis_core/user_api/secret/__init__.py create mode 100644 genesis_core/user_api/secret/api/__init__.py create mode 100644 genesis_core/user_api/secret/api/controllers.py create mode 100644 genesis_core/user_api/secret/api/routes.py create mode 100644 migrations/0020-init-secret-a643b1.py diff --git a/etc/genesis_universal_agent/genesis_universal_agent.conf b/etc/genesis_universal_agent/genesis_universal_agent.conf index ca9f7cad..8346e1da 100644 --- a/etc/genesis_universal_agent/genesis_universal_agent.conf +++ b/etc/genesis_universal_agent/genesis_universal_agent.conf @@ -11,11 +11,11 @@ connection_pool_size = 2 [universal_agent] orch_endpoint = http://localhost:11011 status_endpoint = http://localhost:11012 -caps_drivers = CoreCapabilityDriver +caps_drivers = CoreCapabilityDriver,PasswordCapabilityDriver [universal_agent_scheduler] -capabilities = em_core_* +capabilities = em_core_*,password [CoreCapabilityDriver] @@ -25,3 +25,4 @@ user_api_base_url = http://localhost:11010 project_id = 12345678-c625-4fee-81d5-f691897b8142 em_core_compute_nodes = /v1/nodes/ em_core_config_configs = /v1/config/configs/ +em_core_secret_passwords = /v1/secret/passwords/ diff --git a/genesis_core/agent/universal/driver.py b/genesis_core/agent/universal/drivers/core.py similarity index 100% rename from genesis_core/agent/universal/driver.py rename to genesis_core/agent/universal/drivers/core.py diff --git a/genesis_core/agent/universal/drivers/secret/__init__.py b/genesis_core/agent/universal/drivers/secret/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/agent/universal/drivers/secret/backend/__init__.py b/genesis_core/agent/universal/drivers/secret/backend/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/agent/universal/drivers/secret/backend/db.py b/genesis_core/agent/universal/drivers/secret/backend/db.py new file mode 100644 index 00000000..a5e54465 --- /dev/null +++ b/genesis_core/agent/universal/drivers/secret/backend/db.py @@ -0,0 +1,121 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import secrets +import logging +import typing as tp + +from restalchemy.dm import filters as dm_filters +from restalchemy.storage import exceptions as ra_exc +from gcl_sdk.agents.universal.dm import models +from gcl_sdk.agents.universal.clients.backend import base +from gcl_sdk.agents.universal.clients.backend import exceptions + +from genesis_core.secret import constants as sc +from genesis_core.secret.dm import models as secret_dm +from genesis_core.agent.universal.drivers.secret.dm import models as driver_dm + + +LOG = logging.getLogger(__name__) + + +class DatabaseSecretBackendClient(base.AbstractBackendClient): + """Secret Backend client based on SQL database.""" + + def get(self, resource: models.Resource) -> dict[str, tp.Any]: + """Get the resource value in dictionary format.""" + try: + driver_password = driver_dm.Password.objects.get_one( + filters={ + "uuid": dm_filters.EQ(resource.uuid), + }, + ) + except ra_exc.RecordNotFound: + raise exceptions.ResourceNotFound(resource=resource) + + return driver_password.meta + + def create(self, resource: models.Resource) -> dict[str, tp.Any]: + """Creates the resource. Returns the created resource.""" + try: + self.get(resource) + except exceptions.ResourceNotFound: + pass + else: + raise exceptions.ResourceAlreadyExists(resource=resource) + + password = secret_dm.Password.from_ua_resource(resource) + + # Validate structure of password model + if ( + sc.SecretMethod[password.method].is_auto + and password.value is not None + ): + raise ValueError("Cannot create auto-generated password.") + + if ( + not sc.SecretMethod[password.method].is_auto + and password.value is None + ): + raise ValueError("Cannot create non-auto-generated password.") + + # Generate plain password + if sc.SecretMethod[password.method].is_auto: + if password.method == sc.SecretMethod.AUTO_HEX: + plain_password = secrets.token_hex(16) + elif password.method == sc.SecretMethod.AUTO_URL_SAFE: + plain_password = secrets.token_urlsafe(16) + else: + raise ValueError("Unknown auto-generated password method.") + else: + plain_password = password.value + + # Build password from the plain view + pass_value = password.constructor.build(plain_password) + + # Build storagable password and save + driver_password = driver_dm.Password.from_password_resource( + resource, pass_value + ) + driver_password.save() + return driver_password.meta + + def update(self, resource: models.Resource) -> dict[str, tp.Any]: + """Update the resource. Returns the updated resource.""" + + # The simplest implementation. Update via recreation. + self.delete(resource) + return self.create(resource) + + def list(self, kind: str, **kwargs) -> list[dict[str, tp.Any]]: + """Lists all resources by kind.""" + secrets = driver_dm.Password.objects.get_all() + return [s.meta for s in secrets] + + def delete(self, resource: models.Resource) -> None: + """Delete the resource.""" + try: + self.get(resource) + except exceptions.ResourceNotFound: + raise exceptions.ResourceNotFound(resource=resource) + + password = driver_dm.Password.objects.get_one( + filters={ + "uuid": dm_filters.EQ(resource.uuid), + } + ) + password.delete() diff --git a/genesis_core/agent/universal/drivers/secret/dm/__init__.py b/genesis_core/agent/universal/drivers/secret/dm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/agent/universal/drivers/secret/dm/models.py b/genesis_core/agent/universal/drivers/secret/dm/models.py new file mode 100644 index 00000000..ed8b36bc --- /dev/null +++ b/genesis_core/agent/universal/drivers/secret/dm/models.py @@ -0,0 +1,61 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import typing as tp + +from restalchemy.dm import properties +from restalchemy.dm import models +from restalchemy.dm import types +from restalchemy.storage.sql import orm +from gcl_sdk.agents.universal.dm import models as ua_models + +from genesis_core.secret import constants as sc +from genesis_core.secret.dm import models as secret_dm + + +class Password( + models.ModelWithUUID, + models.ModelWithTimestamp, + orm.SQLStorableMixin, +): + __tablename__ = "storage_passwords" + + status = properties.property( + types.Enum([s.value for s in sc.SecretStatus]), + default=sc.SecretStatus.NEW.value, + ) + value = properties.property( + types.String(min_length=1, max_length=512), + required=True, + ) + # Some additional metadata about the secret + meta = properties.property(types.Dict(), default=lambda: {}) + + @classmethod + def from_password_resource( + cls, resource: ua_models.TargetResource, password_value: str + ) -> Password: + meta = resource.value.copy() + meta["value"] = password_value + meta["status"] = sc.SecretStatus.ACTIVE.value + + return cls( + uuid=resource.uuid, + value=password_value, + status=sc.SecretStatus.ACTIVE.value, + meta=meta, + ) diff --git a/genesis_core/agent/universal/drivers/secret/password.py b/genesis_core/agent/universal/drivers/secret/password.py new file mode 100644 index 00000000..ea2173ac --- /dev/null +++ b/genesis_core/agent/universal/drivers/secret/password.py @@ -0,0 +1,44 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging + +from gcl_sdk.agents.universal.drivers import direct +from gcl_sdk.agents.universal.storage import fs + +from genesis_core.agent.universal.drivers.secret.backend import db as db_back + + +LOG = logging.getLogger(__name__) + +AGENT_WORK_DIR = "/var/lib/genesis/universal_agent/" + + +class PasswordCapabilityDriver(direct.DirectAgentDriver): + """Password capability driver.""" + + def __init__(self): + storage = fs.FileAgentStorage( + AGENT_WORK_DIR, "password_cap_storage.json" + ) + client = db_back.DatabaseSecretBackendClient() + + super().__init__(storage=storage, client=client) + + def get_capabilities(self) -> list[str]: + """Returns a list of capabilities supported by the driver.""" + return ["password"] diff --git a/genesis_core/config/constants.py b/genesis_core/config/constants.py index 0e92e719..71938f8a 100644 --- a/genesis_core/config/constants.py +++ b/genesis_core/config/constants.py @@ -18,6 +18,8 @@ import typing as tp DEFAULT_SQL_LIMIT = 100 +CONFIG_KIND = "config" +RENDER_KIND = "render" class ConfigStatus(str, enum.Enum): diff --git a/genesis_core/config/dm/models.py b/genesis_core/config/dm/models.py index e190dc03..c77c50f3 100644 --- a/genesis_core/config/dm/models.py +++ b/genesis_core/config/dm/models.py @@ -19,12 +19,10 @@ import uuid as sys_uuid from restalchemy.dm import models -from restalchemy.dm import filters as dm_filters from restalchemy.dm import properties from restalchemy.dm import types from restalchemy.dm import types_dynamic from restalchemy.storage.sql import orm -from restalchemy.storage.sql import engines from gcl_sdk.agents.universal.dm import models as ua_models @@ -135,6 +133,7 @@ class Config( cm.ModelWithFullAsset, orm.SQLStorableMixin, ua_models.TargetResourceMixin, + ua_models.TargetResourceSQLStorableMixin, ): __tablename__ = "config_configs" @@ -205,86 +204,29 @@ def render(self, node: sys_uuid.UUID) -> ua_models.TargetResource: ) resource = render.to_ua_resource("render", master=self.uuid) - resource.calculate_hash() + resource.status = cc.ConfigStatus.IN_PROGRESS.value return resource @classmethod def get_new_configs( cls, limit: int = cc.DEFAULT_SQL_LIMIT ) -> list["Config"]: - expression = ( - "SELECT " - " config_configs.uuid as uuid " - "FROM config_configs LEFT JOIN ua_target_resources ON " - " config_configs.uuid = ua_target_resources.uuid " - "WHERE ua_target_resources.uuid is NULL " - "LIMIT %s;" - ) - params = (limit,) - - engine = engines.engine_factory.get_engine() - with engine.session_manager() as session: - curs = session.execute(expression, params) - response = curs.fetchall() - - if not response: - return [] - - return cls.objects.get_all( - filters={"uuid": dm_filters.In(str(r["uuid"]) for r in response)}, - ) + return cls.get_new_entities(cls.__tablename__, cc.CONFIG_KIND, limit) @classmethod def get_updated_configs( cls, limit: int = cc.DEFAULT_SQL_LIMIT ) -> list["Config"]: - expression = ( - "SELECT " - " config_configs.uuid as uuid " - "FROM config_configs INNER JOIN ua_target_resources ON " - " config_configs.uuid = ua_target_resources.uuid " - "WHERE config_configs.updated_at != ua_target_resources.tracked_at " - "LIMIT %s;" - ) - params = (limit,) - - engine = engines.engine_factory.get_engine() - with engine.session_manager() as session: - curs = session.execute(expression, params) - response = curs.fetchall() - - if not response: - return [] - - return cls.objects.get_all( - filters={"uuid": dm_filters.In(str(r["uuid"]) for r in response)}, + return cls.get_updated_entities( + cls.__tablename__, cc.CONFIG_KIND, limit ) @classmethod def get_deleted_config_renders( cls, limit: int = cc.DEFAULT_SQL_LIMIT ) -> list[ua_models.TargetResource]: - expression = ( - "SELECT " - " ua_target_resources.uuid as uuid " - "FROM ua_target_resources LEFT JOIN config_configs ON " - " ua_target_resources.uuid = config_configs.uuid " - "WHERE ua_target_resources.kind = 'config' " - " AND config_configs.uuid is NULL " - "LIMIT %s;" - ) - params = (limit,) - - engine = engines.engine_factory.get_engine() - with engine.session_manager() as session: - curs = session.execute(expression, params) - response = curs.fetchall() - - if not response: - return [] - - return ua_models.TargetResource.objects.get_all( - filters={"uuid": dm_filters.In(str(r["uuid"]) for r in response)}, + return cls.get_deleted_target_resources( + cls.__tablename__, cc.CONFIG_KIND, limit ) diff --git a/genesis_core/config/service.py b/genesis_core/config/service.py index 5d14c2b3..5f0556fd 100644 --- a/genesis_core/config/service.py +++ b/genesis_core/config/service.py @@ -33,13 +33,11 @@ LOG = logging.getLogger(__name__) -CONFIG_CAPABILITY_RESOURCE = "config" -RENDER_CAPABILITY_RESOURCE = "render" ORPHAN_CFG_ITERATION_FREQUENCY = 10 DEF_OUTDATE_MIN_PERIOD = datetime.timedelta(minutes=10) -class ConfigService(basic.BasicService): +class ConfigServiceBuilder(basic.BasicService): def _get_new_configs( self, @@ -67,18 +65,11 @@ def _get_outdated_renders( list[tuple[ua_models.TargetResource, ua_models.Resource]], ]: renders = ua_models.OutdatedResource.objects.get_all( - filters={"kind": dm_filters.EQ(RENDER_CAPABILITY_RESOURCE)}, + filters={"kind": dm_filters.EQ(cc.RENDER_KIND)}, limit=limit, ) render_map = collections.defaultdict(list) for render in renders: - # Updated resource aren't outdated - if ( - render.target_resource.updated_at - > render.actual_resource.updated_at - ): - continue - render_map[render.target_resource.master].append( (render.target_resource, render.actual_resource) ) @@ -122,7 +113,7 @@ def _actualize_new_config( if len(target_nodes) == 0: return - config_resource = config.to_ua_resource(CONFIG_CAPABILITY_RESOURCE) + config_resource = config.to_ua_resource(cc.CONFIG_KIND) config_resource.insert() # Make renders for this config @@ -132,12 +123,7 @@ def _actualize_new_config( # Hack for scheduler render.agent = node.uuid - # It's possible that the render already exists so just skip it - try: - render.insert() - - except ra_exceptions.ConflictRecords: - LOG.warning("Render %s already exists", render.uuid) + render.insert() config.status = cc.ConfigStatus.IN_PROGRESS.value config.save() @@ -193,13 +179,21 @@ def _actualize_changed_configs(self) -> None: config_resources = ua_models.TargetResource.objects.get_all( filters={ "uuid": dm_filters.In(str(uc.uuid) for uc in changed_configs), - "kind": dm_filters.EQ(CONFIG_CAPABILITY_RESOURCE), + "kind": dm_filters.EQ(cc.CONFIG_KIND), + } + ) + render_resources = ua_models.TargetResource.objects.get_all( + filters={ + "master": dm_filters.In( + str(uc.uuid) for uc in changed_configs + ), + "kind": dm_filters.EQ(cc.RENDER_KIND), } ) - for cfg in config_resources: + for cfg in render_resources + config_resources: cfg.delete() - LOG.debug("Outdated config resource %s deleted", cfg.uuid) + LOG.debug("Outdated resource (config/render) %s deleted", cfg.uuid) # Now they are new configs self._actualize_new_configs(changed_configs) @@ -217,21 +211,32 @@ def _actualize_outdated_config( # Update target renders with actual information from the DP. for target_render, actual_render in renders: target_render.full_hash = actual_render.full_hash - target_render.status = actual_render.status + + # `ACTIVE` only if the hash is the same + if ( + actual_render.status == cc.ConfigStatus.ACTIVE + and target_render.hash == actual_render.hash + ): + target_render.status = actual_render.status + elif ( + actual_render.status != cc.ConfigStatus.ACTIVE + and target_render.status != actual_render.status + ): + target_render.status = actual_render.status target_render.update() LOG.debug("Outdated render %s actualized", target_render.uuid) # Actualize status if needed. status = None if all(r.status == cc.ConfigStatus.ACTIVE for r, _ in renders): - status = cc.ConfigStatus.ACTIVE.value + status = cc.ConfigStatus.ACTIVE elif any(r.status == cc.ConfigStatus.NEW for r, _ in renders): - status = cc.ConfigStatus.NEW.value + status = cc.ConfigStatus.NEW elif any(r.status == cc.ConfigStatus.IN_PROGRESS for r, _ in renders): - status = cc.ConfigStatus.IN_PROGRESS.value + status = cc.ConfigStatus.IN_PROGRESS if status is not None and config.status != status: - config.status = status + config.status = status.value config.update() config_resource.tracked_at = config.updated_at config_resource.update() @@ -260,17 +265,26 @@ def _actualize_outdated_configs(self) -> None: def _actualize_deleted_configs(self) -> None: """Actualize configs deleted by user.""" - deleted_configs = self._get_deleted_configs() + deleted_config_resources = self._get_deleted_configs() - if len(deleted_configs) == 0: + if len(deleted_config_resources) == 0: return - for config in deleted_configs: + render_resources = ua_models.TargetResource.objects.get_all( + filters={ + "master": dm_filters.In( + str(uc.uuid) for uc in deleted_config_resources + ), + "kind": dm_filters.EQ(cc.RENDER_KIND), + } + ) + + for resource in render_resources + deleted_config_resources: try: - config.delete() - LOG.debug("Outdated resource %s deleted", config.uuid) + resource.delete() + LOG.debug("Outdated resource %s deleted", resource.uuid) except Exception: - LOG.exception("Error deleting resource %s", config.uuid) + LOG.exception("Error deleting resource %s", resource.uuid) def _handle_orphan_configs( self, outdate_min_period: datetime.timedelta = DEF_OUTDATE_MIN_PERIOD diff --git a/genesis_core/elements/dm/models.py b/genesis_core/elements/dm/models.py index 4bcdbbeb..e4614e0f 100644 --- a/genesis_core/elements/dm/models.py +++ b/genesis_core/elements/dm/models.py @@ -448,9 +448,13 @@ def actualize(self): hash = sdk_utils.calculate_hash(target_state) self.full_hash = self.calculate_full_hash() if self.target_resource is None: + res_uuid = sdk_models.TargetResource.gen_res_uuid( + self.uuid, self.kind + ) target_resource = sdk_models.TargetResource( uuid=self.uuid, kind=self.kind, + res_uuid=res_uuid, value=target_state, hash=hash, full_hash=self.full_hash, diff --git a/genesis_core/gservice/service.py b/genesis_core/gservice/service.py index 38b0a6f7..e6bf0f64 100644 --- a/genesis_core/gservice/service.py +++ b/genesis_core/gservice/service.py @@ -27,6 +27,7 @@ from genesis_core.node.machine import service as n_machine_service from genesis_core.network import service as n_network_service from genesis_core.config import service as config_service +from genesis_core.secret import service as secret_service LOG = logging.getLogger(__name__) @@ -71,7 +72,8 @@ def __init__(self, iter_min_period=1, iter_pause=0.1): n_machine = n_machine_service.MachineAgentService( iter_min_period=1, iter_pause=0.1 ) - cfg_service = config_service.ConfigService() + cfg_service = config_service.ConfigServiceBuilder() + secret_svc = secret_service.SecretServiceBuilder() event_sender = senders.EventSenderService.build_from_config() em_builder = em_builders.ElementManagerBuilder( iter_min_period=1, iter_pause=0.1 @@ -83,6 +85,7 @@ def __init__(self, iter_min_period=1, iter_pause=0.1): n_builder, n_machine, cfg_service, + secret_svc, event_sender, em_builder, ] diff --git a/genesis_core/secret/__init__.py b/genesis_core/secret/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/secret/constants.py b/genesis_core/secret/constants.py new file mode 100644 index 00000000..41cd843c --- /dev/null +++ b/genesis_core/secret/constants.py @@ -0,0 +1,37 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import enum + +DEFAULT_SQL_LIMIT = 100 +PASSWORD_KIND = "password" + + +class SecretStatus(str, enum.Enum): + NEW = "NEW" + IN_PROGRESS = "IN_PROGRESS" + ACTIVE = "ACTIVE" + ERROR = "ERROR" + + +class SecretMethod(str, enum.Enum): + AUTO_HEX = "AUTO_HEX" + AUTO_URL_SAFE = "AUTO_URL_SAFE" + MANUAL = "MANUAL" + + @property + def is_auto(self): + return self in {SecretMethod.AUTO_HEX, SecretMethod.AUTO_URL_SAFE} diff --git a/genesis_core/secret/dm/__init__.py b/genesis_core/secret/dm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/secret/dm/models.py b/genesis_core/secret/dm/models.py new file mode 100644 index 00000000..228acd47 --- /dev/null +++ b/genesis_core/secret/dm/models.py @@ -0,0 +1,107 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from restalchemy.dm import properties +from restalchemy.dm import types +from restalchemy.dm import types_dynamic +from restalchemy.dm import models as ra_models +from restalchemy.storage.sql import orm + +from gcl_sdk.agents.universal.dm import models as ua_models + +from genesis_core.common.dm import models as cm +from genesis_core.secret import constants as sc + + +class AbstractPasswordConstructor( + types_dynamic.AbstractKindModel, ra_models.SimpleViewMixin +): + + def build(self, plain_password: str) -> str: + raise NotImplementedError() + + +class PlainPasswordConstructor(AbstractPasswordConstructor): + KIND = "plain" + + def build(self, plain_password: str) -> str: + return plain_password + + +class Password( + cm.ModelWithFullAsset, + orm.SQLStorableMixin, + ua_models.TargetResourceMixin, + ua_models.TargetResourceSQLStorableMixin, +): + __tablename__ = "secret_passwords" + + method = properties.property( + types.Enum([s.value for s in sc.SecretMethod]), + default=sc.SecretMethod.AUTO_HEX.value, + ) + status = properties.property( + types.Enum([s.value for s in sc.SecretStatus]), + default=sc.SecretStatus.NEW.value, + ) + constructor = properties.property( + types_dynamic.KindModelSelectorType( + types_dynamic.KindModelType(PlainPasswordConstructor), + ), + required=True, + default=PlainPasswordConstructor, + ) + value = properties.property( + types.AllowNone(types.String(min_length=1, max_length=512)), + default=None, + ) + + def get_resource_target_fields(self) -> set[str]: + """Return the collection of target fields. + + Refer to the Resource model for more details about target fields. + """ + return { + "method", + "constructor", + "name", + "project_id", + "uuid", + "description", + } + + @classmethod + def get_new_passwords( + cls, limit: int = sc.DEFAULT_SQL_LIMIT + ) -> list["Password"]: + return cls.get_new_entities(cls.__tablename__, sc.PASSWORD_KIND, limit) + + @classmethod + def get_updated_passwords( + cls, limit: int = sc.DEFAULT_SQL_LIMIT + ) -> list["Password"]: + return cls.get_updated_entities( + cls.__tablename__, sc.PASSWORD_KIND, limit + ) + + @classmethod + def get_deleted_passwords( + cls, limit: int = sc.DEFAULT_SQL_LIMIT + ) -> list[ua_models.TargetResource]: + return cls.get_deleted_target_resources( + cls.__tablename__, sc.PASSWORD_KIND, limit + ) diff --git a/genesis_core/secret/service.py b/genesis_core/secret/service.py new file mode 100644 index 00000000..c02393e3 --- /dev/null +++ b/genesis_core/secret/service.py @@ -0,0 +1,232 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import logging +import collections +import typing as tp +import uuid as sys_uuid + +from restalchemy.common import contexts +from restalchemy.dm import filters as dm_filters +from gcl_looper.services import basic +from gcl_sdk.agents.universal.dm import models as ua_models + +from genesis_core.secret.dm import models +from genesis_core.secret import constants as sc + + +LOG = logging.getLogger(__name__) + + +class SecretServiceBuilder(basic.BasicService): + + def _get_new_passwords( + self, + limit: int = sc.DEFAULT_SQL_LIMIT, + ) -> list[models.Password]: + return models.Password.get_new_passwords(limit=limit) + + def _get_changed_passwords( + self, + limit: int = sc.DEFAULT_SQL_LIMIT, + ) -> list[models.Password]: + return models.Password.get_updated_passwords(limit=limit) + + def _get_deleted_passwords( + self, + limit: int = sc.DEFAULT_SQL_LIMIT, + ) -> list[ua_models.TargetResource]: + return models.Password.get_deleted_passwords(limit=limit) + + def _get_outdated_resources( + self, + limit: int = sc.DEFAULT_SQL_LIMIT, + ) -> dict[ + sys_uuid.UUID, # Resource UUID + tuple[ua_models.TargetResource, ua_models.Resource], + ]: + outdated = ua_models.OutdatedResource.objects.get_all( + filters={"kind": dm_filters.EQ(sc.PASSWORD_KIND)}, + limit=limit, + ) + return { + pair.target_resource.uuid: ( + pair.target_resource, + pair.actual_resource, + ) + for pair in outdated + } + + def _get_outdated_passwords( + self, uuids: tp.Collection[sys_uuid.UUID] + ) -> list[models.Password]: + return models.Password.objects.get_all( + filters={"uuid": dm_filters.In(str(p) for p in uuids)}, + ) + + def _actualize_new_passwords( + self, passwords: list[models.Password] | None = None + ) -> None: + """Actualize new passwords.""" + passwords = passwords or self._get_new_passwords() + + if len(passwords) == 0: + return + + # Just create resources for new passwords + for password in passwords: + password_resource = password.to_ua_resource(sc.PASSWORD_KIND) + try: + password_resource.insert() + password.status = sc.SecretStatus.IN_PROGRESS.value + password.save() + + # TODO(akremenetsky): Improve this snippet in the future + password_resource.tracked_at = password.updated_at + password_resource.status = password.status + password_resource.update() + LOG.info( + "Password resource %s created", password_resource.uuid + ) + except Exception: + LOG.exception( + "Error creating password resource %s", password.uuid + ) + + def _actualize_changed_passwords(self) -> None: + """Actualize passwords changed by user.""" + changed_passwords = {p.uuid: p for p in self._get_changed_passwords()} + + if len(changed_passwords) == 0: + return + + password_resources = ua_models.TargetResource.objects.get_all( + filters={ + "uuid": dm_filters.In( + str(p) for p in changed_passwords.keys() + ), + "kind": dm_filters.EQ(sc.PASSWORD_KIND), + } + ) + + # Update every resource in accordance with the new password + for resource in password_resources: + password = changed_passwords[resource.uuid] + new_resource = password.to_ua_resource(sc.PASSWORD_KIND) + + # Update the original resource + resource.update_value(new_resource) + try: + password.status = sc.SecretStatus.IN_PROGRESS.value + password.save() + + resource.tracked_at = password.updated_at + resource.status = password.status + resource.update() + LOG.debug("Password resource %s updated", resource.uuid) + except Exception: + LOG.exception( + "Error updating password resource %s", resource.uuid + ) + + def _actualize_outdated_password( + self, + password: models.Password, + target_resource: ua_models.TargetResource, + actual_resource: ua_models.Resource, + ) -> None: + """Actualize outdated password.""" + password_updated = False + saved_password = models.Password.from_ua_resource(actual_resource) + + # Actualize password + if ( + saved_password.status != password.status + or saved_password.value != password.value + ): + password.status = saved_password.status + password.value = saved_password.value + password.save() + password_updated = True + + # Actualize resource + if ( + password_updated + or actual_resource.status != target_resource.status + or actual_resource.full_hash != target_resource.full_hash + ): + target_resource.status = actual_resource.status + target_resource.full_hash = actual_resource.full_hash + target_resource.tracked_at = password.updated_at + target_resource.update() + + def _actualize_outdated_passwords(self) -> None: + """Actualize outdated passwords. + + It means some changes oscurred in the system and the passwords + are outdated now. For instance, their status is incorrect. + """ + resource_map = self._get_outdated_resources() + + if len(resource_map) == 0: + return + + passwords = self._get_outdated_passwords(tuple(resource_map.keys())) + + for password in passwords: + target, actual = resource_map[password.uuid] + try: + self._actualize_outdated_password(password, target, actual) + LOG.debug("Password %s actualized", password.uuid) + except Exception: + LOG.exception("Error actualizing password %s", password.uuid) + + def _actualize_deleted_passwords(self) -> None: + """Actualize passwords deleted by user.""" + deleted_passwords = self._get_deleted_passwords() + + if len(deleted_passwords) == 0: + return + + for password in deleted_passwords: + try: + password.delete() + LOG.debug("Outdated resource %s deleted", password.uuid) + except Exception: + LOG.exception("Error deleting resource %s", password.uuid) + + def _iteration(self) -> None: + with contexts.Context().session_manager(): + try: + self._actualize_new_passwords() + except Exception: + LOG.exception("Error actualizing new passwords") + + try: + self._actualize_changed_passwords() + except Exception: + LOG.exception("Error actualizing changed passwords") + + try: + self._actualize_outdated_passwords() + except Exception: + LOG.exception("Error actualizing outdated passwords") + + try: + self._actualize_deleted_passwords() + except Exception: + LOG.exception("Error actualizing deleted passwords") diff --git a/genesis_core/status_api/api/routes.py b/genesis_core/status_api/api/routes.py index 3a6b0307..c8e043d3 100644 --- a/genesis_core/status_api/api/routes.py +++ b/genesis_core/status_api/api/routes.py @@ -27,4 +27,4 @@ class ApiEndpointRoute(routes.Route): __allow_methods__ = [routes.FILTER] agents = routes.route(status_routes.UniversalAgentsRoute) - resources = routes.route(status_routes.ResourcesRoute) + kind = routes.route(status_routes.KindRoute) diff --git a/genesis_core/tests/functional/conftest.py b/genesis_core/tests/functional/conftest.py index c70c31e3..b3c66805 100644 --- a/genesis_core/tests/functional/conftest.py +++ b/genesis_core/tests/functional/conftest.py @@ -34,6 +34,8 @@ from genesis_core.tests.functional import utils as test_utils from genesis_core.config.dm import models as conf_models from genesis_core.config import constants as cc +from genesis_core.secret import constants as sc +from genesis_core.secret.dm import models as secret_models FIRST_MIGRATION = "0000-root-d34de1.py" @@ -458,6 +460,38 @@ def factory( return factory +@pytest.fixture +def password_factory(): + def factory( + uuid: sys_uuid.UUID | None = None, + name: str = "password", + constructor: secret_models.AbstractPasswordConstructor | None = None, + method: sc.SecretMethod = sc.SecretMethod.AUTO_HEX, + project_id: sys_uuid.UUID = c.SERVICE_PROJECT_ID, + status: str = cc.ConfigStatus.NEW.value, + **kwargs, + ) -> tp.Dict[str, tp.Any]: + uuid = uuid or sys_uuid.uuid4() + constructor = ( + secret_models.PlainPasswordConstructor() + if constructor is None + else constructor + ) + config = secret_models.Password( + uuid=uuid, + name=name, + method=method.value, + project_id=project_id, + status=status, + constructor=constructor, + **kwargs, + ) + view = config.dump_to_simple_view() + return view + + return factory + + @pytest.fixture def builder_factory() -> tp.Callable: def factory( diff --git a/genesis_core/tests/functional/restapi/secret/__init__.py b/genesis_core/tests/functional/restapi/secret/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/tests/functional/restapi/secret/test_passwords.py b/genesis_core/tests/functional/restapi/secret/test_passwords.py new file mode 100644 index 00000000..f6de0129 --- /dev/null +++ b/genesis_core/tests/functional/restapi/secret/test_passwords.py @@ -0,0 +1,208 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import typing as tp + +import pytest +from bazooka import exceptions as bazooka_exc +from gcl_iam.tests.functional import clients as iam_clients + +from genesis_core.secret import constants as sc +from genesis_core.secret.dm import models as secret_models + + +class TestPasswordsUserApi: + + # Utils + + @staticmethod + def _secret_cmp_shallow( + cfg_foo: tp.Dict[str, tp.Any], + cfg_bar: tp.Dict[str, tp.Any], + ): + return all( + (cfg_foo[key] == cfg_bar[key]) + for key in ( + "uuid", + "name", + "method", + "constructor", + "status", + ) + ) + + # Tests + + def test_passwords_list( + self, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + url = client.build_collection_uri(["secret/passwords"]) + + response = client.get(url) + + assert response.status_code == 200 + assert len(response.json()) == 0 + + def test_passwords_add( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + password = password_factory() + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert self._secret_cmp_shallow(password, output) + + def test_passwords_add_several( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + urls = [] + url = client.build_collection_uri(["secret/passwords"]) + for i in range(3): + password = password_factory() + response = client.post(url, json=password) + output = response.json() + assert response.status_code == 201 + assert self._secret_cmp_shallow(password, output) + urls.append(url + "/" + output["uuid"]) + + for url in urls: + response = client.get(url) + assert response.status_code == 200 + + def test_passwords_add_not_default( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + password = password_factory( + method=sc.SecretMethod.AUTO_URL_SAFE, + constructor=secret_models.PlainPasswordConstructor(), + ) + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert output["method"] == sc.SecretMethod.AUTO_URL_SAFE + assert output["constructor"]["kind"] == "plain" + + def test_passwords_update( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + password = password_factory() + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert self._secret_cmp_shallow(password, output) + + update = {"name": "foo-password"} + url = client.build_resource_uri(["secret/passwords", output["uuid"]]) + response = client.put(url, json=update) + output = response.json() + + assert response.status_code == 200 + assert output["name"] == "foo-password" + + response = client.get(url) + output = response.json() + assert response.status_code == 200 + assert output["name"] == "foo-password" + + def test_passwords_update_status_new( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + password = password_factory() + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert self._secret_cmp_shallow(password, output) + + # Manually change status + password_obj = secret_models.Password.objects.get_one( + filters={"uuid": output["uuid"]} + ) + password_obj.status = "IN_PROGRESS" + password_obj.update() + + url = client.build_resource_uri(["secret/passwords", output["uuid"]]) + response = client.get(url) + output = response.json() + assert response.status_code == 200 + assert output["status"] == "IN_PROGRESS" + + update = {"name": "foo-password"} + url = client.build_resource_uri(["secret/passwords", output["uuid"]]) + response = client.put(url, json=update) + output = response.json() + + assert response.status_code == 200 + assert output["name"] == "foo-password" + assert output["status"] == "NEW" + + def test_passwords_delete( + self, + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + client = user_api_client(auth_user_admin) + + password = password_factory() + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert self._secret_cmp_shallow(password, output) + + url = client.build_resource_uri(["secret/passwords", output["uuid"]]) + response = client.delete(url) + assert response.status_code == 204 + + with pytest.raises(bazooka_exc.NotFoundError): + client.get(url) diff --git a/genesis_core/tests/functional/service/test_config.py b/genesis_core/tests/functional/service/test_config.py index 286889d5..cd146ef1 100644 --- a/genesis_core/tests/functional/service/test_config.py +++ b/genesis_core/tests/functional/service/test_config.py @@ -24,11 +24,11 @@ from genesis_core.config.dm import models -class TestConfigService: +class TestConfigServiceBuilder: def setup_method(self) -> None: # Run service - self._service = service.ConfigService() + self._service = service.ConfigServiceBuilder() def teardown_method(self) -> None: pass @@ -77,7 +77,7 @@ def test_new_config( config = configs[0] assert config.status == "IN_PROGRESS" - assert render.status == "ACTIVE" + assert render.status == "IN_PROGRESS" assert str(render.agent) == default_node["uuid"] def test_new_config_fake_node( diff --git a/genesis_core/tests/functional/service/test_secrets.py b/genesis_core/tests/functional/service/test_secrets.py new file mode 100644 index 00000000..6e7de96d --- /dev/null +++ b/genesis_core/tests/functional/service/test_secrets.py @@ -0,0 +1,221 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import typing as tp +import uuid as sys_uuid + +from restalchemy.storage.sql import engines +from gcl_iam.tests.functional import clients as iam_clients +from gcl_sdk.agents.universal.dm import models as ua_models + +from genesis_core.secret import service +from genesis_core.secret.dm import models + + +class TestSecretsServiceBuilder: + + def setup_method(self) -> None: + # Run service + self._service = service.SecretServiceBuilder() + + def teardown_method(self) -> None: + pass + + def test_no_passwords( + self, + default_node: tp.Dict[str, tp.Any], + ): + self._service._iteration() + + def test_new_password( + self, + default_node: tp.Dict[str, tp.Any], + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + agent = ua_models.UniversalAgent( + uuid=sys_uuid.UUID(default_node["uuid"]), + node=sys_uuid.UUID(default_node["uuid"]), + name="UniversalAgent", + ) + agent.insert() + + client = user_api_client(auth_user_admin) + + password = password_factory( + target_node=sys_uuid.UUID(default_node["uuid"]) + ) + + url = client.build_collection_uri(["secret/passwords"]) + response = client.post(url, json=password) + output = response.json() + + assert response.status_code == 201 + assert output["status"] == "NEW" + + self._service._iteration() + + target_resources = ua_models.TargetResource.objects.get_all() + passwords = models.Password.objects.get_all() + + assert len(target_resources) == 1 + assert len(passwords) == 1 + password = passwords[0] + + assert password.status == "IN_PROGRESS" + + def test_in_progress_passwords( + self, + default_node: tp.Dict[str, tp.Any], + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + agent = ua_models.UniversalAgent( + uuid=sys_uuid.UUID(default_node["uuid"]), + node=sys_uuid.UUID(default_node["uuid"]), + name="UniversalAgent", + ) + agent.insert() + + client = user_api_client(auth_user_admin) + + password = password_factory() + + url = client.build_collection_uri(["secret/passwords"]) + client.post(url, json=password) + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "IN_PROGRESS" + + target_resources = ua_models.TargetResource.objects.get_all() + view = target_resources[0].dump_to_simple_view() + view.pop("master", None) + view.pop("agent", None) + view.pop("tracked_at", None) + view["status"] = "ACTIVE" + view["full_hash"] = "1111" + view["value"]["status"] = "ACTIVE" + view["value"]["value"] = "mynewpassword" + render_actual_resource = ua_models.Resource.restore_from_simple_view( + **view + ) + render_actual_resource.insert() + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "ACTIVE" + assert password.value == "mynewpassword" + + def test_update_passwords( + self, + default_node: tp.Dict[str, tp.Any], + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + agent = ua_models.UniversalAgent( + uuid=sys_uuid.UUID(default_node["uuid"]), + node=sys_uuid.UUID(default_node["uuid"]), + name="UniversalAgent", + ) + agent.insert() + + client = user_api_client(auth_user_admin) + + password = password_factory() + + url = client.build_collection_uri(["secret/passwords"]) + client.post(url, json=password) + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "IN_PROGRESS" + + target_resources = ua_models.TargetResource.objects.get_all() + view = target_resources[0].dump_to_simple_view() + view.pop("master", None) + view.pop("agent", None) + view.pop("tracked_at", None) + view["status"] = "ACTIVE" + view["full_hash"] = "1111" + view["value"]["status"] = "ACTIVE" + view["value"]["value"] = "mynewpassword" + render_actual_resource = ua_models.Resource.restore_from_simple_view( + **view + ) + render_actual_resource.insert() + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "ACTIVE" + + update = {"name": "test"} + url = client.build_resource_uri( + ["secret/passwords", str(password.uuid)] + ) + response = client.put(url, json=update) + assert response.status_code == 200 + + output = response.json() + assert output["name"] == "test" + + password = models.Password.objects.get_one() + assert password.status == "NEW" + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "IN_PROGRESS" + + def test_delete_passwords( + self, + default_node: tp.Dict[str, tp.Any], + password_factory: tp.Callable, + user_api_client: iam_clients.GenesisCoreTestRESTClient, + auth_user_admin: iam_clients.GenesisCoreAuth, + ): + agent = ua_models.UniversalAgent( + uuid=sys_uuid.UUID(default_node["uuid"]), + node=sys_uuid.UUID(default_node["uuid"]), + name="UniversalAgent", + ) + agent.insert() + + client = user_api_client(auth_user_admin) + + password = password_factory() + + url = client.build_collection_uri(["secret/passwords"]) + client.post(url, json=password) + + self._service._iteration() + + password = models.Password.objects.get_one() + assert password.status == "IN_PROGRESS" + + password.delete() + + self._service._iteration() + + target_resources = ua_models.TargetResource.objects.get_all() + assert len(target_resources) == 0 diff --git a/genesis_core/user_api/api/routes.py b/genesis_core/user_api/api/routes.py index d7a6aebe..a21e38f8 100644 --- a/genesis_core/user_api/api/routes.py +++ b/genesis_core/user_api/api/routes.py @@ -21,6 +21,7 @@ from genesis_core.user_api.em.api import routes as em_routes from genesis_core.user_api.iam.api import routes as iam_routes from genesis_core.user_api.config.api import routes as config_routes +from genesis_core.user_api.secret.api import routes as secret_routes # TODO(e.frolov): should be raw route @@ -66,6 +67,7 @@ class ApiEndpointRoute(routes.Route): iam = routes.route(iam_routes.IamRoute) em = routes.route(em_routes.ElementManagerRoute) config = routes.route(config_routes.ConfigRoute) + secret = routes.route(secret_routes.SecretRoute) nodes = routes.route(NodeRoute) machines = routes.route(MachineRoute) hypervisors = routes.route(HypervisorRoute) diff --git a/genesis_core/user_api/secret/__init__.py b/genesis_core/user_api/secret/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/user_api/secret/api/__init__.py b/genesis_core/user_api/secret/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/user_api/secret/api/controllers.py b/genesis_core/user_api/secret/api/controllers.py new file mode 100644 index 00000000..0f6db185 --- /dev/null +++ b/genesis_core/user_api/secret/api/controllers.py @@ -0,0 +1,47 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from gcl_iam import controllers as iam_controllers +from restalchemy.api import controllers +from restalchemy.api import resources + +from genesis_core.secret.dm import models +from genesis_core.secret import constants as sc + + +class SecretController(controllers.RoutesListController): + + __TARGET_PATH__ = "/v1/secret/" + + +class PasswordsController(iam_controllers.PolicyBasedController): + """Controller for /v1/secret/passwords/ endpoint""" + + __policy_name__ = "password" + __policy_service_name__ = "password" + + __resource__ = resources.ResourceByRAModel( + model_class=models.Password, + process_filters=True, + convert_underscore=False, + ) + + def update(self, uuid, **kwargs): + # Force config to be NEW + # In order to regenerate renders + kwargs["status"] = sc.SecretStatus.NEW.value + + return super().update(uuid, **kwargs) diff --git a/genesis_core/user_api/secret/api/routes.py b/genesis_core/user_api/secret/api/routes.py new file mode 100644 index 00000000..9fd721bf --- /dev/null +++ b/genesis_core/user_api/secret/api/routes.py @@ -0,0 +1,34 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from restalchemy.api import routes + +from genesis_core.user_api.secret.api import controllers + + +class PasswordsRoute(routes.Route): + """Handler for /v1/secret/passwords/ endpoint""" + + __controller__ = controllers.PasswordsController + + +class SecretRoute(routes.Route): + """Handler for /v1/secret/ endpoint""" + + __allow_methods__ = [routes.FILTER] + __controller__ = controllers.SecretController + + passwords = routes.route(PasswordsRoute) diff --git a/migrations/0018-add-elements-76bca4.py b/migrations/0018-add-elements-76bca4.py index 09b8297c..6e05acdf 100644 --- a/migrations/0018-add-elements-76bca4.py +++ b/migrations/0018-add-elements-76bca4.py @@ -138,9 +138,9 @@ def upgrade(self, session): "resource_link_prefix" VARCHAR(256) NOT NULL, "value" JSONB NOT NULL DEFAULT '{}', "target_resource" UUID DEFAULT NULL REFERENCES - ua_target_resources("uuid"), + ua_target_resources("res_uuid"), "actual_resource" UUID DEFAULT NULL REFERENCES - ua_actual_resources("uuid"), + ua_actual_resources("res_uuid"), "full_hash" VARCHAR(256) NOT NULL DEFAULT '', "created_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), "updated_at" TIMESTAMP(6) NOT NULL DEFAULT NOW() @@ -182,9 +182,17 @@ def upgrade(self, session): SELECT COALESCE("er"."uuid", "utr"."uuid") AS "uuid", "er"."uuid" AS "em_resource", - "utr"."uuid" AS "target_resource" + "utr"."res_uuid" AS "target_resource" FROM "em_resources" "er" - FULL OUTER JOIN "ua_target_resources" "utr" + FULL OUTER JOIN ( + SELECT + "uuid", + "res_uuid", + "updated_at", + "tracked_at" + FROM "ua_target_resources" + WHERE "kind" like 'em_core_%' + ) AS "utr" on "er"."uuid" = "utr"."uuid" WHERE "er"."uuid" IS NULL @@ -199,8 +207,13 @@ def upgrade(self, session): "uar"."status" AS "actual_status" FROM "em_resources" "er" - LEFT JOIN - "ua_actual_resources" "uar" + LEFT JOIN ( + SELECT + "uuid", + "status" + FROM "ua_actual_resources" + WHERE "kind" like 'em_core_%' + ) AS "uar" ON "er"."uuid" = "uar"."uuid" WHERE diff --git a/migrations/0019-init-dns-40a307.py b/migrations/0019-init-dns-40a307.py index df8fb519..713584d0 100644 --- a/migrations/0019-init-dns-40a307.py +++ b/migrations/0019-init-dns-40a307.py @@ -34,7 +34,7 @@ def is_manual(self): def upgrade(self, session): expressions = [ """ -CREATE SEQUENCE dns_domain_id_seq; +CREATE SEQUENCE IF NOT EXISTS dns_domain_id_seq; """, """ CREATE TABLE dns_domains ( diff --git a/migrations/0020-init-secret-a643b1.py b/migrations/0020-init-secret-a643b1.py new file mode 100644 index 00000000..adb7c3d1 --- /dev/null +++ b/migrations/0020-init-secret-a643b1.py @@ -0,0 +1,98 @@ +# Copyright 2016 Eugene Frolov +# Copyright 2025 Genesis Corporation +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from restalchemy.storage.sql import migrations + + +class MigrationStep(migrations.AbstarctMigrationStep): + + def __init__(self): + self._depends = ["0019-init-dns-40a307.py"] + + @property + def migration_id(self): + return "a643b104-f95d-47cd-aaa8-55a1a0104ba6" + + @property + def is_manual(self): + return False + + def upgrade(self, session): + sql_expressions = [ + # TABLES + """ + DROP TYPE IF EXISTS enum_secret_status; + CREATE TYPE "enum_secret_status" AS ENUM ( + 'NEW', + 'IN_PROGRESS', + 'ACTIVE', + 'ERROR' + ); + """, + """ + CREATE TABLE IF NOT EXISTS secret_passwords ( + "uuid" UUID NOT NULL PRIMARY KEY, + "name" varchar(255) NOT NULL, + "description" varchar(255) NOT NULL, + "project_id" UUID NOT NULL, + "status" enum_secret_status NOT NULL DEFAULT 'NEW', + "constructor" JSONB NOT NULL, + "value" varchar(512) NULL DEFAULT NULL, + "method" varchar(64) NOT NULL, + "created_at" timestamp NOT NULL DEFAULT current_timestamp, + "updated_at" timestamp NOT NULL DEFAULT current_timestamp + ); + """, + """ + CREATE INDEX IF NOT EXISTS secret_passwords_project_id_idx + ON secret_passwords (project_id); + """, + """ + CREATE TABLE IF NOT EXISTS storage_passwords ( + "uuid" UUID NOT NULL PRIMARY KEY, + "status" enum_secret_status NOT NULL DEFAULT 'NEW', + "value" varchar(512) NOT NULL, + "meta" JSONB NOT NULL, + "created_at" timestamp NOT NULL DEFAULT current_timestamp, + "updated_at" timestamp NOT NULL DEFAULT current_timestamp + ); + """, + ] + + for expr in sql_expressions: + session.execute(expr, None) + + def downgrade(self, session): + sql_types = [ + """ + DROP TYPE IF EXISTS enum_secret_status; + """, + ] + + tables = [ + "storage_passwords", + "secret_passwords", + ] + + for table_name in tables: + self._delete_table_if_exists(session, table_name) + + for expr in sql_types: + session.execute(expr, None) + + +migration_step = MigrationStep() diff --git a/requirements.txt b/requirements.txt index 65cb4fc2..7ab16834 100644 --- a/requirements.txt +++ b/requirements.txt @@ -9,7 +9,7 @@ bazooka>=1.3.0,<2.0.0 # Apache-2.0 Jinja2>=3.1.5,<4.0.0 # BSD License (BSD-3-Clause) izulu>=0.50.0,<1.0.0 # MIT License gcl_iam>=0.11.0,<1.0.0 # Apache-2.0 -gcl_sdk>=0.3.0,<1.0.0 # Apache-2.0 +gcl_sdk>=0.4.0,<1.0.0 # Apache-2.0 pyotp>=2.9.0,<3.0.0 # MIT License pyyaml>=6.0.0,<7.0.0 # MIT netaddr>=1.3.0,<2.0.0 # BSD License (BSD License) diff --git a/setup.cfg b/setup.cfg index bd6ac91b..a4913099 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,4 +42,5 @@ gcl_sdk_event_payloads = IamUserRegistration = genesis_core.events.payloads:RegistrationEventPayload IamUserResetPassword = genesis_core.events.payloads:ResetPasswordEventPayload gcl_sdk_universal_agent = - CoreCapabilityDriver = genesis_core.agent.universal.driver:CoreCapabilityDriver \ No newline at end of file + CoreCapabilityDriver = genesis_core.agent.universal.drivers.core:CoreCapabilityDriver + PasswordCapabilityDriver = genesis_core.agent.universal.drivers.secret.password:PasswordCapabilityDriver