diff --git a/.vscode/launch.json b/.vscode/launch.json index 96e7fa68..4fe0fa2f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -29,6 +29,29 @@ "args": ["--config-file", "etc/genesis_core/genesis_core.conf", "--config-dir", "etc/genesis_core/genesis_core.d/"], "console": "integratedTerminal", "justMyCode": false + }, { + "name": "Python Debugger: Manifest Parser", + "type": "debugpy", + "request": "launch", + "program": ".tox/develop/bin/gc-manifest-parser", + "args": [ + "--config-file", "etc/genesis_core/genesis_core.conf", + "--config-dir", "etc/genesis_core/genesis_core.d/", + "--manifest-path", "genesis/manifests/core.element.yaml" + ], + "console": "integratedTerminal", + "justMyCode": false + }, { + "name": "Python Debugger: Genesis Universal Agent", + "type": "debugpy", + "request": "launch", + "program": ".tox/develop/bin/genesis-universal-agent", + "args": [ + "--config-file", "etc/genesis_core/genesis_core.conf", + "--config-dir", "etc/genesis_core/genesis_core.d/", + ], + "console": "integratedTerminal", + "justMyCode": false }, { "name": "Python Debugger: Apply Migrations", "type": "debugpy", diff --git a/genesis/manifests/examples/core.element.yaml b/genesis/manifests/examples/core.element.yaml new file mode 100644 index 00000000..97bcfee5 --- /dev/null +++ b/genesis/manifests/examples/core.element.yaml @@ -0,0 +1,367 @@ +Name: core +Description: Core element. +SchemaVersion: 1 +Version: "0.0.1" + +Requirements: [] + +Resources: + + $core.endpoints: + management: + kind: internal_client + #endpoint: http://localhost:11010/v1/ + + + $core.compute.nodes: + bootstrap: + name: bootstrap + root_disk_size: 20 + cores: 1 + ram: 1024 + iamge: http://192.168.100.31:8081/genesis-base.raw + + + $core.element.services: + migrations: + name: Genesis Core Migrations + description: Genesis Core Migrations service + target: + kind: node + node: $core.compute.nodes.$bootstrap:uuid + controller: + kind: systemd + service: + kind: oneshot + exec_start: ra-apply-migration --config-file /etc/genesis_core/genesis_core.conf --path /opt/genesis_core/migrations/ + + user_api: + name: Genesis Core User Api + description: Genesis Core User API service + target: + kind: node + node: $core.compute.nodes.$bootstrap:uuid + controller: + kind: systemd + service: + kind: daemon + exec_start: gc-user-api --config-file /etc/genesis_core/genesis_core.conf + depends_on: + - $core.element.services.$migrations:get_uri() + + orch_api: + name: Genesis Core Orch Api + description: Genesis Core Orch API service + target: + kind: node + node: $core.compute.nodes.$bootstrap:uuid + controller: + kind: systemd + service: + kind: daemon + exec_start: gc-orch-api --config-file /etc/genesis_core/genesis_core.conf + depends_on: + - $core.element.services.$migrations:get_uri() + + core_agent: + name: Genesis Core Agent + description: Genesis Core Agent + target: + kind: node + node: $core.compute.nodes.$bootstrap:uuid + controller: + kind: systemd + service: + kind: daemon + exec_start: gc-gservice --config-file /etc/genesis_core/genesis_core.conf + depends_on: + - $core.element.services.$migrations:get_uri() + + + $core.iam.users: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + username: "admin" + description: "System administrator" + first_name: "Admin" + last_name: "User" + surname: "" + email: "admin@example.com" + + + $core.iam.organizations: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + name: "admin" + description: "Admin Organization" + genesis_corporation: + uuid: "11111111-1111-1111-1111-111111111111" + name: "Genesis Corporation" + description": "The organization serves as the central platform for all services and elements developed by Genesis Corporation." + + + $core.iam.organization_members: + admin: + uuid: "210a146e-0ddd-4021-85cd-51760d9b0cbd" + organization: "$core.iam.organizations.$admin:get_uri()" + user: "$core.iam.users.$admin:get_uri()" + role: "OWNER" + + + $core.iam.projects: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + name: "admin" + description: "Admin Project" + organization: $core.iam.organizations.$admin:get_uri() + iam_core: + uuid: "c938820c-ab9c-56e4-919c-acd0e881996f" + name: "iam-core" + description: "Identity and Access Management Core Project" + organization: $core.iam.organizations.$genesis_corporation:get_uri() + compute: + uuid: "46f57877-f477-5916-bcd1-718d34cf3414" + name: "compute-core" + description: "Comptue and Baremetal Core Project" + organization: "$core.iam.organizations.$genesis_corporation:get_uri()" + + + $core.iam.permissions: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + name: "*.*.*" + description: "Allow All" + + iam_user_list: + uuid: "a8a7eea2-7514-53ac-a408-8b1235352310" + name: "iam.user.list" + description: "Allows listing users in the system" + iam_user_read_all: + uuid: "f64bbebd-c389-53b1-bed2-c725dc7dd1e8" + name: "iam.user.read_all" + description: "Allows reading all user profiles" + iam_user_write_all: + uuid: "880c3129-34ee-5fae-ab1f-6c8412333d49" + name: "iam.user.write_all" + description: "Allows modifying any user`s data" + iam_user_delete_all: + uuid: "e25b27ac-85c1-50c9-b0a9-696048649c28" + name: "iam.user.delete_all" + description: "Allows deleting any user account" + iam_user_delete: + uuid: "bccfb7f4-9775-535b-8967-1d42c5d213f9" + name: "iam.user.delete" + description: "Allows users to delete their own account" + + iam_organization_create: + uuid: "51330c56-c241-520f-8b83-234a0ab7ba1d" + name: "iam.organization.create" + description: "Allows creating new organizations" + iam_organization_read_all: + uuid: "c228157c-7923-59bf-8177-ab12c1e6f9c7" + name: "iam.organization.read_all" + description: "Allows viewing all organization details" + iam_organization_write_all: + uuid: "87c5691b-3f77-50eb-853b-abb0b4ff4a82" + name: "iam.organization.write_all" + description: "Allows modifying any organization`s data" + iam_organization_delete: + uuid: "eea5828b-f702-5286-80b5-8afcc75cee71" + name: "iam.organization.delete" + description: "Allows deleting own organization" + iam_organization_delete_all: + uuid: "28da4c50-4487-5514-8131-dd898a7e0abd" + name: "iam.organization.delete_all" + description: "Allows deleting any organization" + + iam_project_list_all: + uuid: "eb8f0274-61eb-52ed-b054-18cd08b4ec99" + name: "iam.project.list_all" + description: "Allows listing projects in the system" + iam_project_read_all: + uuid: "626d4b57-3cc7-56d6-bf30-7c2ada04d6b0" + name: "iam.project.read_all" + description: "Allows reading all project details" + iam_project_write_all: + uuid: "c6c21054-6d17-5d67-b981-0ce0ef20e4e2" + name: "iam.project.write_all" + description: "Allows modifying any project`s data" + iam_project_delete_all: + uuid: "654071ab-12af-52c0-8391-ebbb582a0423" + name: "iam.project.delete_all" + description: "Allows deleting any project" + + iam_permission_create: + uuid: "d9727cf5-6857-55dc-8030-d13693cc15df" + name: "iam.permission.create" + description: "Allows creating new permissions" + iam_permission_read: + uuid: "98938bd5-b26b-5733-931b-471504c27c1c" + name: "iam.permission.read" + description: "Allows reading permissions" + iam_permission_update: + uuid: "9c540d33-dfd1-565c-82b4-3099ff277c6c" + name: "iam.permission.update" + description: "Allows updating existing permissions" + iam_permission_delete: + uuid: "03ca3f7f-9ef9-5169-a147-85fa0ff6a28d" + name: "iam.permission.delete" + description: "Allows deleting permissions" + + iam_permission_binding_create: + uuid: "9c60333d-ab15-504c-af52-095a752237e2" + name: "iam.permission_binding.create" + description: "Allows creating permission bindings" + iam_permission_binding_read: + uuid: "e56faea9-9c58-5ab2-b508-6f467117c9f9" + name: "iam.permission_binding.read" + description: "Allows reading permission bindings" + iam_permission_binding_update: + uuid: "f4c9feb1-a5fc-5a92-b75b-eb47821514e2" + name: "iam.permission_binding.update" + description: "Allows updating permission bindings" + iam_permission_binding_delete: + uuid: "81a119f2-3d32-5ab9-908b-f9b63a9cc8e6" + name: "iam.permission_binding.delete" + description: "Allows deleting permission bindings" + + iam_role_create: + uuid: "5d3e126d-85be-5efb-9200-b7655db64fb4" + name: "iam.role.create" + description: "Allows creating new roles" + iam_role_read: + uuid: "4737a4f2-dd9e-584c-9cb2-3927364481db" + name: "iam.role.read" + description: "Allows reading roles" + iam_role_write: + uuid: "b9d6d6ee-8310-58a2-a25f-ca972a7e009f" + name: "iam.role.write" + description: "Allows updating existing roles" + iam_role_delete: + uuid: "131ccaac-c6e5-5dfe-8ec2-4eeca913c938" + name: "iam.role.delete" + description: "Allows deleting roles" + + iam_role_binding_create: + uuid: "4c6a94fc-e18f-5d0e-8ada-daaf5d470dd2" + name: "iam.role_binding.create" + description: "Allows creating role bindings" + iam_role_binding_read: + uuid: "ff96b360-0f40-5aa4-9396-5de7aba31cb1" + name: "iam.role_binding.read" + description: "Allows reading role bindings" + iam_role_binding_update: + uuid: "91f74ad1-1359-5b5e-b4ff-62a580caa746" + name: "iam.role_binding.update" + description: "Allows updating role bindings" + iam_role_binding_delete: + uuid: "d72c8adf-737b-5bbe-a026-61834dc309b6" + name: "iam.role_binding.delete" + description: "Allows deleting role bindings" + + iam_client_create: + uuid: "29eeff9b-d316-590d-916a-766238fd669d" + name: "iam.iam_client.create" + description: "Allows creating IAM clients" + iam_client_read_all: + uuid: "4435daf0-0d20-5a00-adc8-90fd9a298147" + name: "iam.iam_client.read_all" + description: "Allows reading all IAM clients" + iam_client_update: + uuid: "6d891b35-d0d9-5327-8b85-aa7a191b69c7" + name: "iam.iam_client.update" + description: "Allows updating IAM clients" + iam_client_delete: + uuid: "9da293c0-1ba8-548a-a490-287d0266b507" + name: "iam.iam_client.delete" + description: "Allows deleting IAM clients" + + compute_node_read: + uuid: "62dc01b3-f556-5ba3-bcd6-709c52d7418c" + name: "compute.node.read" + description: "List and read own nodes" + compute_node_create: + uuid: "086988df-a84b-5c42-8f3a-1fd3d0dd3878" + name: "compute.node.create" + description: "Create own nodes" + compute_node_update: + uuid: "44b6c954-867f-558b-9dba-b8c7baf9df38" + name: "compute.node.update" + description: "Update own nodes" + compute_node_delete: + uuid: "31eff665-2e96-5202-873a-a2f3c812a281" + name: "compute.node.delete" + description: "Delete own nodes" + + + $core.iam.roles: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + name: "admin" + description: "Admin Role" + newcomer: + uuid: "726f6c65-0000-0000-0000-000000000001" + name: "newcomer" + description: "Default role for newly registered users. Provides basic system access and onboarding capabilities." + owner: + uuid: "726f6c65-0000-0000-0000-000000000002" + name: "owner" + description: "Project ownership role. Grants full administrative privileges within a specific project. Automatically assigned during project creation process." + + + $core.iam.permission_bindings: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + role: "$core.iam.roles.$admin:get_uri()" + permission: "$core.iam.permissions.$admin:get_uri()" + newcomer_iam_user_delete: + uuid: "82870529-2614-49ef-b583-7f2100d13c50" + project_id: "$core.iam.projects.$iam_core:uuid" + role: "$core.iam.roles.$newcomer:get_uri()" + permission: "$core.iam.permissions.$iam_user_delete:get_uri()" + newcomer_iam_organization_create: + uuid: "1f7d5e14-6f66-4bfe-a825-c669c8409021" + project_id: "$core.iam.projects.$iam_core:uuid" + role: "$core.iam.roles.$newcomer:get_uri()" + permission: "$core.iam.permissions.$iam_organization_create:get_uri()" + newcomer_iam_organization_delete: + uuid: "4e68d259-9c70-48b9-be23-55961b332338" + project_id: "$core.iam.projects.$iam_core:uuid" + role: "$core.iam.roles.$newcomer:get_uri()" + permission: "$core.iam.permissions.$iam_organization_delete:get_uri()" + owner_compute_node_read: + uuid: "82f1a5a6-f03b-4503-8aa8-dd59fed64097" + project_id: "$core.iam.projects.$compute:uuid" + role: "$core.iam.roles.$owner:get_uri()" + permission: "$core.iam.permissions.$compute_node_read:get_uri()" + owner_compute_node_create: + uuid: "b916340d-db13-4cd3-90ea-a410bcb99270" + project_id: "$core.iam.projects.$compute:uuid" + role: "$core.iam.roles.$owner:get_uri()" + permission: "$core.iam.permissions.$compute_node_create:get_uri()" + owner_compute_node_update: + uuid: "5fa0e254-f7ff-4652-8d4a-70b6e54fcabf" + project_id: "$core.iam.projects.$compute:uuid" + role: "$core.iam.roles.$owner:get_uri()" + permission: "$core.iam.permissions.$compute_node_update:get_uri()" + owner_compute_node_delete: + uuid: "a500cddf-0431-4766-ad01-f09d0ee39d57" + project_id: "$core.iam.projects.$compute:uuid" + role: "$core.iam.roles.$owner:get_uri()" + permission: "$core.iam.permissions.$compute_node_delete:get_uri()" + + + $core.iam.role_bindings: + admin: + uuid: "00000000-0000-0000-0000-000000000000" + user: "$core.iam.users.$admin:get_uri()" + role: "$core.iam.roles.$admin:get_uri()" + + + $core.iam.clients: + genesis_core: + uuid: "00000000-0000-0000-0000-000000000000" + name: "GenesisCoreClient" + description: "Genesis Core OIDC Client" + client_id: "GenesisCoreClientId" + redirect_url: "http://127.0.0.1:11010/v1/" diff --git a/genesis_core/cmd/manifest_parser.py b/genesis_core/cmd/manifest_parser.py new file mode 100644 index 00000000..6b19a1bd --- /dev/null +++ b/genesis_core/cmd/manifest_parser.py @@ -0,0 +1,72 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import sys + +from oslo_config import cfg +from restalchemy.storage.sql import engines +from restalchemy.common import config_opts as ra_config_opts + +from genesis_core.common import config +from genesis_core.common import log as infra_log +from genesis_core.elements.dm import models + + +cmd_opts = [ + cfg.StrOpt( + "manifest-path", + required=True, + help="Path to the manifest file", + ), +] + + +CONF = cfg.CONF +ra_config_opts.register_posgresql_db_opts(CONF) +CONF.register_cli_opts(cmd_opts) + + +def main() -> None: + # Parse config + config.parse(sys.argv[1:]) + + infra_log.configure() + log = logging.getLogger(__name__) + + engines.engine_factory.configure_postgresql_factory(CONF) + + element_engine = models.element_engine + element_engine.load_element_from_manifest_file(CONF.manifest_path) + print(element_engine._namespaces) + print( + "Manifest:", + element_engine._namespaces["$core"] + ._namespace_resources["$core.v1.iam.projects.$compute"] + .manifest_state, + ) + print( + "Render:", + element_engine._namespaces["$core"] + ._namespace_resources["$core.v1.iam.projects.$compute"] + .render_target_state(), + ) + + log.debug("GoodBye!") + + +if __name__ == "__main__": + main() diff --git a/genesis_core/elements/__init__.py b/genesis_core/elements/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/elements/dm/__init__.py b/genesis_core/elements/dm/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/elements/dm/models.py b/genesis_core/elements/dm/models.py new file mode 100644 index 00000000..4bcdbbeb --- /dev/null +++ b/genesis_core/elements/dm/models.py @@ -0,0 +1,720 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import enum +import yaml +import uuid as sys_uuid + +from gcl_sdk.agents.universal.dm import models as sdk_models +from gcl_sdk.agents.universal import utils as sdk_utils +from restalchemy.dm import filters as ra_filters +from restalchemy.dm import models +from restalchemy.dm import properties +from restalchemy.dm import relationships +from restalchemy.dm import types as ra_types +from restalchemy.storage.sql import orm +from restalchemy.storage.sql import engines + +from genesis_core.elements.dm import utils + + +LOG = logging.getLogger(__name__) + + +class Status(str, enum.Enum): + NEW = "NEW" + IN_PROGRESS = "IN_PROGRESS" + ACTIVE = "ACTIVE" + + +class AlwaysActiveStatus(str, enum.Enum): + ACTIVE = "ACTIVE" + + +class InstallTypes(str, enum.Enum): + MANUAL = "MANUAL" + AUTO_AS_DEPENDENCY = "AUTO_AS_DEPENDENCY" + + +class Manifest( + models.ModelWithUUID, + models.ModelWithRequiredNameDesc, + models.ModelWithTimestamp, + orm.SQLStorableMixin, +): + __tablename__ = "em_manifests" + + STATUS = AlwaysActiveStatus + + status = properties.property( + ra_types.Enum([s for s in Status]), + default=STATUS.ACTIVE, + ) + version = properties.property( + ra_types.String(min_length=5, max_length=64), + read_only=True, + required=True, + ) + schema_version = properties.property( + ra_types.Integer(min_value=1, max_value=1), + read_only=True, + default=1, + ) + project_id = properties.property( + ra_types.UUID(), + read_only=True, + default=sys_uuid.UUID("12345678-cd58-4c33-a0c3-23a1086a53b7"), + ) + requirements = properties.property( + ra_types.Dict(), + read_only=True, + mutable=True, + default=dict, + ) + resources = properties.property( + ra_types.Dict(), + read_only=True, + mutable=True, + default=dict, + ) + + def install(self): + element = Element( + uuid=utils.get_element_uuid(self.name, self.version), + name=self.name, + version=self.version, + description=self.description, + project_id=utils.get_project_id(), + ) + element.insert() + # prepare resources: + for resource_link_prefix, resource in self.resources.items(): + for resource_name, resource_value in resource.items(): + uuid = sys_uuid.UUID( + resource_value.get("uuid", str(sys_uuid.uuid4())) + ) + resource = Resource( + uuid=uuid, + name=resource_name, + element=element, + resource_link_prefix=resource_link_prefix, + value=resource_value, + ) + resource.insert() + return self + + def uninstall(self): + elements = Element.objects.get_all( + filters={ + "uuid": ra_filters.EQ( + utils.get_element_uuid(self.name, self.version) + ), + } + ) + for element in elements: + element.delete() + return self + + +class Element( + models.ModelWithUUID, + models.ModelWithRequiredNameDesc, + models.ModelWithTimestamp, + models.CustomPropertiesMixin, + orm.SQLStorableMixin, +): + __tablename__ = "em_elements" + __custom_properties__ = { + "link": ra_types.String(), + } + + STATUSES = Status + INSTALL_TYPES = InstallTypes + + status = properties.property( + ra_types.Enum([s for s in STATUSES]), + default=STATUSES.NEW, + ) + + version = properties.property( + ra_types.String(min_length=5, max_length=64), + required=True, + ) + + install_type = properties.property( + ra_types.Enum([s for s in INSTALL_TYPES]), + default=INSTALL_TYPES.MANUAL, + ) + + @property + def link(self): + return f"${self.name}" + + # def get_requirements(self): + # return self._requirements + + # def _add_requirement_to_list(self, requirement, list_of_requirements): + # for req in list_of_requirements: + # if req.name == requirement.name: + # raise exceptions.ConflictRequirement(requirement=requirement) + # list_of_requirements.append(requirement) + # return list_of_requirements + + # def add_requirement(self, requirement): + # self._requirements = self._add_requirement_to_list( + # requirement, + # self._requirements, + # ) + + # def add_requirements(self, requirements): + # copy_requirements = self._requirements + # for req in requirements: + # copy_requirements = self._add_requirement_to_list( + # req, + # copy_requirements, + # ) + # self._requirements = copy_requirements + + # def get_resources(self): + # return self._resources + + def __init__(self, **kwargs): + super().__init__(**kwargs) + self._requirements = [] + + +class ElementIncorrectStatusesView( + models.ModelWithUUID, + orm.SQLStorableMixin, +): + __tablename__ = "em_incorrect_statuses_view" + + STATUS = Status + + name = properties.property( + ra_types.String(min_length=1, max_length=255), + read_only=True, + ) + api_status = properties.property( + ra_types.Enum([s for s in Status]), + read_only=True, + ) + actual_status = properties.property( + ra_types.Enum([s for s in Status]), + read_only=True, + ) + + def actualize_status(self): + engine = engines.engine_factory.get_engine() + with engine.session_manager() as s: + s.execute( + f""" + UPDATE {Element.__tablename__} + SET status = %s + WHERE uuid = %s; + """, + ( + self.actual_status, + self.uuid, + ), + ) + + +class Requirement( + models.ModelWithUUID, + orm.SQLStorableMixin, +): + __tablename__ = "em_requirements" + + element = relationships.relationship( + Element, + prefetch=True, + required=True, + ) + name = properties.property( + ra_types.String(min_length=1, max_length=64), + required=True, + ) + from_version = properties.property( + ra_types.AllowNone(ra_types.String(min_length=5, max_length=64)), + default=None, + ) + to_version = properties.property( + ra_types.AllowNone(ra_types.String(min_length=5, max_length=64)), + default=None, + ) + + +class Resource( + models.ModelWithUUID, + models.ModelWithTimestamp, + models.CustomPropertiesMixin, + orm.SQLStorableMixin, +): + __tablename__ = "em_resources" + __custom_properties__ = { + # "project_id": ra_types.UUID(), + # "target_state": ra_types.Dict(), + # "target_hash": ra_types.String(min_length=32, max_length=32), + # "actual_hash": ra_types.String(min_length=32, max_length=32), + "link": ra_types.String(min_length=2, max_length=256), + "kind": ra_types.String(min_length=2, max_length=256), + } + __allowed_methods_from_manifest__ = [ + "get_uri", + ] + + element = relationships.relationship( + Element, + prefetch=True, + required=True, + ) + name = properties.property( + ra_types.String(min_length=1, max_length=255), + read_only=True, + ) + status = properties.property( + ra_types.Enum(Status), + default=Status.NEW, + ) + resource_link_prefix = properties.property( + ra_types.String(min_length=1, max_length=256), + required=True, + ) + value = properties.property( + ra_types.Dict(), + required=True, + mutable=True, + ) + target_resource = relationships.relationship( + sdk_models.TargetResource, + default=None, + prefetch=True, + ) + actual_resource = relationships.relationship( + sdk_models.Resource, + default=None, + prefetch=True, + ) + full_hash = properties.property( + ra_types.String(max_length=256), + default="", + ) + + def get_uri(self): + parts = self.link.split(".") + return f"/{"/".join(parts[1:-1])}/{self.uuid}" + + def get_parameter_value(self, parameter): + parts = parameter.split(":") + resource_name = parts[0][1:] + if resource_name != self.name: + raise ValueError( + f"Resource name `{resource_name}` does not match the" + f" current resource name `{self.name}`" + ) + resource_parameter_path = parts[1:] + if len(resource_parameter_path) == 0: + return self.actual_resource.value + elif len(resource_parameter_path) == 1: + if resource_parameter_path[0][-2:] == "()": + func_name = resource_parameter_path[0][:-2] + if func_name in self.__allowed_methods_from_manifest__: + func = getattr(self, func_name) + return func() + + result_value = self.get_actual_state_safe() + for part in resource_parameter_path: + result_value = result_value[part] + return result_value + + def get_actual_state_safe(self): + if self.actual_resource is None: + return self.render_target_state() + return self.actual_resource.value + + def render_target_state(self, engine=None): + + engine = engine or element_engine + + def render_value(value): + if value.startswith("$"): + link = utils.ResourceLink(value) + try: + resource = engine.get_resource_by_link( + element=self.element, + link=link.location, + ) + return resource.get_parameter_value( + parameter=link.parameter, + ) + except ValueError as e: + raise ValueError( + f"Can't render value `{value}` for resource" + f" `{repr(self)}` by reason: {e}" + ) + return value + + def recursive_render(data): + if isinstance(data, dict): + result = {} + for key, value in data.items(): + if isinstance( + value, + ( + dict, + list, + ), + ): + value = recursive_render(value) + result[key] = value + elif isinstance(value, str): + result[key] = render_value(value) + else: + result[key] = value + elif isinstance(data, list): + result = [] + for item in data: + if isinstance( + item, + ( + dict, + list, + ), + ): + item = recursive_render(item) + result.append(item) + elif isinstance(item, str): + result.append(render_value(item)) + else: + result.append(item) + else: + result = data + return result + + return recursive_render(self.value) + + @property + def link(self): + return f"{self.resource_link_prefix}.${self.name}" + + @property + def kind(self): + parts = [p for p in self.resource_link_prefix.split(".")[1:]] + return f"em_{self.element.name}_{'_'.join(parts)}" + + def calculate_full_hash(self): + if self.actual_resource is not None: + return self.actual_resource.full_hash + target_state = self.render_target_state() + return sdk_utils.calculate_hash(target_state) + + def _find_actual_resource(self): + if self.actual_resource is None: + for actual_resource in sdk_models.Resource.objects.get_all( + filters={ + "uuid": ra_filters.EQ(self.uuid), + "kind": ra_filters.EQ(self.kind), + }, + ): + return actual_resource + return None + + def actualize(self): + try: + target_state = self.render_target_state() + except KeyError as e: + LOG.debug( + "Target state is not available for resource %s by reason: %r", + self, + str(e), + ) + return + self.actual_resource = self._find_actual_resource() + hash = sdk_utils.calculate_hash(target_state) + self.full_hash = self.calculate_full_hash() + if self.target_resource is None: + target_resource = sdk_models.TargetResource( + uuid=self.uuid, + kind=self.kind, + value=target_state, + hash=hash, + full_hash=self.full_hash, + tracked_at=self.updated_at, + ) + target_resource.insert() + self.target_resource = target_resource + self.update() + LOG.debug("Target resource %s has been created.", target_resource) + elif self.target_resource.hash != hash: + self.target_resource.value = target_state + self.target_resource.calculate_hash() + self.target_resource.full_hash = self.full_hash + self.target_resource.tracked_at = self.updated_at + self.target_resource.update() + LOG.debug( + "Target resource %s has been updated.", + self.target_resource, + ) + elif self.target_resource.full_hash != self.full_hash: + self.target_resource.full_hash = self.full_hash + self.target_resource.update() + LOG.debug( + "Target resource %s full hash has been updated.", + self.target_resource, + ) + elif self.target_resource.tracked_at != self.updated_at: + self.target_resource.tracked_at = self.updated_at + self.target_resource.update() + LOG.debug( + "Target resource %s tracked_at has been updated.", + self.target_resource, + ) + else: + LOG.debug( + "Target resource %s is actual state.", + self.target_resource, + ) + self.update() + + def delete(self, session=None): + for ts in sdk_models.TargetResource.objects.get_all( + filters={"uuid": ra_filters.EQ(self.uuid)} + ): + ts.delete(session=session) + super().delete(session=session) + + +class OutdatedResources(models.ModelWithUUID, orm.SQLStorableMixin): + + __tablename__ = "em_outdated_resources_view" + + em_resource = relationships.relationship( + Resource, + default=None, + prefetch=True, + ) + target_resource = relationships.relationship( + sdk_models.TargetResource, + default=None, + prefetch=True, + ) + + +class ResourceIncorrectStatusesView( + models.ModelWithUUID, orm.SQLStorableMixin +): + __tablename__ = "em_incorrect_resource_statuses_view" + + current_status = properties.property( + ra_types.String(), + required=True, + ) + actual_status = properties.property( + ra_types.AllowNone(ra_types.String()), + default=None, + ) + + def actualize_status(self, session): + new_status = Status.NEW + if self.actual_status == Status.ACTIVE: + new_status = Status.ACTIVE + elif self.actual_status is not None: + new_status = Status.IN_PROGRESS + + session.execute( + f'UPDATE {Resource.__tablename__} SET status=%s WHERE "uuid"=%s;', + (new_status, self.uuid), + ) + + +class Namespace: + + def __init__(self, element): + super().__init__() + self._element = element + # NOTE(efrolov): map of resources by link string + self._namespace_resources = {} + + def add_resource(self, resource): + if resource.link in self._namespace_resources: + raise ValueError( + f"Resource with link string '{resource.link}' already exists." + ) + self._namespace_resources[resource.link] = resource + + def get_resources(self): + return list(self._namespace_resources.values()) + + def delete_resource(self, resource): + if resource.link not in self._namespace_resources: + raise ValueError( + f"Resource with link string '{resource.link}' does not exist." + ) + del self._namespace_resources[resource.link] + + def get_resource_by_link(self, link): + clear_link = utils.clear_parameters(link) + if clear_link not in self._namespace_resources: + raise ValueError( + f"Resource with link string '{clear_link}' does not exist." + ) + return self._namespace_resources[clear_link] + + +class BaseManifestParser: + SCHEMA_VERSION = 0 + + def __init__(self, manifest): + super().__init__() + self._manifest = manifest + + def load_element_from_manifest(self, manifest): + raise NotImplementedError("Subclasses must implement this method.") + + +class ManifestParserV1(BaseManifestParser): + SCHEMA_VERSION = 1 + + def load_element( + self, + ): + name = utils.get_required_field(self._manifest, "Name") + version = utils.get_required_field(self._manifest, "Version") + description = self._manifest.get("Description", "") + uuid = utils.get_element_uuid(name, version) + project_id = utils.get_project_id() + element = Element( + uuid=uuid, + name=name, + version=version, + description=description, + project_id=project_id, + ) + return element + + def load_resources(self, element): + result = [] + data = self._manifest.get("Resources", {}) + for resource_link_prefix, resource_data in data.items(): + for resource_name, resource_data in resource_data.items(): + resource_uuid = resource_data.get( + "uuid", str(sys_uuid.uuid4()) + ) + project_id = resource_data.get( + "project_id", str(element.project_id) + ) + raw_properties = resource_data.copy() + raw_properties.update( + { + "uuid": resource_uuid, + "project_id": project_id, + } + ) + resource = Resource( + uuid=sys_uuid.UUID(resource_uuid), + element=element, + resource_link_prefix=resource_link_prefix, + name=resource_name, + manifest_state=raw_properties, + ) + result.append(resource) + return result + + +class ElementEngine: + + MANIFEST_PARSER_MAP = { + ManifestParserV1.SCHEMA_VERSION: ManifestParserV1, + } + + def __init__(self): + super().__init__() + self._namespaces = {} + + def load_element_from_manifest(self, manifest): + schema_version = utils.get_required_field(manifest, "SchemaVersion") + manifest_parser_class = self.MANIFEST_PARSER_MAP.get( + schema_version, + BaseManifestParser, + ) + manifest_parser = manifest_parser_class(manifest=manifest) + element = manifest_parser.load_element() + self.add_element(element) + for resource in manifest_parser.load_resources(element): + self.add_resource(resource) + + def load_from_database(self): + self._namespaces = {} + for element in Element.objects.get_all(): + self.add_element(element) + + for resource in Resource.objects.get_all(): + self.add_resource(resource) + + def load_element_from_manifest_file(self, manifest_file_path): + with open(manifest_file_path, "r") as file: + manifest = yaml.safe_load(file) + self.load_element_from_manifest(manifest) + + def add_resource(self, resource): + element = resource.element + if element.link not in self._namespaces: + ValueError( + f"The element '{element}' is unknown. Please add the element" + " before adding resources to it." + ) + namespace = self._namespaces[resource.element.link] + namespace.add_resource(resource) + + def get_resources(self): + result = [] + for namespace in self._namespaces.values(): + result.extend(namespace.get_resources()) + return result + + def get_resource_by_link(self, element, link): + if element.link not in self._namespaces: + raise ValueError( + f"Can't load element {element}. Element" + f" {self._namespaces[element.link].element} is not found." + ) + namespace = self._namespaces[element.link] + return namespace.get_resource_by_link(link) + + def add_element(self, element): + if element.link in self._namespaces: + raise ValueError( + f"Can't load element {element}. Element" + f" {self._namespaces[element.link].element} already exists" + " with the same UUID." + ) + self._namespaces[element.link] = Namespace(element) + + def remove_element(self, element): + if element.link not in self._namespaces: + raise ValueError( + f"Can't remove element {element}. Element does not exist." + ) + + del self._namespaces[element.link] + + def save_to_database(self): + pass + + +element_engine = ElementEngine() diff --git a/genesis_core/elements/dm/utils.py b/genesis_core/elements/dm/utils.py new file mode 100644 index 00000000..e99f4622 --- /dev/null +++ b/genesis_core/elements/dm/utils.py @@ -0,0 +1,67 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import uuid as sys_uuid + + +ELEMENT_NAMESPACE = sys_uuid.UUID("f277e88a-cd58-4c33-a0c3-23a1086a53b7") +UUID_PREFIX = "12345678" + + +def clear_parameters(resource_link): + parts = resource_link.split(".") + return ".".join(parts[:-1] + [parts[-1].split(":")[0]]) + + +class ResourceLink: + + def __init__(self, link_string): + super().__init__() + self._link_string = link_string + + @property + def location(self): + return clear_parameters(self._link_string) + + @property + def parameter(self): + return self._link_string.split(".")[-1] + + +def get_element_uuid(element_name, element_version): + tmp = str( + sys_uuid.uuid5( + ELEMENT_NAMESPACE, + f"{element_name}-{element_version}", + ) + ) + return sys_uuid.UUID(f"{UUID_PREFIX}{tmp[8:]}") + + +def get_project_id(): + # return sys_uuid.UUID(f"{UUID_PREFIX}{str(sys_uuid.uuid4())[8:]}") + return sys_uuid.UUID( + f"{UUID_PREFIX}{str("00000000-0000-0000-0000-000000000000")[8:]}" + ) + + +def get_required_field(data, field_name): + if field_name not in data: + raise ValueError( + f"required field '{field_name}' is missing in the provided" + f" data: {data}" + ) + return data[field_name] diff --git a/genesis_core/elements/services/__init__.py b/genesis_core/elements/services/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/elements/services/builders.py b/genesis_core/elements/services/builders.py new file mode 100644 index 00000000..ddbdb664 --- /dev/null +++ b/genesis_core/elements/services/builders.py @@ -0,0 +1,81 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging + +from gcl_looper.services import basic +from restalchemy.common import contexts + +from genesis_core.elements.dm import models + + +LOG = logging.getLogger(__name__) + + +class ElementManagerBuilder(basic.BasicService): + + def __init__( + self, + iter_min_period: int = 1, + iter_pause: float = 0.1, + ): + super().__init__(iter_min_period, iter_pause) + self._element_engine = models.element_engine + self._first_step = True + + def _actualize_target_resources(self): + # Delete outdated resources that do not have a corresponding EM + for info in models.OutdatedResources.objects.get_all(): + if info.em_resource is None: + info.target_resource.delete() + LOG.info(" Resource %s has been deleted", info.target_resource) + + for resource in self._element_engine.get_resources(): + resource.actualize() + + def _actualize_statuses(self, session): + incorrect_resource_statuses = ( + models.ResourceIncorrectStatusesView.objects.get_all() + ) + for em_status_model in incorrect_resource_statuses: + LOG.info( + "Actualizing status for resource (%s): %s -> %s...", + em_status_model.uuid, + em_status_model.current_status, + em_status_model.actual_status, + ) + em_status_model.actualize_status(session) + + incorrect_element_statuses = ( + models.ElementIncorrectStatusesView.objects.get_all() + ) + for em_status_model in incorrect_element_statuses: + LOG.info( + "Actualizing status for element (%s): %s -> %s...", + em_status_model.name, + em_status_model.api_status, + em_status_model.actual_status, + ) + em_status_model.actualize_status() + + def _iteration(self): + with contexts.Context().session_manager() as session: + if self._first_step: + self._element_engine.load_from_database() + self._first_step = True # Disable optimization + self._actualize_target_resources() + self._actualize_statuses(session) + LOG.debug("Starting iteration") diff --git a/genesis_core/gservice/service.py b/genesis_core/gservice/service.py index 6907ff79..38b0a6f7 100644 --- a/genesis_core/gservice/service.py +++ b/genesis_core/gservice/service.py @@ -19,6 +19,7 @@ from gcl_looper.services import basic from gcl_sdk.events.services import senders +from genesis_core.elements.services import builders as em_builders from genesis_core.node.scheduler.driver.filters import available from genesis_core.node.scheduler.driver.weighter import relative from genesis_core.node.scheduler import service as n_scheduler_service @@ -72,6 +73,9 @@ def __init__(self, iter_min_period=1, iter_pause=0.1): ) cfg_service = config_service.ConfigService() event_sender = senders.EventSenderService.build_from_config() + em_builder = em_builders.ElementManagerBuilder( + iter_min_period=1, iter_pause=0.1 + ) self._services = [ n_scheduler, @@ -80,6 +84,7 @@ def __init__(self, iter_min_period=1, iter_pause=0.1): n_machine, cfg_service, event_sender, + em_builder, ] def _setup(self): diff --git a/genesis_core/tests/functional/utils.py b/genesis_core/tests/functional/utils.py index f905f0cb..f67e5311 100644 --- a/genesis_core/tests/functional/utils.py +++ b/genesis_core/tests/functional/utils.py @@ -75,14 +75,6 @@ def apply_migrations( return migrations def apply_all_migrations(self) -> None: - - # SDK migrations - self._sdk_migration = self.apply_migrations( - migration_path=sdk_migrations.__path__[0], - first_migration=sdk_conftest.FIRST_MIGRATION, - ) - - # Service migrations self._migration = self.apply_migrations( migration_path=os.path.join( os.path.dirname(os.path.abspath(__file__)), @@ -179,4 +171,3 @@ def teardown_method(self) -> None: # Rollback migrations self._migration.rollback_migration(self.__FIRST_MIGRATION__) - self._sdk_migration.rollback_migration(sdk_conftest.FIRST_MIGRATION) diff --git a/genesis_core/user_api/api/routes.py b/genesis_core/user_api/api/routes.py index 4743ffba..eef98e3a 100644 --- a/genesis_core/user_api/api/routes.py +++ b/genesis_core/user_api/api/routes.py @@ -17,6 +17,7 @@ from restalchemy.api import routes from genesis_core.user_api.api import controllers +from genesis_core.user_api.em.api import routes as em_routes from genesis_core.user_api.iam.api import routes as iam_routes from genesis_core.user_api.config.api import routes as config_routes @@ -61,6 +62,7 @@ class ApiEndpointRoute(routes.Route): health = routes.route(HealthRoute) iam = routes.route(iam_routes.IamRoute) + em = routes.route(em_routes.ElementManagerRoute) config = routes.route(config_routes.ConfigRoute) nodes = routes.route(NodeRoute) machines = routes.route(MachineRoute) diff --git a/genesis_core/user_api/em/__init__.py b/genesis_core/user_api/em/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/user_api/em/api/__init__.py b/genesis_core/user_api/em/api/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/genesis_core/user_api/em/api/controllers.py b/genesis_core/user_api/em/api/controllers.py new file mode 100644 index 00000000..3cfb8b9c --- /dev/null +++ b/genesis_core/user_api/em/api/controllers.py @@ -0,0 +1,107 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +from gcl_iam import controllers as iam_controllers +from restalchemy.api import actions +from restalchemy.api import controllers +from restalchemy.api import resources + +from genesis_core.elements.dm import models + + +class ElementManagerController(controllers.RoutesListController): + __TARGET_PATH__ = "/v1/em/" + + +class ManifestController( + iam_controllers.PolicyBasedWithoutProjectController, + controllers.BaseResourceControllerPaginated, +): + __policy_service_name__ = "em" + __policy_name__ = "manifest" + __resource__ = resources.ResourceByRAModel( + model_class=models.Manifest, + convert_underscore=False, + hidden_fields=resources.HiddenFieldMap( + update=["status", "created_at", "updated_at"], + ), + ) + + @actions.post + def install(self, resource): + return resource.install() + + @actions.post + def uninstall(self, resource): + return resource.uninstall() + + +class ElementController( + iam_controllers.PolicyBasedWithoutProjectController, + controllers.BaseResourceControllerPaginated, +): + __policy_service_name__ = "em" + __policy_name__ = "element" + __resource__ = resources.ResourceByModelWithCustomProps( + model_class=models.Element, + convert_underscore=False, + hidden_fields=resources.HiddenFieldMap( + update=[ + "status", + "created_at", + "updated_at", + "name", + "version", + ], + ), + ) + + +class ElementResourceController( + # iam_controllers.PolicyBasedWithoutProjectController, # nested + controllers.BaseNestedResourceControllerPaginated, +): + __pr_name__ = "element" + # __policy_service_name__ = "em" + # __policy_name__ = "resource" + __resource__ = resources.ResourceByModelWithCustomProps( + model_class=models.Resource, + convert_underscore=False, + hidden_fields=resources.HiddenFieldMap( + filter=[ + "target_resource", + "actual_resource", + ], + get=[ + "target_resource", + "actual_resource", + ], + create=[ + "target_resource", + "actual_resource", + ], + update=[ + "target_resource", + "actual_resource", + "status", + "created_at", + "updated_at", + "name", + "version", + ], + ), + ) diff --git a/genesis_core/user_api/em/api/routes.py b/genesis_core/user_api/em/api/routes.py new file mode 100644 index 00000000..304df91e --- /dev/null +++ b/genesis_core/user_api/em/api/routes.py @@ -0,0 +1,64 @@ +# Copyright 2025 Genesis Corporation. +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from restalchemy.api import routes + +from genesis_core.user_api.em.api import controllers + + +class ManifestInstallActionRoute(routes.Action): + """Handler for /v1/em/manifests//actions/install/invoke endpoint""" + + __controller__ = controllers.ManifestController + + +class ManifestUninstallActionRoute(routes.Action): + """Handler for /v1/em/manifests//actions/uninstall/invoke endpoint""" + + __controller__ = controllers.ManifestController + + +class ManifestRoute(routes.Route): + """Handler for /v1/em/manifests/ endpoint""" + + __controller__ = controllers.ManifestController + + install = routes.action(ManifestInstallActionRoute, invoke=True) + uninstall = routes.action(ManifestUninstallActionRoute, invoke=True) + + +class ElementResourceRoute(routes.Route): + """Handler for /v1/em/elements//resources/ endpoint""" + + __controller__ = controllers.ElementResourceController + + +class ElementRoute(routes.Route): + """Handler for /v1/em/elements// endpoint""" + + __controller__ = controllers.ElementController + + resources = routes.route(ElementResourceRoute, resource_route=True) + + +class ElementManagerRoute(routes.Route): + """Handler for /v1/em/ endpoint""" + + __controller__ = controllers.ElementManagerController + __allow_methods__ = [routes.FILTER] + + manifests = routes.route(ManifestRoute) + elements = routes.route(ElementRoute) diff --git a/migrations/0018-add-elements-76bca4.py b/migrations/0018-add-elements-76bca4.py new file mode 100644 index 00000000..09b8297c --- /dev/null +++ b/migrations/0018-add-elements-76bca4.py @@ -0,0 +1,239 @@ +# Copyright 2016 Eugene Frolov +# +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os + +from gcl_sdk import migrations as sdk_migrations + +from restalchemy.storage.sql import migrations + + +SDK_MIGRATION_FILE_NAME = "0001-universal-agent-391c09.py" + + +LOG = logging.getLogger(__name__) + + +class MigrationEngine(migrations.MigrationEngine): + + def apply_migration(self, migration_name, session): + filename = self.get_file_name(migration_name) + self._init_migration_table(session) + migrations = self._load_migration_controllers(session) + + migration = migrations[filename] + if migration.is_applied(): + LOG.warning("Migration '%s' is already applied", migration.name) + else: + LOG.info("Applying migration '%s'", migration.name) + migrations[filename].apply(session, migrations) + + def rollback_migration(self, migration_name, session): + filename = self.get_file_name(migration_name) + self._init_migration_table(session) + migrations = self._load_migration_controllers(session) + migration = migrations[filename] + if not migration.is_applied(): + LOG.warning("Migration '%s' is not applied", migration.name) + else: + LOG.info("Rolling back migration '%s'", migration.name) + migrations[filename].rollback(session, migrations) + + +class MigrationStep(migrations.AbstarctMigrationStep): + + def __init__(self): + self._depends = ["0017-init-configs-c3bbc6.py"] + + @property + def migration_id(self): + return "76bca4a1-46ab-495e-b956-99bdab62d7c5" + + @property + def is_manual(self): + return False + + def _get_migration_engine(self): + sdk_migration_path = os.path.dirname(sdk_migrations.__file__) + return MigrationEngine(migrations_path=sdk_migration_path) + + def upgrade(self, session): + expressions = [ + """ + CREATE TABLE IF NOT EXISTS "em_manifests" ( + "uuid" UUID PRIMARY KEY, + "name" VARCHAR(255) NOT NULL, + "description" VARCHAR(255) NOT NULL DEFAULT '', + "status" VARCHAR(20) NOT NULL DEFAULT 'NEW' + CHECK ( + status IN ( + 'ACTIVE' + ) + ), + "version" VARCHAR(64) NOT NULL, + "schema_version" INTEGER NOT NULL DEFAULT 1, + "project_id" UUID NOT NULL, + "requirements" JSONB NOT NULL DEFAULT '{}', + "resources" JSONB NOT NULL DEFAULT '{}', + "created_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP(6) NOT NULL DEFAULT NOW() + ); + """, + """ + CREATE TABLE IF NOT EXISTS "em_elements" ( + "uuid" UUID PRIMARY KEY, + "name" VARCHAR(255) NOT NULL, + "description" VARCHAR(255) NOT NULL DEFAULT '', + "status" VARCHAR(20) NOT NULL DEFAULT 'NEW' + CHECK ( + status IN ( + 'NEW', + 'IN_PROGRESS', + 'ACTIVE' + ) + ), + "version" VARCHAR(64) NOT NULL, + "install_type" VARCHAR(20) NOT NULL DEFAULT 'MANUAL' + CHECK ( + install_type IN ( + 'MANUAL', + 'AUTO_AS_DEPENDENCY' + ) + ), + "created_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), + CONSTRAINT unique_em_elements_name_version_idx UNIQUE ( + name, version + ) + ); + """, + """ + CREATE TABLE IF NOT EXISTS "em_resources" ( + "uuid" UUID PRIMARY KEY, + "name" VARCHAR(255) NOT NULL, + "element" UUID NOT NULL REFERENCES em_elements("uuid") + ON DELETE CASCADE, + "status" VARCHAR(20) NOT NULL DEFAULT 'NEW' + CHECK ( + status IN ( + 'NEW', + 'IN_PROGRESS', + 'ACTIVE' + ) + ), + "resource_link_prefix" VARCHAR(256) NOT NULL, + "value" JSONB NOT NULL DEFAULT '{}', + "target_resource" UUID DEFAULT NULL REFERENCES + ua_target_resources("uuid"), + "actual_resource" UUID DEFAULT NULL REFERENCES + ua_actual_resources("uuid"), + "full_hash" VARCHAR(256) NOT NULL DEFAULT '', + "created_at" TIMESTAMP(6) NOT NULL DEFAULT NOW(), + "updated_at" TIMESTAMP(6) NOT NULL DEFAULT NOW() + ); + """, + """ + CREATE OR REPLACE VIEW "em_incorrect_statuses_view" AS + WITH "em_incorrect_resource_statuses" AS ( + WITH "tmp" AS ( + SELECT + "element", + BOOL_OR("status" = 'IN_PROGRESS') AS + "has_in_progress", + BOOL_AND("status" = 'ACTIVE') AS "all_active", + BOOL_AND("status" = 'NEW') AS "all_new", + COUNT(*) AS "resources_count" + FROM "em_resources" + GROUP BY "element" + ) + SELECT + "e"."uuid" "uuid", + "e"."name" "name", + "e"."status" "api_status", + CASE + WHEN "tmp"."resources_count" IS NULL THEN 'ACTIVE' + WHEN "tmp"."has_in_progress" THEN 'IN_PROGRESS' + WHEN "tmp"."all_active" THEN 'ACTIVE' + WHEN "tmp"."all_new" THEN 'NEW' + ELSE 'IN_PROGRESS' + END AS "actual_status" + FROM "em_elements" "e" + LEFT JOIN "tmp" "tmp" ON "e"."uuid" = "tmp"."element" + ) + SELECT * FROM "em_incorrect_resource_statuses" "eis" + WHERE "eis"."api_status" != "eis"."actual_status"; + """, + """ + CREATE OR REPLACE VIEW "em_outdated_resources_view" AS + SELECT + COALESCE("er"."uuid", "utr"."uuid") AS "uuid", + "er"."uuid" AS "em_resource", + "utr"."uuid" AS "target_resource" + FROM "em_resources" "er" + FULL OUTER JOIN "ua_target_resources" "utr" + on "er"."uuid" = "utr"."uuid" + WHERE + "er"."uuid" IS NULL + OR "utr"."uuid" IS NULL + OR "er"."updated_at" <> "utr"."tracked_at"; + """, + """ + CREATE OR REPLACE VIEW "em_incorrect_resource_statuses_view" AS + SELECT + "er"."uuid" AS "uuid", + "er"."status" AS "current_status", + "uar"."status" AS "actual_status" + FROM + "em_resources" "er" + LEFT JOIN + "ua_actual_resources" "uar" + ON + "er"."uuid" = "uar"."uuid" + WHERE + "er"."status" <> "uar"."status"; + """, + ] + + migration_engine = self._get_migration_engine() + migration_engine.apply_migration(SDK_MIGRATION_FILE_NAME, session) + + for expression in expressions: + session.execute(expression) + + def downgrade(self, session): + tables = [ + "em_manifests", + "em_elements", + "em_resources", + ] + views = [ + "em_incorrect_statuses_view", + "em_incorrect_resource_statuses_view", + "em_outdated_resources_view", + ] + + for view in reversed(views): + self._delete_view_if_exists(session, view) + + for table in reversed(tables): + self._delete_table_if_exists(session, table) + + migration_engine = self._get_migration_engine() + migration_engine.rollback_migration(SDK_MIGRATION_FILE_NAME, session) + + +migration_step = MigrationStep() diff --git a/setup.cfg b/setup.cfg index 694f9040..bd6ac91b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -30,6 +30,7 @@ console_scripts = gc-status-api = genesis_core.cmd.status_api:main gc-gservice = genesis_core.cmd.gservice:main gc-bootstrap = genesis_core.cmd.bootstrap:main + gc-manifest-parser = genesis_core.cmd.manifest_parser:main gcn_machine_pool_driver = DummyPoolDriver = genesis_core.node.machine.pool.driver.base:DummyPoolDriver diff --git a/tools/convert_multiline_to_oneline.py b/tools/convert_multiline_to_oneline.py new file mode 100755 index 00000000..62fc0176 --- /dev/null +++ b/tools/convert_multiline_to_oneline.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import sys + + +def main(): + if len(sys.argv) != 2: + print("Usage: conver_multiline_to_oneline.py ") + sys.exit(1) + file_path = sys.argv[1] + try: + with open(file_path, "r") as file: + content = file.read() + # Replace newlines with spaces + oneline_content = content.replace("\n", "\\n").replace('"', '\\"') + print(oneline_content) + except FileNotFoundError: + print(f"File not found: {file_path}") + sys.exit(1) + + +if __name__ == "__main__": + main()