validate exports/imports

Author: slashburygin

genesis/manifests/examples/invalid_exports.yaml

@@ -15,8 +15,13 @@ resources:
       description: "Example node"
       cores: 1
       ram: 1024
-#      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
       disk_spec:
         kind: "root_disk"
         size: 10
         image: "{{ base_image_url | default('https://repository.genesis-core.tech/genesis-base/0.4.1/genesis-base.raw.gz') }}"
+
+exports:
+  example_node:
+    # link: "$core.compute.nodes.example_node"
+    no_link: "$core.compute.nodes.example_node"

genesis/manifests/examples/invalid_imports.yaml

@@ -21,6 +21,12 @@ resources:
         size: 10
         image: "{{ base_image_url | default('https://repository.genesis-core.tech/genesis-base/0.4.1/genesis-base.raw.gz') }}"
 
-exports:
-  example_node:
-    no_link: "$core.compute.nodes.example_node"
+imports:
+  pg18_valid:
+    element: "$dbaas"
+    kind: "resource"
+    link: "$dbaas.types.postgres.versions.$pg18"
+  var_default_cores_invalid:
+#    element: "$core"
+    kind: "resource"
+    link: "$core.vs.variables.$default_cores"

genesis/manifests/specification/base_spec.yaml

@@ -26,12 +26,47 @@ properties:
       - v1
   requirements:
     type: object
+    additionalProperties:
+      $ref: '#/components/schemas/requirements'
   resources:
     type: object
     properties: {}
   exports:
     type: object
+    additionalProperties:
+      $ref: '#/components/schemas/export'
   imports:
     type: object
+    additionalProperties:
+      $ref: '#/components/schemas/import'
 components:
-  schemas: {}
+  schemas:
+    export:
+      type: object
+      properties:
+        link:
+          type: string
+      required:
+        - link
+    requirements:
+      type: object
+      properties:
+        from_version:
+          type: string
+        to_version:
+          type: string
+    import:
+      type: object
+      properties:
+        element:
+          type: string
+        kind:
+          type: string
+          enum:
+            - resource
+        link:
+          type: string
+      required:
+        - element
+        - kind
+        - link

genesis_core/common/exceptions.py

@@ -34,7 +34,7 @@ class CommonValueErrorException(GCException):
     __template__ = "The provided value is invalid."
 
 
-class OpenApiValidateException(GCException):
+class OpenApiValidateException(CommonValueErrorException):
     __template__ = "OpenApiValidateException: {err}"
 
     err: str

genesis_core/elements/dm/models.py

@@ -42,7 +42,7 @@
 from genesis_core.common import utils as cm_utils
 from genesis_core.elements.dm import utils
 from genesis_core.elements.dm.validate import (
-    load_base_manifest_schema,
+    load_full_manifest_schema,
     validate_manifest,
 )
 from genesis_core.elements import constants as cc
@@ -1072,7 +1072,7 @@ def __init__(self):
 
     def load_schema(self):
         if not self.schema:
-            self.schema = load_base_manifest_schema()
+            self.schema = load_full_manifest_schema()
 
     def load_element_from_manifest(self, manifest):
         schema_version = utils.get_required_field(manifest, "SchemaVersion")

genesis_core/elements/dm/validate.py

@@ -74,5 +74,34 @@ def validate_manifest(data: dict, schema: tp.Optional[dict]) -> None:
             )
         except ValidationError as err:
             LOG.exception("Failed to validate data %s: %s", data, err)
-            raise exceptions.OpenApiValidateException(err=str(err))
+            raise exceptions.OpenApiValidateException(
+                err=f"{err.message} in {err.json_path}"
+            )
     return None
+
+
+def build_full_schema(base_manifest_schema: dict, user_api_spec: dict) -> dict:
+    for path, path_obj in user_api_spec["paths"].items():
+        path_parts = path.split("/")
+        if len(path_parts) > 5:
+            continue
+        post_path = path_obj.get("post")
+        if post_path:
+            operation_id = post_path.get("operationId")
+            if operation_id and operation_id.startswith("Create_v1"):
+                schema_ref = post_path["requestBody"]["content"]["application/json"][
+                    "schema"
+                ]
+                model_name = schema_ref["$ref"].split("/")[-1]
+                api_part_1 = path_parts[2]
+                api_part_2 = path_parts[3]
+                model = user_api_spec["components"]["schemas"][model_name]
+                resource = f"$core.{api_part_1}.{api_part_2}"
+                base_manifest_schema["components"]["schemas"][model_name] = model
+                base_manifest_schema["properties"]["resources"]["properties"][
+                    resource
+                ] = {
+                    "type": "object",
+                    "additionalProperties": schema_ref,
+                }
+    return base_manifest_schema

genesis_core/tests/functional/manifests/test_validate.py

@@ -14,14 +14,18 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import os
 import pytest
 import requests
 import yaml
 
 from genesis_core.elements.dm.validate import (
     validate_manifest,
     dump_full_manifest_schema,
+    build_full_schema,
 )
+from genesis_core.common import exceptions
+from genesis_core.common.utils import PROJECT_PATH
 
 REPO_URL = "https://repository.genesis-core.tech/genesis-elements"
 
@@ -49,41 +53,29 @@ def test_validate_manifests(
         validate_manifest(manifest, base_manifest_schema)
         validate_manifest(manifest, full_manifest_schema)
 
+    @pytest.mark.parametrize(
+        "invalid_manifest",
+        [
+            "invalid_exports.yaml",
+            "invalid_imports.yaml",
+            "invalid_resource.yaml",
+        ],
+    )
+    def test_validate_error(
+        self, invalid_manifest, base_manifest_schema, full_manifest_schema
+    ):
+        with open(
+            os.path.join(
+                PROJECT_PATH, "genesis", "manifests", "examples", invalid_manifest
+            ),
+            "r",
+        ) as f:
+            manifest = yaml.safe_load(f)
+        with pytest.raises(exceptions.OpenApiValidateException):
+            validate_manifest(manifest, base_manifest_schema)
+            validate_manifest(manifest, full_manifest_schema)
+
     @pytest.mark.skip(reason="for manual running")
     def test_build_full_schema(self, base_manifest_schema, user_api_spec):
-        path_schema = []
-        for path, path_obj in user_api_spec["paths"].items():
-            path_parts = path.split("/")
-            if len(path_parts) > 5:
-                continue
-            post_path = path_obj.get("post")
-            if post_path:
-                operation_id = post_path.get("operationId")
-                if operation_id and operation_id.startswith("Create_v1"):
-                    schema_ref = post_path["requestBody"]["content"][
-                        "application/json"
-                    ]["schema"]
-                    model_name = schema_ref["$ref"].split("/")[-1]
-                    api_part_1 = path_parts[2]
-                    api_part_2 = path_parts[3]
-                    model = user_api_spec["components"]["schemas"][model_name]
-                    resource = f"$core.{api_part_1}.{api_part_2}"
-                    path_schema.append(
-                        {
-                            "path": path,
-                            "schema": schema_ref,
-                            "resource": resource,
-                            "model_name": model_name,
-                            "model": model,
-                        }
-                    )
-                    base_manifest_schema["components"]["schemas"][model_name] = model
-                    base_manifest_schema["properties"]["resources"]["properties"][
-                        resource
-                    ] = {
-                        "type": "object",
-                        "additionalProperties": schema_ref,
-                    }
-
-        assert path_schema
-        dump_full_manifest_schema(base_manifest_schema)
+        full_schema = build_full_schema(base_manifest_schema, user_api_spec)
+        dump_full_manifest_schema(full_schema)

genesis_core/tests/functional/service/test_elements.py

@@ -14,10 +14,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import typing as tp
-
-from gcl_iam.tests.functional import clients as iam_clients
-
 from genesis_core.elements.services import builders
 
 
@@ -31,8 +27,5 @@ def teardown_method(self) -> None:
 
     def test_element_manager_builder(
         self,
-        password_factory: tp.Callable,
-        user_api_client: iam_clients.GenesisCoreTestRESTClient,
-        auth_user_admin: iam_clients.GenesisCoreAuth,
     ):
         self._service._iteration()