Merge pull request #1 from Peyman-N/pipeline

Peyman-N · web-flow · commit 107005883554 · 2025-02-06T14:53:08.000+01:00
Pipeline
diff --git a/build.py b/build.py
@@ -16,7 +16,7 @@
 include_instances = True  # to speed up the build during development, set this to False
 
 parser = argparse.ArgumentParser(prog=sys.argv[0], description="Generate Python package for openMINDS")
-parser.add_argument('--branch', help="The branch to build from ('main' or 'development')", default="main")
+parser.add_argument("--branch", help="The branch to build from ('main' or 'development')", default="main")
 args = parser.parse_args()
 
 print("*******************************************************************************")
@@ -47,6 +47,7 @@
             instances[version][instance_data["@type"]].append(instance_data)
 
 python_modules = defaultdict(list)
+
 for schema_version in schema_loader.get_schema_versions():
 
     # Step 3 - find all involved schemas for the current version
@@ -55,14 +56,17 @@
     # Step 4a - figure out which schemas are embedded and which are linked
     embedded = set()
     linked = set()
+    class_to_module_map = {}
     for schema_file_path in schemas_file_paths:
         emb, lnk = PythonBuilder(schema_file_path, schema_loader.schemas_sources).get_edges()
+        class_to_module_map = PythonBuilder(
+            schema_file_path, schema_loader.schemas_sources
+        ).update_class_to_module_map(class_to_module_map)
         embedded.update(emb)
         linked.update(lnk)
     conflicts = linked.intersection(embedded)
     if conflicts:
-        print(f"Found schema(s) in version {schema_version} "
-              f"that are both linked and embedded: {conflicts}")
+        print(f"Found schema(s) in version {schema_version} " f"that are both linked and embedded: {conflicts}")
         # conflicts should not happen in new versions.
         # There is one conflict in v1.0, QuantitativeValue,
         # which we treat as embedded
@@ -76,7 +80,7 @@
             schema_loader.schemas_sources,
             instances=instances.get(schema_version, None),
             additional_methods=additional_methods,
-        ).build(embedded=embedded)
+        ).build(embedded=embedded, class_to_module_map=class_to_module_map)
 
         parts = module_path.split(".")
         parent_path = ".".join(parts[:-1])
@@ -106,10 +110,7 @@
     with open(init_file_path, "w") as fp:
         fp.write(f"from . import ({', '.join(sorted(module_list))})\n")
 
-env = Environment(
-    loader=FileSystemLoader(os.path.dirname(os.path.realpath(__file__))),
-    autoescape=select_autoescape()
-)
+env = Environment(loader=FileSystemLoader(os.path.dirname(os.path.realpath(__file__))), autoescape=select_autoescape())
 context = {
     "version": "0.3.0",
 }
diff --git a/pipeline/src/base.py b/pipeline/src/base.py
@@ -27,9 +27,7 @@ def value_to_jsonld(value, include_empty_properties=True, embed_linked_nodes=Tru
             )
         else:
             if hasattr(value, "id") and value.id is None:
-                raise ValueError(
-                    "Exporting as a stand-alone JSON-LD document requires @id to be defined."
-                )
+                raise ValueError("Exporting as a stand-alone JSON-LD document requires @id to be defined.")
             item = {"@id": value.id}
     elif isinstance(value, EmbeddedMetadata):
         item = value.to_jsonld(
@@ -64,16 +62,17 @@ def has_property(self, name):
                 return True
         return False
 
-    def to_jsonld(
-        self, include_empty_properties=True, embed_linked_nodes=True, with_context=True
-    ):
+    def to_jsonld(self, include_empty_properties=True, embed_linked_nodes=True, with_context=True):
         """
         Return a represention of this metadata node as a dictionary that can be directly serialized to JSON-LD.
         """
 
         data = {"@type": self.type_}
         if with_context:
-            data["@context"] = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
+            if self.type_.startswith("https://openminds.ebrains.eu/"):
+                data["@context"] = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
+            else:
+                data["@context"] = {"@vocab": "https://openminds.om-i.org/props/"}
         if hasattr(self, "id") and self.id:
             data["@id"] = self.id
         for property in self.__class__.properties:
diff --git a/pipeline/src/collection.py b/pipeline/src/collection.py
@@ -90,6 +90,12 @@ def save(self, path, individual_files=False, include_empty_properties=False):
         # we first re-add all child nodes to the collection.
         # This is probably not the most elegant or fast way to do this, but it is simple and robust.
         for node in tuple(self.nodes.values()):
+
+            if node.type_.startswith("https://openminds.ebrains.eu/"):
+                data_context = {"@vocab": "https://openminds.ebrains.eu/vocab/"}
+            else:
+                data_context = {"@vocab": "https://openminds.om-i.org/props/"}
+
             for linked_node in node.links:
                 self._add_node(linked_node)
         # Now we can actually save the nodes
@@ -103,7 +109,7 @@ def save(self, path, individual_files=False, include_empty_properties=False):
                     os.makedirs(parent_dir, exist_ok=True)
             self._sort_nodes_by_id()
             data = {
-                "@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
+                "@context": data_context,
                 "@graph": [
                     node.to_jsonld(
                         embed_linked_nodes=False, include_empty_properties=include_empty_properties, with_context=False
@@ -118,7 +124,9 @@ def save(self, path, individual_files=False, include_empty_properties=False):
             if not os.path.exists(path):
                 os.makedirs(path, exist_ok=True)
             if not os.path.isdir(path):
-                raise OSError(f"If saving to multiple files, `path` must be a directory. path={path}, pwd={os.getcwd()}")
+                raise OSError(
+                    f"If saving to multiple files, `path` must be a directory. path={path}, pwd={os.getcwd()}"
+                )
             self._sort_nodes_by_id()
             output_paths = []
             for node in self:
@@ -161,9 +169,13 @@ def load(self, *paths):
             with open(path, "r") as fp:
                 data = json.load(fp)
             if "@graph" in data:
+                if data["@context"]["@vocab"].startswith("https://openminds.ebrains.eu/"):
+                    version = "v3"
+                else:
+                    version = "latest"
                 for item in data["@graph"]:
                     if "@type" in item:
-                        cls = lookup_type(item["@type"])
+                        cls = lookup_type(item["@type"], version=version)
                         node = cls.from_jsonld(item)
                     else:
                         # allow links to metadata instances outside this collection
diff --git a/pipeline/src/module_template.py.txt b/pipeline/src/module_template.py.txt
@@ -16,7 +16,7 @@ class {{ class_name }}({{ base_class }}):
     """
     type_ = "{{ openminds_type }}"
     context = {
-        "@vocab": "https://openminds.ebrains.eu/vocab/"
+        "@vocab": "{{ context_vocab }}"
     }
     schema_version = "{{ schema_version }}"
 
@@ -61,4 +61,4 @@ class {{ class_name }}({{ base_class }}):
         {{key}}={{value}},
     {%- endif %}
     {% endfor -%}
-){% endfor %}
+){% endfor %}
diff --git a/pipeline/src/properties.py b/pipeline/src/properties.py
@@ -118,7 +118,7 @@ def validate(self, value, ignore=None):
                             failures["type"].append(
                                 f"{self.name}: Expected {', '.join(t.__name__ for t in self.types)}, "
                                 f"value contains {type(item)}"
-                        )
+                            )
                     elif isinstance(item, (Node, IRI)):
                         failures.update(item.validate(ignore=ignore))
                 if self.min_items:
@@ -148,7 +148,8 @@ def validate(self, value, ignore=None):
             elif not isinstance(value, self.types):
                 if "type" not in ignore:
                     failures["type"].append(
-                        f"{self.name}: Expected {', '.join(t.__name__ for t in self.types)}, " f"value is {type(value)}"
+                        f"{self.name}: Expected {', '.join(t.__name__ for t in self.types)}, "
+                        f"value is {type(value)}"
                     )
             elif isinstance(value, (Node, IRI)):
                 failures.update(value.validate(ignore=ignore))
diff --git a/pipeline/src/registry.py b/pipeline/src/registry.py
@@ -68,6 +68,7 @@ def lookup_type(class_type: str, version: str = "latest") -> Node:
 
 class Registry(type):
     """Metaclass for registering Knowledge Graph classes."""
+
     properties: List[Property] = []
     type_: Union[str, List[str]]
     context: Dict[str, str]
@@ -107,4 +108,3 @@ def property_names(cls) -> List[str]:
     @property
     def required_property_names(cls) -> List[str]:
         return [p.name for p in cls.properties if p.required]
-
diff --git a/pipeline/tests/test_instantiation.py b/pipeline/tests/test_instantiation.py
@@ -55,10 +55,7 @@ def test_json_roundtrip():
 
 
 def test_IRI():
-    valid_iris = [
-        "https://example.com/path/to/my/file.txt",
-        "file:///path/to/my/file.txt"
-    ]
+    valid_iris = ["https://example.com/path/to/my/file.txt", "file:///path/to/my/file.txt"]
     for value in valid_iris:
         iri = IRI(value)
         assert iri.value == value
@@ -67,9 +64,7 @@ def test_IRI():
             assert not failures
         else:
             assert failures["value"][0] == "IRI points to a local file path"
-    invalid_iris = [
-        "/path/to/my/file.txt"
-    ]
+    invalid_iris = ["/path/to/my/file.txt"]
     for value in invalid_iris:
         with pytest.raises(ValueError) as exc_info:
             iri = IRI(value)
diff --git a/pipeline/tests/test_regressions.py b/pipeline/tests/test_regressions.py
@@ -12,7 +12,7 @@ def test_issue_0002():
 
     node = build_fake_node(omcore.Person)
     data = node.to_jsonld()
-    assert data["@type"] == "https://openminds.ebrains.eu/core/Person"
+    assert data["@type"] == "https://openminds.om-i.org/types/Person"
 
 
 def test_issue_0003():
@@ -38,21 +38,22 @@ def test_issue_0003():
     )
     # on export, a single item should be wrapped in a list, where the property expects an array
     expected = {
-        "@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
-        "@type": "https://openminds.ebrains.eu/core/FileArchive",
+        "@context": {"@vocab": "https://openminds.om-i.org/props/"},
+        "@type": "https://openminds.om-i.org/types/FileArchive",
         "IRI": "http://example.com/archive.zip",
         "format": {
-            "@type": "https://openminds.ebrains.eu/core/ContentType",
+            "@type": "https://openminds.om-i.org/types/ContentType",
             "name": "application/zip",
         },
         "sourceData": [
             {
-                "@type": "https://openminds.ebrains.eu/core/File",
+                "@type": "https://openminds.om-i.org/types/File",
                 "IRI": "http://example.com/some_file.txt",
                 "name": "some_file.txt",
             }
         ],
     }
+
     assert (
         node1.to_jsonld(include_empty_properties=False) == node2.to_jsonld(include_empty_properties=False) == expected
     )
@@ -89,23 +90,19 @@ def test_issue0007():
 
     actual = person.to_jsonld(include_empty_properties=False, embed_linked_nodes=False, with_context=True)
     expected = {
-        "@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
+        "@context": {"@vocab": "https://openminds.om-i.org/props/"},
         "@id": "_:001",
-        "@type": "https://openminds.ebrains.eu/core/Person",
+        "@type": "https://openminds.om-i.org/types/Person",
         "familyName": "Professor",
         "givenName": "A",
         "affiliation": [
             {
-                "@type": "https://openminds.ebrains.eu/core/Affiliation",
-                "memberOf": {
-                    "@id": "_:002"
-                },
+                "@type": "https://openminds.om-i.org/types/Affiliation",
+                "memberOf": {"@id": "_:002"},
             },
             {
-                "@type": "https://openminds.ebrains.eu/core/Affiliation",
-                "memberOf": {
-                    "@id": "_:003"
-                },
+                "@type": "https://openminds.om-i.org/types/Affiliation",
+                "memberOf": {"@id": "_:003"},
             },
         ],
     }
@@ -119,36 +116,32 @@ def test_issue0007():
         saved_data = json.load(fp)
     os.remove("issue0007.jsonld")
     expected_saved_data = {
-        "@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
+        "@context": {"@vocab": "https://openminds.om-i.org/props/"},
         "@graph": [
             {
                 "@id": "_:001",
-                "@type": "https://openminds.ebrains.eu/core/Person",
+                "@type": "https://openminds.om-i.org/types/Person",
                 "affiliation": [
                     {
-                        "@type": "https://openminds.ebrains.eu/core/Affiliation",
-                        "memberOf": {
-                            "@id": "_:002"
-                        },
+                        "@type": "https://openminds.om-i.org/types/Affiliation",
+                        "memberOf": {"@id": "_:002"},
                     },
                     {
-                        "@type": "https://openminds.ebrains.eu/core/Affiliation",
-                        "memberOf": {
-                            "@id": "_:003"
-                        },
+                        "@type": "https://openminds.om-i.org/types/Affiliation",
+                        "memberOf": {"@id": "_:003"},
                     },
                 ],
                 "familyName": "Professor",
                 "givenName": "A",
             },
             {
                 "@id": "_:002",
-                "@type": "https://openminds.ebrains.eu/core/Organization",
+                "@type": "https://openminds.om-i.org/types/Organization",
                 "fullName": "University of This Place",
             },
             {
                 "@id": "_:003",
-                "@type": "https://openminds.ebrains.eu/core/Organization",
+                "@type": "https://openminds.om-i.org/types/Organization",
                 "fullName": "University of That Place",
             },
         ],
@@ -170,16 +163,14 @@ def test_issue0008():
     )
     actual = person.to_jsonld(include_empty_properties=False, embed_linked_nodes=False, with_context=True)
     expected = {
-        "@context": {"@vocab": "https://openminds.ebrains.eu/vocab/"},
+        "@context": {"@vocab": "https://openminds.om-i.org/props/"},
         "@id": "_:002",
-        "@type": "https://openminds.ebrains.eu/core/Person",
+        "@type": "https://openminds.om-i.org/types/Person",
         "affiliation": [
             {
-                "@type": "https://openminds.ebrains.eu/core/Affiliation",
+                "@type": "https://openminds.om-i.org/types/Affiliation",
                 "endDate": "2023-09-30",
-                "memberOf": {
-                    "@id": "_:001"
-                },
+                "memberOf": {"@id": "_:001"},
             }
         ],
         "familyName": "Professor",
@@ -195,10 +186,7 @@ def test_issue0026():
 
     uni1 = omcore.Organization(full_name="University of This Place", id="_:uthisp")
     person = omcore.Person(
-        given_name="A",
-        family_name="Professor",
-        affiliations = [omcore.Affiliation(member_of=uni1)],
-        id="_:ap"
+        given_name="A", family_name="Professor", affiliations=[omcore.Affiliation(member_of=uni1)], id="_:ap"
     )
 
     c = Collection(person)
@@ -223,16 +211,9 @@ def test_issue0023():
 
     uni1 = omcore.Organization(full_name="University of This Place", id="_:uthisp")
     person = omcore.Person(
-        given_name="A",
-        family_name="Professor",
-        affiliations = [omcore.Affiliation(member_of=uni1)],
-        id="_:ap"
-    )
-    dv = omcore.DatasetVersion(
-        full_name="The name of the dataset version",
-        custodians=[person],
-        id="_:dv"
+        given_name="A", family_name="Professor", affiliations=[omcore.Affiliation(member_of=uni1)], id="_:ap"
     )
+    dv = omcore.DatasetVersion(full_name="The name of the dataset version", custodians=[person], id="_:dv")
 
     c = Collection(dv)
 
diff --git a/pipeline/translator.py b/pipeline/translator.py
diff --git a/pipeline/utils.py b/pipeline/utils.py
