Refactor schema evolution related code generation (#797)

* Factor out check into dedicated method

* Factor out filtering of changes to same datatypes

* Cleanup duplicated information

* Make function more general in light of future use

* Rename class to better transport intent

Remove unnecessary duplicated information in members

* Move functionality into separate method

* Move functionality into separate method

* Remove redundant storage of schema version

* Remove unused code

Not used and will most likely not be used in the future

* Remove some code duplication

* Remove unnecessary branch

* Remove unnecessary information that is passed to jinja

* Remove test that doesn't test anything

* Remove no longer necessary nolint statements

* Add clarifying comment

Author: tmadlener

python/podio_gen/cpp_generator.py

@@ -33,6 +33,11 @@ def replace_component_in_paths(oldname, newname, paths):
             paths[index] = newPath
 
 
+def _versioned(typename, version):
+    """Return a versioned name of the typename"""
+    return f"{typename}v{version}"
+
+
 class IncludeFrom(IntEnum):
     """Enum to signify if an include is needed and from where it should come"""
 
@@ -72,7 +77,6 @@ def __init__(  # pylint: disable=too-many-arguments
         self.old_yamlfile = old_description
         self.evolution_file = evolution_file
         self.old_schema_version = None
-        self.old_schema_version_int = None
         self.old_datamodel = None
         self.old_datamodels_components = set()
         self.old_datamodels_datatypes = set()
@@ -117,21 +121,7 @@ def do_process_component(self, name, component):
         component["includes"] = self._sort_includes(includes)
 
         self._fill_templates("Component", component)
-        # Add potentially older schema for schema evolution
-        # based on ROOT capabilities for now
-        if name in self.root_schema_dict:
-            schema_evolutions = self.root_schema_dict[name]
-            component = deepcopy(component)
-            for schema_evolution in schema_evolutions:
-                if isinstance(schema_evolution, RenamedMember):
-                    for member in component["Members"]:
-                        if member.name == schema_evolution.member_name_new:
-                            member.name = schema_evolution.member_name_old
-                    component["class"] = DataType(name + self.old_schema_version)
-                else:
-                    raise NotImplementedError
-            self._fill_templates("Component", component)
-            self.root_schema_component_names.add(name + self.old_schema_version)
+        self._preprocess_schema_evolution_component(name, component)
 
         return component
 
@@ -143,47 +133,7 @@ def do_process_datatype(self, name, datatype):
         self._preprocess_for_obj(datatype)
         self._preprocess_for_collection(datatype)
 
-        # ROOT schema evolution preparation
-        # Compute and prepare the potential schema evolution parts
-        schema_evolution_datatype = deepcopy(datatype)
-        needs_schema_evolution = False
-        for member in schema_evolution_datatype["Members"]:
-            if member.is_array:
-                if member.array_type in self.root_schema_dict:
-                    needs_schema_evolution = True
-                    replace_component_in_paths(
-                        member.array_type,
-                        member.array_type + self.old_schema_version,
-                        schema_evolution_datatype["includes_data"],
-                    )
-                    member.full_type = member.full_type.replace(
-                        member.array_type, member.array_type + self.old_schema_version
-                    )
-                    member.array_type = member.array_type + self.old_schema_version
-
-            else:
-                if member.full_type in self.root_schema_dict:
-                    needs_schema_evolution = True
-                    # prepare the ROOT I/O rule
-                    replace_component_in_paths(
-                        member.full_type,
-                        member.full_type + self.old_schema_version,
-                        schema_evolution_datatype["includes_data"],
-                    )
-                    member.full_type = member.full_type + self.old_schema_version
-                    member.bare_type = member.bare_type + self.old_schema_version
-
-        if needs_schema_evolution:
-            print(f"  Preparing explicit schema evolution for {name}")
-            schema_evolution_datatype["class"].bare_type = (
-                schema_evolution_datatype["class"].bare_type + self.old_schema_version
-            )  # noqa
-            schema_evolution_datatype["old_schema_version"] = self.old_schema_version_int
-            self._fill_templates("Data", schema_evolution_datatype)
-            self.root_schema_datatype_names.add(name + self.old_schema_version)
-            self._fill_templates("Collection", datatype, schema_evolution_datatype)
-        else:
-            self._fill_templates("Collection", datatype)
+        self._preprocess_schema_evolution_datatype(name, datatype)
 
         self._fill_templates("Data", datatype)
         self._fill_templates("Object", datatype)
@@ -215,7 +165,7 @@ def do_process_link(self, _, link):
         for rel in ("From", "To"):
             rel_type = link[rel]
             include_header = f"{rel_type.bare_type}Collection"
-            if self._is_interface(rel_type.full_type):
+            if self._is_in(rel_type.full_type, "interfaces"):
                 # Interfaces do not have a Collection header
                 include_header = rel_type.bare_type
             link["include_types"].append(
@@ -253,7 +203,7 @@ def _preprocess_for_class(self, datatype):
                 member.sub_members = self.datamodel.components[member.full_type]["Members"]
 
         for relation in datatype["OneToOneRelations"]:
-            if self._is_interface(relation.full_type):
+            if self._is_in(relation.full_type, "interfaces"):
                 relation.interface_types = self.datamodel.interfaces[relation.full_type]["Types"]
             if self._needs_include(relation.full_type):
                 fwd_declarations[relation.namespace].append(relation.bare_type)
@@ -265,7 +215,7 @@ def _preprocess_for_class(self, datatype):
             includes.add('#include "podio/RelationRange.h"')
 
         for relation in datatype["OneToManyRelations"]:
-            if self._is_interface(relation.full_type):
+            if self._is_in(relation.full_type, "interfaces"):
                 relation.interface_types = self.datamodel.interfaces[relation.full_type]["Types"]
             if self._needs_include(relation.full_type):
                 includes.add(self._build_include(relation))
@@ -335,7 +285,7 @@ def _preprocess_for_collection(self, datatype):
         for relation in datatype["OneToManyRelations"] + datatype["OneToOneRelations"]:
             if datatype["class"].bare_type != relation.bare_type:
                 include_from = self._needs_include(relation.full_type)
-                if self._is_interface(relation.full_type):
+                if self._is_in(relation.full_type, "interfaces"):
                     includes_cc.add(
                         self._build_include_for_class(relation.bare_type, include_from)
                     )
@@ -405,8 +355,7 @@ def _pre_process_schema_evolution(self):
             )
             comparator.read()
             comparator.compare()
-            self.old_schema_version = f"v{comparator.datamodel_old.schema_version}"
-            self.old_schema_version_int = comparator.datamodel_old.schema_version
+            self.old_schema_version = comparator.datamodel_old.schema_version
             # some sanity checks
             if len(comparator.errors) > 0:
                 print(
@@ -431,6 +380,65 @@ def _pre_process_schema_evolution(self):
                     # add whatever is relevant to our ROOT schema evolution
                     self.root_schema_dict.setdefault(item.klassname, []).append(item)
 
+    def _preprocess_schema_evolution_datatype(self, name, datatype):
+        """Preprocess this datatype (and generate the necessary code) in case
+        schema evolution is necessary
+
+        NOTE: currently limited to support only ROOT schema evolution needs
+        """
+        schema_evolution_datatype = deepcopy(datatype)
+        needs_schema_evolution = False
+        for member in schema_evolution_datatype["Members"]:
+            member_type = member.array_type if member.is_array else member.full_type
+            if member_type in self.root_schema_dict:
+                needs_schema_evolution = True
+                replace_component_in_paths(
+                    member_type,
+                    _versioned(member_type, self.old_schema_version),
+                    schema_evolution_datatype["includes_data"],
+                )
+                if member.is_array:
+                    member.full_type = member.full_type.replace(
+                        member.array_type, _versioned(member.array_type, self.old_schema_version)
+                    )
+                    member.array_type = _versioned(member.array_type, self.old_schema_version)
+                else:
+                    member.full_type = _versioned(member.full_type, self.old_schema_version)
+                    member.bare_type = _versioned(member.bare_type, self.old_schema_version)
+
+        if needs_schema_evolution:
+            print(f"  Preparing explicit schema evolution for {name}")
+            schema_evolution_datatype["class"].bare_type = _versioned(
+                schema_evolution_datatype["class"].bare_type, self.old_schema_version
+            )
+            self._fill_templates("Data", schema_evolution_datatype)
+            self.root_schema_datatype_names.add(_versioned(name, self.old_schema_version))
+
+    def _preprocess_schema_evolution_component(self, name, component):
+        """Preprocess this component (and generate the necessary code) in case
+        schema evolution is necessary
+
+        NOTE: currently limited to support only ROOT schema evolution needs
+        """
+        try:
+            schema_evolutions = self.root_schema_dict[name]
+            component = deepcopy(component)
+            for schema_evolution in schema_evolutions:
+                if isinstance(schema_evolution, RenamedMember):
+                    for member in component["Members"]:
+                        if member.name == schema_evolution.member_name_new:
+                            member.name = schema_evolution.member_name_old
+                    component["class"] = DataType(_versioned(name, self.old_schema_version))
+                else:
+                    raise NotImplementedError
+
+            self._fill_templates("Component", component)
+            self.root_schema_component_names.add(_versioned(name, self.old_schema_version))
+
+        except KeyError:
+            # We didn't find any schema evolution for this component
+            pass
+
     def _invert_interfaces(self):
         """'Invert' the interfaces to have a mapping of types and their usage in
         interfaces.
@@ -465,7 +473,7 @@ def _prepare_iorules(self):
                     iorule = RootIoRule()
                     iorule.sourceClass = type_name
                     iorule.targetClass = type_name
-                    iorule.version = self.old_schema_version.lstrip("v")
+                    iorule.version = self.old_schema_version
                     iorule.source = f"{member_type} {schema_change.member_name_old}"
                     iorule.target = schema_change.member_name_new
                     iorule.code = f"{iorule.target} = onfile.{schema_change.member_name_old};"
@@ -596,7 +604,7 @@ def _create_selection_xml(self):
             "old_schema_components": [
                 DataType(d)
                 for d in self.root_schema_datatype_names | self.root_schema_component_names
-            ],  # noqa
+            ],
             "iorules": self.root_schema_iorules,
         }
 

python/podio_gen/generator_base.py

@@ -259,14 +259,8 @@ def get_fn_format(tmpl):
 
         return fn_templates
 
-    def _eval_template(self, template, data, old_schema_data=None):
+    def _eval_template(self, template, data):
         """Fill the specified template"""
-        # merge the info of data and the old schema into a single dict
-        if old_schema_data:
-            data["OneToOneRelations_old"] = old_schema_data["OneToOneRelations"]
-            data["OneToManyRelations_old"] = old_schema_data["OneToManyRelations"]
-            data["VectorMembers_old"] = old_schema_data["VectorMembers"]
-
         return self.env.get_template(template).render(data)
 
     def _write_file(self, name, content):
@@ -284,7 +278,7 @@ def _write_file(self, name, content):
             changed = write_file_if_changed(fullname, content)
             self.any_changes = changed or self.any_changes
 
-    def _fill_templates(self, template_base, data, old_schema_data=None):
+    def _fill_templates(self, template_base, data):
         """Fill the template and write the results to file"""
         # Update the passed data with some global things that are the same for all
         # files
@@ -294,11 +288,17 @@ def _fill_templates(self, template_base, data, old_schema_data=None):
         for filename, template in self._get_filenames_templates(
             template_base, data["class"].bare_type
         ):
-            self._write_file(filename, self._eval_template(template, data, old_schema_data))
+            self._write_file(filename, self._eval_template(template, data))
+
+    def _is_in(self, classname, category):
+        """Check whether classname is a member of the category (components,
+        datatypes, interfaces)"""
+        if category not in ("datatypes", "components", "interfaces"):
+            raise ValueError(f"{category=} is not a valid category")
 
-    def _is_interface(self, classname):
-        """Check whether this is an interface type or a regular datatype"""
-        all_interfaces = self.datamodel.interfaces
+        all_classes = getattr(self.datamodel, category)
         if self.upstream_edm:
-            all_interfaces = list(self.datamodel.interfaces) + list(self.upstream_edm.interfaces)
-        return classname in all_interfaces
+            all_classes = list(getattr(self.datamodel, category)) + list(
+                getattr(self.upstream_edm, category)
+            )
+        return classname in all_classes

python/podio_gen/julia_generator.py

@@ -56,7 +56,7 @@ def do_process_component(self, _, component):
     def do_process_datatype(self, _, datatype):
         """Do the julia specific processing for a datatype"""
         if any(
-            self._is_interface(r.full_type)
+            self._is_in(r.full_type, "interfaces")
             for r in datatype["OneToOneRelations"] + datatype["OneToManyRelations"]
         ):
             # Julia doesn't support any interfaces yet, so we have to also sort out