Fix copying of typemap and refactor TypeConfigurator (#1302)

Author: rly

CHANGELOG.md

@@ -1,9 +1,14 @@
 # HDMF Changelog
 
-## HDMF 4.1.1 (Upcoming)
-### Enhancements
+## HDMF 4.1.1 (August 12, 2025)
+
+### Fixed
+- Fixed copying of `TypeMap` and `TypeConfigurator`. Previously, the same global `TypeConfigurator` instance was used in all copies of a `TypeMap`. @rly [#1302](https://github.com/hdmf-dev/hdmf/pull/1302)
+
+### Added
 - Added a check for a compound datatype that is not defined in the schema or spec. This is currently not supported. @mavaylon1 [#1276](https://github.com/hdmf-dev/hdmf/pull/1276)
 
+
 ## HDMF 4.1.0 (May 28, 2025)
 
 ### Enhancements
@@ -25,6 +30,7 @@
 - Fixed `get_data_shape` returning the wrong shape for lists of Data objects. @rly [#1270](https://github.com/hdmf-dev/hdmf/pull/1270)
 - Added protection against complex numbers in arrays and other data types. @bendichter [#1279](https://github.com/hdmf-dev/hdmf/pull/1279)
 
+
 ## HDMF 4.0.0 (January 22, 2025)
 
 ### Breaking changes
@@ -48,6 +54,7 @@
 ### Fixed
 - Fixed issue with `DynamicTable.add_column` not allowing subclasses of `DynamicTableRegion` or `EnumData`. @rly [#1091](https://github.com/hdmf-dev/hdmf/pull/1091)
 
+
 ## HDMF 3.14.6 (December 20, 2024)
 
 ### Enhancements

src/hdmf/build/manager.py

@@ -431,7 +431,7 @@ def container_types(self):
         return self.__container_types
 
     def __copy__(self):
-        ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls, self.type_config)
+        ret = TypeMap(copy(self.__ns_catalog), self.__default_mapper_cls, TypeConfigurator(self.type_config.paths))
         ret.merge(self)
         return ret
 
@@ -464,6 +464,7 @@ def merge(self, type_map, ns_catalog=False):
         for custom_generators in reversed(type_map.__class_generator_manager.custom_generators):
             # iterate in reverse order because generators are stored internally as a stack
             self.register_generator(custom_generators)
+        # NOTE: the type config is not merged from the input type map to the new one. add if there is a clear use case
 
     @docval({"name": "generator", "type": type, "doc": "the CustomClassGenerator class to register"})
     def register_generator(self, **kwargs):

src/hdmf/common/__init__.py

@@ -4,6 +4,7 @@
 import os.path
 from copy import deepcopy
 from collections.abc import Callable
+import warnings
 
 CORE_NAMESPACE = 'hdmf-common'
 EXP_NAMESPACE = 'hdmf-experimental'
@@ -26,35 +27,34 @@
         is_method=False)
 def load_type_config(**kwargs):
     """
-    This method will either load the default config or the config provided by the path.
-    NOTE: This config is global and shared across all type maps.
+    This method will either load the config at the given path into either the global type map or a specific type map.
     """
     config_path = kwargs['config_path']
-    type_map = kwargs['type_map'] or get_type_map()
+    type_map = kwargs['type_map'] or __TYPE_MAP
 
     type_map.type_config.load_type_config(config_path)
 
 @docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
         is_method=False)
 def get_loaded_type_config(**kwargs):
     """
-    This method returns the entire config file.
+    This method returns a dictionary with the configuration for each namespace and data type.
     """
-    type_map = kwargs['type_map'] or get_type_map()
+    type_map = kwargs['type_map'] or __TYPE_MAP
 
     if type_map.type_config.config is None:
         msg = "No configuration is loaded."
         raise ValueError(msg)
-    else:
-        return type_map.type_config.config
+
+    return type_map.type_config.config
 
 @docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None},
         is_method=False)
 def unload_type_config(**kwargs):
     """
-    Unload the configuration file.
+    Unload all type configurations from the global type map or a specific type map.
     """
-    type_map = kwargs['type_map'] or get_type_map()
+    type_map = kwargs['type_map'] or __TYPE_MAP
 
     return type_map.type_config.unload_type_config()
 
@@ -183,6 +183,7 @@ def get_type_map(**kwargs):
     if extensions is None:
         type_map = deepcopy(__TYPE_MAP)
     else:
+        warnings.warn("The 'extensions' argument is deprecated and will be removed in HDMF 5.0", DeprecationWarning)
         if isinstance(extensions, TypeMap):
             type_map = extensions
         else:

src/hdmf/container.py

@@ -92,6 +92,8 @@ def setter(self, val):
         return setter
 
     def _get_type_map(self):
+        # TODO: refactor this so that it does not call get_type_map every time an attribute is set
+        # and there is non circular import
         from hdmf.common import get_type_map # circular import
         return get_type_map()
 
@@ -114,57 +116,55 @@ def _field_config(self, arg_name, val, type_map):
         """
         configurator = type_map.type_config
 
-        if len(configurator.path)>0:
-            # The type_map has a config always set; however, when toggled off, the config path is empty.
-            CUR_DIR = os.path.dirname(os.path.realpath(configurator.path[0]))
-            termset_config = configurator.config
-        else:
+        if not configurator.paths:
             return val
 
+        # The type_map has a config always set; however, when toggled off, the config path is empty.
+        # TODO account for more than one different configurator path
+        CUR_DIR = os.path.dirname(os.path.realpath(configurator.paths[0]))
+        termset_config = configurator.config
+
         # If the val has been manually wrapped then skip checking the config for the attr
         if isinstance(val, TermSetWrapper):
             msg = "Field value already wrapped with TermSetWrapper."
             warn(msg)
             return val
 
-        # check to see that the namespace for the container is in the config
+        # return the value if the namespace for the container is not in the config
         if self.namespace not in termset_config['namespaces']:
-            msg = "%s not found within loaded configuration." % self.namespace
+            return val
+
+        # check to see that the container type is in the config under the namespace
+        config_namespace = termset_config['namespaces'][self.namespace]
+        data_type = self.data_type
+
+        # return the value if the data type for the container is not in the config
+        if data_type not in config_namespace['data_types']:
+            return val
+
+        # Get the ObjectMapper
+        obj_mapper = type_map.get_map(self)
+
+        # Get the spec for the constructor arg
+        spec = obj_mapper.get_carg_spec(arg_name)
+        if spec is None:
+            msg = "Spec not found for %s." % arg_name
             warn(msg)
             return val
-        else:
-            # check to see that the container type is in the config under the namespace
-            config_namespace = termset_config['namespaces'][self.namespace]
-            data_type = self.data_type
-
-            if data_type not in config_namespace['data_types']:
-                msg = '%s not found within the configuration for %s' % (data_type, self.namespace)
-                warn(msg)
-                return val
-            else:
-                # Get the ObjectMapper
-                obj_mapper = type_map.get_map(self)
-
-                # Get the spec for the constructor arg
-                spec = obj_mapper.get_carg_spec(arg_name)
-                if spec is None:
-                    msg = "Spec not found for %s." % arg_name
-                    warn(msg)
-                    return val
-
-                # Get spec attr name
-                mapped_attr_name = obj_mapper.get_attribute(spec)
-
-                config_data_type = config_namespace['data_types'][data_type]
-                try:
-                    config_termset_path = config_data_type[mapped_attr_name]
-                except KeyError:
-                    return val
-
-                termset_path = os.path.join(CUR_DIR, config_termset_path['termset'])
-                termset = TermSet(term_schema_path=termset_path)
-                val = TermSetWrapper(value=val, termset=termset)
-                return val
+
+        # Get spec attr name
+        mapped_attr_name = obj_mapper.get_attribute(spec)
+
+        config_data_type = config_namespace['data_types'][data_type]
+        try:
+            config_termset_path = config_data_type[mapped_attr_name]
+        except KeyError:
+            return val
+
+        termset_path = os.path.join(CUR_DIR, config_termset_path['termset'])
+        termset = TermSet(term_schema_path=termset_path)
+        val = TermSetWrapper(value=val, termset=termset)
+        return val
 
     @classmethod
     def _getter(cls, field):

src/hdmf/term_set.py

@@ -355,14 +355,13 @@ class TypeConfigurator:
     When toggled on, every instance of a configuration file supported data type will be validated
     according to the corresponding TermSet.
     """
-    @docval({'name': 'path', 'type': str, 'doc': 'Path to the configuration file.', 'default': None})
+    @docval({'name': 'paths', 'type': list, 'doc': 'Paths to configuration files.', 'default': None})
     def __init__(self, **kwargs):
         self.config = None
-        if kwargs['path'] is None:
-            self.path = []
-        else:
-            self.path = [kwargs['path']]
-            self.load_type_config(config_path=self.path[0])
+        self.paths = []
+        if kwargs["paths"]:
+            for p in kwargs["paths"]:
+                self.load_type_config(p)
 
     @docval({'name': 'data_type', 'type': str,
              'doc': 'The desired data type within the configuration file.'},
@@ -386,19 +385,19 @@ def get_config(self, data_type, namespace):
             raise ValueError(msg)
 
     @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'})
-    def load_type_config(self,config_path):
+    def load_type_config(self, config_path):
         """
         Load the configuration file for validation on the fields defined for the objects within the file.
         """
         with open(config_path, 'r') as config:
-            yaml=YAML(typ='safe')
+            yaml = YAML(typ='safe')
             termset_config = yaml.load(config)
             if self.config is None: # set the initial config/load after config has been unloaded
                 self.config = termset_config
-                if len(self.path)==0: # for loading after an unloaded config
-                    self.path.append(config_path)
+                if len(self.paths) == 0: # for loading after an unloaded config
+                    self.paths.append(config_path)
             else: # append/replace to the existing config
-                if config_path in self.path:
+                if config_path in self.paths:
                     msg = 'This configuration file path already exists within the configurator.'
                     raise ValueError(msg)
                 else:
@@ -415,12 +414,12 @@ def load_type_config(self,config_path):
                                     new_config = termset_config['namespaces'][namespace]['data_types'][data_type]
                                     self.config['namespaces'][namespace]['data_types'][data_type] = new_config
 
-                    # append path to self.path
-                    self.path.append(config_path)
+                    # append path to self.paths
+                    self.paths.append(config_path)
 
     def unload_type_config(self):
         """
-        Remove validation according to termset configuration file.
+        Remove validation with all loaded termset configuration files. Effectively reset this instance.
         """
-        self.path = []
         self.config = None
+        self.paths = []

tests/unit/common/test_common.py

@@ -5,6 +5,9 @@
 
 class TestCommonTypeMap(TestCase):
 
+    def tearDown(self):
+        unload_type_config()
+
     def test_base_types(self):
         tm = get_type_map()
         cls = tm.get_dt_container_cls('Container', 'hdmf-common')
@@ -24,5 +27,4 @@ def test_copy_ts_config(self):
                  {'ExtensionContainer': {'description': None}}}}}
 
         self.assertEqual(tm.type_config.config, config)
-        self.assertEqual(tm.type_config.path, [path])
-        unload_type_config()
+        self.assertEqual(tm.type_config.paths, [path])