Add core namespace to NamespaceCatalog (#2077)

Co-authored-by: Ryan Ly <[email protected]>

Author: stephprince

CHANGELOG.md

@@ -13,6 +13,8 @@
 - When an external file is detected when initializing an ImageSeries and no format is provided, automatically set format to "external" instead of raising an error. @stephprince [#2060](https://github.com/NeurodataWithoutBorders/pynwb/pull/2060)
 - Added mask_type option to `mock_PlaneSegmentation`. @pauladkisson [#2067](https://github.com/NeurodataWithoutBorders/pynwb/pull/2067)
 - Improved the documentation of the `spike_times` in the Units table methods @h-mayorquin [#2085](https://github.com/NeurodataWithoutBorders/pynwb/pull/2085)
+- Removed core namespace warning unless cached version is newer. @stephprince [#2077](https://github.com/NeurodataWithoutBorders/pynwb/pull/2077)
+- Bumped minimum HDMF version to 4.1.0. @stephprince [#2077](https://github.com/NeurodataWithoutBorders/pynwb/pull/2077)
 
 ### Bug fixes
 - Fixed `add_data_interface` functionality that was mistakenly removed in PyNWB 3.0. @stephprince [#2052](https://github.com/NeurodataWithoutBorders/pynwb/pull/2052)

environment-ros3.yml

@@ -6,7 +6,7 @@ channels:
 dependencies:
   - python==3.13
   - h5py==3.12.1
-  - hdmf==3.14.5
+  - hdmf==4.1.0
   - matplotlib==3.9.2
   - numpy==2.1.3
   - pandas==2.2.3

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 ]
 dependencies = [
     "h5py>=3.2.0",
-    "hdmf>=3.14.5,<5",
+    "hdmf>=4.1.0,<5",
     "numpy>=1.24.0",
     "pandas>=1.2.0",
     "python-dateutil>=2.8.2",

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==3.2.0
-hdmf==3.14.5
+hdmf==4.1.0
 numpy==1.24.0
 pandas==1.2.0
 python-dateutil==2.8.2

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.12.1
-hdmf==3.14.5
+hdmf==4.1.0
 numpy==2.1.1; python_version > "3.9"  # numpy 2.1+ is not compatible with py3.9
 numpy==2.0.2; python_version == "3.9"
 pandas==2.2.3

src/pynwb/__init__.py

@@ -88,7 +88,7 @@ def __get_resources() -> dict:
 # a global type map
 global __TYPE_MAP
 
-__ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
+__ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace, core_namespaces=[CORE_NAMESPACE])
 
 hdmf_typemap = hdmf.common.get_type_map()
 __TYPE_MAP = TypeMap(__ns_catalog)

tests/back_compat/test_read.py

@@ -39,13 +39,7 @@ class TestReadOldVersions(TestCase):
 
     def get_io(self, path):
         """Get an NWBHDF5IO object for the given path."""
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
-            return NWBHDF5IO(str(path), 'r')
+        return NWBHDF5IO(str(path), 'r')
 
     def test_read(self):
         """Test reading and validating all NWB files in the same folder as this file.

tests/integration/ros3/test_ros3.py

@@ -30,23 +30,13 @@ def setUp(self):
     def test_read(self):
         s3_path = 'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'
         with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
             with NWBHDF5IO(s3_path, mode='r', driver='ros3') as io:
                 nwbfile = io.read()
                 test_data = nwbfile.acquisition['ts_name'].data[:]
                 self.assertEqual(len(test_data), 3)
 
     def test_dandi_read(self):
         with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
             with NWBHDF5IO(path=self.s3_test_path, mode='r', driver='ros3') as io:
                 nwbfile = io.read()
                 test_data = nwbfile.acquisition['TestData'].data[:]

tests/unit/test_extension.py

@@ -1,11 +1,12 @@
 import os
 import random
 import string
+import warnings
 from datetime import datetime
 from dateutil.tz import tzlocal
 from tempfile import gettempdir
 
-from hdmf.spec import RefSpec
+from hdmf.spec import RefSpec, NamespaceCatalog
 from hdmf.utils import get_docval, docval, popargs
 from pynwb import get_type_map, TimeSeries, NWBFile, register_class, load_namespaces, get_class
 from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec, NWBDatasetSpec
@@ -143,6 +144,10 @@ def setUp(self):
         self.ext_source2 = '%s_extension2.yaml' % self.prefix
         self.ns_path2 = '%s_namespace2.yaml' % self.prefix
 
+        self.ns_catalog = get_type_map().namespace_catalog
+        self.core_ns = 'core'
+        self.core_ns_version = self.ns_catalog.get_namespace(self.core_ns)['version']
+
     def tearDown(self):
         files = (self.ext_source1,
                  self.ns_path1,
@@ -169,3 +174,48 @@ def test_catch_dup_name(self):
         ns_builder2.export(self.ns_path2, outdir=self.tempdir)
         type_map = get_type_map(extensions=os.path.join(self.tempdir, self.ns_path1))
         type_map.load_namespaces(os.path.join(self.tempdir, self.ns_path2))
+
+    def test_catch_dup_name_core_newer(self):
+        new_ns_version = '100.0.0'
+        ns_builder1 = NWBNamespaceBuilder('Extension for us in my Lab', self.core_ns, version=new_ns_version)
+        ext1 = NWBGroupSpec('A custom ElectricalSeries for my lab',
+                            attributes=[NWBAttributeSpec(name='trode_id', doc='the tetrode id', dtype='int')],
+                            neurodata_type_inc='ElectricalSeries',
+                            neurodata_type_def='TetrodeSeries')
+        ns_builder1.add_spec(self.ext_source1, ext1)
+        ns_builder1.export(self.ns_path1, outdir=self.tempdir)
+
+        # create new catalog and merge the loaded core namespace catalog
+        ns_catalog = NamespaceCatalog()
+        ns_catalog.merge(self.ns_catalog)
+
+        # test loading newer namespace than one already loaded will warn
+        msg = (f'Ignoring the following cached namespace(s) because another version is already loaded:\n'
+               f'{self.core_ns} - cached version: {new_ns_version}, loaded version: {self.core_ns_version}\n'
+               f'Please update to the latest package versions.')
+        with self.assertWarnsWith(UserWarning, msg):
+            ns_catalog.load_namespaces(os.path.join(self.tempdir, self.ns_path1))
+
+    def test_catch_dup_name_core_older(self):
+        new_ns_version = '0.0.0'
+        ns_builder1 = NWBNamespaceBuilder('Extension for us in my Lab', self.core_ns, version=new_ns_version)
+        ext1 = NWBGroupSpec('A custom ElectricalSeries for my lab',
+                            attributes=[NWBAttributeSpec(name='trode_id', doc='the tetrode id', dtype='int')],
+                            neurodata_type_inc='ElectricalSeries',
+                            neurodata_type_def='TetrodeSeries')
+        ns_builder1.add_spec(self.ext_source1, ext1)
+        ns_builder1.export(self.ns_path1, outdir=self.tempdir)
+
+        # create new catalog and merge the loaded core namespace catalog
+        ns_catalog = NamespaceCatalog()
+        ns_catalog.merge(self.ns_catalog)
+
+        # test no warning if loading older namespace than one already loaded
+        msg = (f'Ignoring the following cached namespace(s) because another version is already loaded:\n'
+               f'{self.core_ns} - cached version: {new_ns_version}, loaded version: {self.core_ns_version}\n'
+               f'Please update to the latest package versions.')
+        with warnings.catch_warnings(record=True) as ws:
+            ns_catalog.load_namespaces(os.path.join(self.tempdir, self.ns_path1))
+        for w in ws:
+            self.assertTrue(str(w.message) != msg)
+            warnings.warn(str(w.message), w.category)

tests/validation/test_validate.py

@@ -4,7 +4,6 @@
 import sys
 from unittest.mock import patch
 from io import StringIO
-import warnings
 
 from pynwb.testing import TestCase
 from pynwb import validate, NWBHDF5IO
@@ -202,13 +201,7 @@ class TestValidateFunction(TestCase):
 
     def get_io(self, path):
         """Get an NWBHDF5IO object for the given path, ignoring the warning about ignoring cached namespaces."""
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
-            return NWBHDF5IO(str(path), 'r')
+        return NWBHDF5IO(str(path), 'r')
 
     def test_validate_io_no_cache(self):
         """Test that validating a file with no cached spec against the core namespace succeeds."""