Merge branch 'nwb-schema-2.9.0' into add-ch-idx-to-event-detection

Author: stephprince

CHANGELOG.md

@@ -19,7 +19,8 @@
 - When an external file is detected when initializing an ImageSeries and no format is provided, automatically set format to "external" instead of raising an error. @stephprince [#2060](https://github.com/NeurodataWithoutBorders/pynwb/pull/2060)
 - Added mask_type option to `mock_PlaneSegmentation`. @pauladkisson [#2067](https://github.com/NeurodataWithoutBorders/pynwb/pull/2067)
 - Improved the documentation of the `spike_times` in the Units table methods @h-mayorquin [#2085](https://github.com/NeurodataWithoutBorders/pynwb/pull/2085)
-- Updated minimum HDMF version to 4.1.0. @mavaylon1 @rly [#2063](https://github.com/NeurodataWithoutBorders/pynwb/pull/2063)
+- Removed core namespace warning unless cached version is newer. @stephprince [#2077](https://github.com/NeurodataWithoutBorders/pynwb/pull/2077)
+- Bumped minimum HDMF version to 4.1.0. @stephprince [#2077](https://github.com/NeurodataWithoutBorders/pynwb/pull/2077)
 
 ### Bug fixes
 - Fixed `add_data_interface` functionality that was mistakenly removed in PyNWB 3.0. @stephprince [#2052](https://github.com/NeurodataWithoutBorders/pynwb/pull/2052)
@@ -29,6 +30,10 @@
 - Fixed missing `IndexSeries.indexed_images`. @rly [#2074](https://github.com/NeurodataWithoutBorders/pynwb/pull/2074)
 - Fixed missing `__nwbfields__` and `_fieldsname` for `NWBData` and its subclasses. @rly [#2082](https://github.com/NeurodataWithoutBorders/pynwb/pull/2082)
 - Fixed caching of the type map when using HDMF 4.1.0. @rly [#2087](https://github.com/NeurodataWithoutBorders/pynwb/pull/2087)
+- Removed use of complex numbers in scratch tutorial because of incompatibilities with HDMF 4.1.0. @stephprince [#2090](https://github.com/NeurodataWithoutBorders/pynwb/pull/2090/)
+
+### Documentation and tutorial enhancements
+- Added NWB AI assistant to the home page of the documentation. @magland [#2076](https://github.com/NeurodataWithoutBorders/pynwb/pull/2076)
 
 ## PyNWB 3.0.0 (February 26, 2025)
 

docs/gallery/general/scratch.py

@@ -83,14 +83,12 @@
 # Now that we have a copy, lets process some data, and add the results as a :py:class:`~pynwb.base.ProcessingModule`
 # to our copy of the file. [#]_
 
-import scipy.signal as sps
-
 mod = nwb_proc.create_processing_module(
     "filtering_module", "a module to store filtering results"
 )
 
 ts1 = nwb_in.acquisition["raw_timeseries"]
-filt_data = sps.correlate(ts1.data, np.ones(128), mode="same") / 128
+filt_data = np.convolve(ts1.data, np.ones(128), mode="same") / 128
 ts2 = TimeSeries(name="filtered_timeseries", data=filt_data, unit="m", timestamps=ts1)
 
 mod.add_container(ts2)
@@ -147,12 +145,13 @@
 
 filt_ts = nwb_scratch.processing["filtering_module"]["filtered_timeseries"]
 
-fft = np.fft.fft(filt_ts.data)
+# simple power spectrum without normalization
+ps = np.abs(np.fft.fft(filt_ts.data))**2
 
 nwb_scratch.add_scratch(
-    fft,
-    name="dft_filtered",
-    description="discrete Fourier transform from filtered data",
+    ps,
+    name="power_spectrum",
+    description="power spectrum from filtered data",
 )
 
 
@@ -171,9 +170,9 @@
 scratch_io = NWBHDF5IO("scratch_analysis.nwb", "r")
 nwb_scratch_in = scratch_io.read()
 
-fft_in = nwb_scratch_in.scratch["dft_filtered"]
+fft_in = nwb_scratch_in.scratch["power_spectrum"]
 
-fft_in = nwb_scratch_in.get_scratch("dft_filtered")
+fft_in = nwb_scratch_in.get_scratch("power_spectrum")
 
 ####################
 #

docs/source/_static/css/nwb_assistant.css

@@ -0,0 +1,66 @@
+.assistant-toggle {
+    display: none;
+    position: fixed;
+    right: 20px;
+    top: 20px;
+    z-index: 1001;
+    padding: 8px 16px;
+    background-color: #2980b9;
+    color: white;
+    border: none;
+    border-radius: 4px;
+    cursor: pointer;
+    font-size: 14px;
+}
+
+/* Hide button when assistant is open */
+.assistant-container.show ~ .assistant-toggle {
+    display: none !important;
+}
+
+.assistant-toggle:hover {
+    background-color: #3498db;
+}
+
+.assistant-container {
+    display: none;
+}
+
+@media (min-width: 1300px) {
+    .assistant-toggle {
+        display: block;
+    }
+
+    .assistant-container {
+        display: block;
+        visibility: hidden;
+        position: fixed;
+        right: 0;
+        top: 0;
+        width: calc(100vw - 1140px);
+        height: 100vh;
+        z-index: 1000;
+        transform: translateX(100%);
+        transition: transform 0.3s ease-in-out, visibility 0.3s ease-in-out;
+    }
+
+    .assistant-container.show {
+        visibility: visible;
+        transform: translateX(0);
+    }
+
+    .assistant-iframe {
+        width: 100%;
+        height: 100%;
+        border: none;
+        background: white;
+    }
+
+    .wy-nav-content {
+        transition: margin-right 0.3s ease-in-out;
+    }
+
+    .assistant-container.show ~ .wy-nav-content {
+        margin-right: calc(100vw - 1140px);
+    }
+}

docs/source/conf.py

@@ -263,6 +263,7 @@ def __call__(self, filename):
 # or fully qualified paths (eg. https://...)
 html_css_files = [
     'css/custom.css',
+    'css/nwb_assistant.css'
 ]
 
 # The name for this set of Sphinx documents.  If None, it defaults to

docs/source/index.rst

@@ -20,6 +20,30 @@ The NWB team consists of neuroscientists and software developers
 who recognize that adoption of a unified data format is an important step toward
 breaking down the barriers to data sharing in neuroscience.
 
+.. raw:: html
+
+   <div class="assistant-container">
+     <iframe class="assistant-iframe"></iframe>
+   </div>
+   <button class="assistant-toggle">Open Assistant</button>
+   <script>
+     document.addEventListener('DOMContentLoaded', function() {
+       const toggle = document.querySelector('.assistant-toggle');
+       const container = document.querySelector('.assistant-container');
+       const iframe = document.querySelector('.assistant-iframe');
+       let iframeLoaded = false;
+
+       toggle.addEventListener('click', function() {
+         const isShowing = container.classList.toggle('show');
+
+         // Load iframe content only when first opened
+         if (isShowing && !iframeLoaded) {
+           iframe.src = 'https://magland.github.io/nwb-assistant/chat';
+           iframeLoaded = true;
+         }
+       });
+     });
+   </script>
 
 .. toctree::
    :maxdepth: 2

pyproject.toml

@@ -34,7 +34,7 @@ classifiers = [
 ]
 dependencies = [
     "h5py>=3.2.0",
-    "hdmf>=4.1.0,<6",
+    "hdmf>=4.1.0,<5",
     "numpy>=1.24.0",
     "pandas>=1.2.0",
     "python-dateutil>=2.8.2",

src/pynwb/__init__.py

@@ -88,7 +88,7 @@ def __get_resources() -> dict:
 # a global type map
 global __TYPE_MAP
 
-__ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace)
+__ns_catalog = NamespaceCatalog(NWBGroupSpec, NWBDatasetSpec, NWBNamespace, core_namespaces=[CORE_NAMESPACE])
 
 hdmf_typemap = hdmf.common.get_type_map()
 __TYPE_MAP = TypeMap(__ns_catalog)

tests/back_compat/test_read.py

@@ -41,13 +41,7 @@ class TestReadOldVersions(TestCase):
 
     def get_io(self, path):
         """Get an NWBHDF5IO object for the given path."""
-        with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
-            return NWBHDF5IO(str(path), 'r')
+        return NWBHDF5IO(str(path), 'r')
 
     def test_read(self):
         """Test reading and validating all NWB files in the same folder as this file.

tests/integration/ros3/test_ros3.py

@@ -30,23 +30,13 @@ def setUp(self):
     def test_read(self):
         s3_path = 'https://dandiarchive.s3.amazonaws.com/ros3test.nwb'
         with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
             with NWBHDF5IO(s3_path, mode='r', driver='ros3') as io:
                 nwbfile = io.read()
                 test_data = nwbfile.acquisition['ts_name'].data[:]
                 self.assertEqual(len(test_data), 3)
 
     def test_dandi_read(self):
         with warnings.catch_warnings():
-            warnings.filterwarnings(
-                "ignore",
-                message=r"Ignoring cached namespace .*",
-                category=UserWarning,
-            )
             with NWBHDF5IO(path=self.s3_test_path, mode='r', driver='ros3') as io:
                 nwbfile = io.read()
                 test_data = nwbfile.acquisition['TestData'].data[:]

tests/unit/test_extension.py

@@ -1,11 +1,12 @@
 import os
 import random
 import string
+import warnings
 from datetime import datetime
 from dateutil.tz import tzlocal
 from tempfile import gettempdir
 
-from hdmf.spec import RefSpec
+from hdmf.spec import RefSpec, NamespaceCatalog
 from hdmf.utils import get_docval, docval, popargs
 from pynwb import get_type_map, TimeSeries, NWBFile, register_class, load_namespaces, get_class
 from pynwb.spec import NWBNamespaceBuilder, NWBGroupSpec, NWBAttributeSpec, NWBDatasetSpec
@@ -143,6 +144,10 @@ def setUp(self):
         self.ext_source2 = '%s_extension2.yaml' % self.prefix
         self.ns_path2 = '%s_namespace2.yaml' % self.prefix
 
+        self.ns_catalog = get_type_map().namespace_catalog
+        self.core_ns = 'core'
+        self.core_ns_version = self.ns_catalog.get_namespace(self.core_ns)['version']
+
     def tearDown(self):
         files = (self.ext_source1,
                  self.ns_path1,
@@ -169,3 +174,48 @@ def test_catch_dup_name(self):
         ns_builder2.export(self.ns_path2, outdir=self.tempdir)
         type_map = get_type_map(extensions=os.path.join(self.tempdir, self.ns_path1))
         type_map.load_namespaces(os.path.join(self.tempdir, self.ns_path2))
+
+    def test_catch_dup_name_core_newer(self):
+        new_ns_version = '100.0.0'
+        ns_builder1 = NWBNamespaceBuilder('Extension for us in my Lab', self.core_ns, version=new_ns_version)
+        ext1 = NWBGroupSpec('A custom ElectricalSeries for my lab',
+                            attributes=[NWBAttributeSpec(name='trode_id', doc='the tetrode id', dtype='int')],
+                            neurodata_type_inc='ElectricalSeries',
+                            neurodata_type_def='TetrodeSeries')
+        ns_builder1.add_spec(self.ext_source1, ext1)
+        ns_builder1.export(self.ns_path1, outdir=self.tempdir)
+
+        # create new catalog and merge the loaded core namespace catalog
+        ns_catalog = NamespaceCatalog()
+        ns_catalog.merge(self.ns_catalog)
+
+        # test loading newer namespace than one already loaded will warn
+        msg = (f'Ignoring the following cached namespace(s) because another version is already loaded:\n'
+               f'{self.core_ns} - cached version: {new_ns_version}, loaded version: {self.core_ns_version}\n'
+               f'Please update to the latest package versions.')
+        with self.assertWarnsWith(UserWarning, msg):
+            ns_catalog.load_namespaces(os.path.join(self.tempdir, self.ns_path1))
+
+    def test_catch_dup_name_core_older(self):
+        new_ns_version = '0.0.0'
+        ns_builder1 = NWBNamespaceBuilder('Extension for us in my Lab', self.core_ns, version=new_ns_version)
+        ext1 = NWBGroupSpec('A custom ElectricalSeries for my lab',
+                            attributes=[NWBAttributeSpec(name='trode_id', doc='the tetrode id', dtype='int')],
+                            neurodata_type_inc='ElectricalSeries',
+                            neurodata_type_def='TetrodeSeries')
+        ns_builder1.add_spec(self.ext_source1, ext1)
+        ns_builder1.export(self.ns_path1, outdir=self.tempdir)
+
+        # create new catalog and merge the loaded core namespace catalog
+        ns_catalog = NamespaceCatalog()
+        ns_catalog.merge(self.ns_catalog)
+
+        # test no warning if loading older namespace than one already loaded
+        msg = (f'Ignoring the following cached namespace(s) because another version is already loaded:\n'
+               f'{self.core_ns} - cached version: {new_ns_version}, loaded version: {self.core_ns_version}\n'
+               f'Please update to the latest package versions.')
+        with warnings.catch_warnings(record=True) as ws:
+            ns_catalog.load_namespaces(os.path.join(self.tempdir, self.ns_path1))
+        for w in ws:
+            self.assertTrue(str(w.message) != msg)
+            warnings.warn(str(w.message), w.category)