Merge branch 'main' into add-figure-script

Author: stephprince

src/nwb_benchmarks/benchmarks/params_remote_download.py

@@ -15,7 +15,7 @@
         name="OphysTestCase",
         https_url=get_https_url(
             dandiset_id="000717",
-            dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb",
+            dandi_path="sub-R6/sub-R6_behavior+ophys.nwb",
             follow_redirects=False,
         ),
     ),
@@ -69,7 +69,7 @@
         name="OphysTestCase",
         https_url=get_https_url(
             dandiset_id="213889",
-            dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.lindi.json",
+            dandi_path="sub-R6/sub-R6_behavior+ophys.nwb.lindi.json",
             follow_redirects=False,
         ),
     ),

src/nwb_benchmarks/benchmarks/params_remote_file_reading.py

@@ -12,7 +12,7 @@
         name="OphysTestCase",
         https_url=get_https_url(
             dandiset_id="000717",
-            dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb",
+            dandi_path="sub-R6/sub-R6_behavior+ophys.nwb",
         ),
     ),
     dict(
@@ -54,7 +54,7 @@
         name="OphysTestCase",
         https_url=get_https_url(
             dandiset_id="213889",
-            dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.lindi.json",
+            dandi_path="sub-R6/sub-R6_behavior+ophys.nwb.lindi.json",
         ),
     ),
     dict(

src/nwb_benchmarks/scripts/generate_lindi_from_remote.ipynb

@@ -2,32 +2,23 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 1,
+   "execution_count": null,
    "id": "abc49392-0615-46f4-9ae6-0f2808c16a09",
    "metadata": {},
    "outputs": [],
    "source": [
-    "import json\n",
     "import lindi\n",
     "from pathlib import Path\n",
     "\n",
-    "max_chunks_to_cache = int(1e9)"
+    "max_chunks_to_cache = None  # int(1e9)"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
    "id": "527e3697-9dc9-4ee7-8e62-3397ca754914",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Writing chunk info for acquisition/data_00002_AD0/data: 100%|█████████| 1502/1502 [00:02<00:00, 639.30it/s]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# ICEPHYS\n",
     "# DANDI:000717\n",
@@ -51,15 +42,7 @@
    "execution_count": null,
    "id": "faa14c01-3940-4057-87e0-74b97fd0427f",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Writing chunk info for acquisition/ElectricalSeriesAp/data: 100%|█████| 1319/1319 [00:03<00:00, 399.32it/s]\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "# ECEPHYS\n",
     "# DANDI:000717\n",
@@ -109,43 +92,26 @@
    "execution_count": null,
    "id": "da3c2d01-4824-4742-90b3-7be00b2410f7",
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Writing chunk info for acquisition/TwoPhotonSeries/data: 100%|█| 1333568/1333568 [00:08<00:00, 163375.64it/\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
+    "# Run on Perlmutter\n",
     "# OPHYS (local)\n",
-    "# DANDI:000717\n",
-    "# path and URL copied from asset page:\n",
-    "# https://api.dandiarchive.org/api/dandisets/000717/versions/draft/assets/a6951f6e-b67e-4df3-a14f-07b7854b821c/\n",
-    "orig_path = Path(\"sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb\")\n",
-    "orig_path.parent.mkdir(parents=True, exist_ok=True)\n",
-    "lindi_path = orig_path.with_suffix(\".lindi.json\")\n",
-    "h5_url = \"https://api.dandiarchive.org/api/assets/a6951f6e-b67e-4df3-a14f-07b7854b821c/download/\"\n",
+    "# import lindi\n",
     "\n",
-    "# Create the h5py-like client\n",
-    "zarr_store_opts = lindi.LindiH5ZarrStoreOpts(num_dataset_chunks_threshold=max_chunks_to_cache)\n",
+    "# DANDI:000717 sub-R6/sub-R6_behavior+ophys.nwb\n",
+    "# https://api.dandiarchive.org/api/dandisets/000717/versions/draft/assets/c15b3e06-f443-4964-a6b0-c44c367d830b/\n",
+    "remote_url = \"https://api.dandiarchive.org/api/assets/c15b3e06-f443-4964-a6b0-c44c367d830b/download/\"\n",
+    "local_path = \"/pscratch/sd/r/rly/nwb_bench/finished_conversions/sub-R6_ses-20200206T210000_behavior+ophys_rechunk.nwb\"\n",
+    "out_path = local_path + \".lindi.json\"\n",
+    "max_chunks_to_cache = None\n",
     "\n",
-    "# Use local refs\n",
-    "url_or_path = \"/Users/rly/Downloads/sub-R6_ses-20200206T210000_behavior+ophys.nwb\"\n",
-    "url = h5_url\n",
-    "local_cache = None\n",
-    "mode = \"r\"\n",
-    "\n",
-    "zarr_store = lindi.LindiH5ZarrStore.from_file(url_or_path, local_cache=local_cache, opts=zarr_store_opts, url=url)\n",
-    "client = lindi.LindiH5pyFile.from_zarr_store(\n",
-    "    zarr_store=zarr_store,\n",
-    "    mode=mode,\n",
-    "    local_cache=local_cache\n",
-    ")\n",
+    "zarr_store_opts = lindi.LindiH5ZarrStoreOpts(num_dataset_chunks_threshold=max_chunks_to_cache)\n",
+    "zarr_store = lindi.LindiH5ZarrStore.from_file(local_path, url=remote_url, opts=zarr_store_opts)\n",
+    "client = lindi.LindiH5pyFile.from_zarr_store(zarr_store=zarr_store, mode=\"r\")\n",
     "\n",
     "# Generate a reference file system and write to JSON file\n",
-    "client.write_lindi_file(str(lindi_path))"
+    "client.write_lindi_file(str(out_path))\n",
+    "client.close()"
    ]
   },
   {
@@ -199,7 +165,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
+   "display_name": "test",
    "language": "python",
    "name": "python3"
   },
@@ -213,7 +179,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.9"
+   "version": "3.13.9"
   }
  },
  "nbformat": 4,