diff --git a/src/nwb_benchmarks/benchmarks/params_remote_download.py b/src/nwb_benchmarks/benchmarks/params_remote_download.py index c5e87b7..1306d1c 100644 --- a/src/nwb_benchmarks/benchmarks/params_remote_download.py +++ b/src/nwb_benchmarks/benchmarks/params_remote_download.py @@ -15,7 +15,7 @@ name="OphysTestCase", https_url=get_https_url( dandiset_id="000717", - dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb", + dandi_path="sub-R6/sub-R6_behavior+ophys.nwb", follow_redirects=False, ), ), @@ -69,7 +69,7 @@ name="OphysTestCase", https_url=get_https_url( dandiset_id="213889", - dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.lindi.json", + dandi_path="sub-R6/sub-R6_behavior+ophys.nwb.lindi.json", follow_redirects=False, ), ), diff --git a/src/nwb_benchmarks/benchmarks/params_remote_file_reading.py b/src/nwb_benchmarks/benchmarks/params_remote_file_reading.py index 902eecb..cf05cd6 100644 --- a/src/nwb_benchmarks/benchmarks/params_remote_file_reading.py +++ b/src/nwb_benchmarks/benchmarks/params_remote_file_reading.py @@ -12,7 +12,7 @@ name="OphysTestCase", https_url=get_https_url( dandiset_id="000717", - dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb", + dandi_path="sub-R6/sub-R6_behavior+ophys.nwb", ), ), dict( @@ -54,7 +54,7 @@ name="OphysTestCase", https_url=get_https_url( dandiset_id="213889", - dandi_path="sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.lindi.json", + dandi_path="sub-R6/sub-R6_behavior+ophys.nwb.lindi.json", ), ), dict( diff --git a/src/nwb_benchmarks/scripts/generate_lindi_from_remote.ipynb b/src/nwb_benchmarks/scripts/generate_lindi_from_remote.ipynb index d88323f..dd148bd 100644 --- a/src/nwb_benchmarks/scripts/generate_lindi_from_remote.ipynb +++ b/src/nwb_benchmarks/scripts/generate_lindi_from_remote.ipynb @@ -2,16 +2,15 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "id": "abc49392-0615-46f4-9ae6-0f2808c16a09", "metadata": {}, "outputs": [], "source": [ - "import json\n", "import lindi\n", "from pathlib import Path\n", "\n", - "max_chunks_to_cache = int(1e9)" + "max_chunks_to_cache = None # int(1e9)" ] }, { @@ -19,15 +18,7 @@ "execution_count": null, "id": "527e3697-9dc9-4ee7-8e62-3397ca754914", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Writing chunk info for acquisition/data_00002_AD0/data: 100%|█████████| 1502/1502 [00:02<00:00, 639.30it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# ICEPHYS\n", "# DANDI:000717\n", @@ -51,15 +42,7 @@ "execution_count": null, "id": "faa14c01-3940-4057-87e0-74b97fd0427f", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Writing chunk info for acquisition/ElectricalSeriesAp/data: 100%|█████| 1319/1319 [00:03<00:00, 399.32it/s]\n" - ] - } - ], + "outputs": [], "source": [ "# ECEPHYS\n", "# DANDI:000717\n", @@ -109,43 +92,26 @@ "execution_count": null, "id": "da3c2d01-4824-4742-90b3-7be00b2410f7", "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Writing chunk info for acquisition/TwoPhotonSeries/data: 100%|█| 1333568/1333568 [00:08<00:00, 163375.64it/\n" - ] - } - ], + "outputs": [], "source": [ + "# Run on Perlmutter\n", "# OPHYS (local)\n", - "# DANDI:000717\n", - "# path and URL copied from asset page:\n", - "# https://api.dandiarchive.org/api/dandisets/000717/versions/draft/assets/a6951f6e-b67e-4df3-a14f-07b7854b821c/\n", - "orig_path = Path(\"sub-R6_ses-20200206T210000_behavior+ophys/sub-R6_ses-20200206T210000_behavior+ophys.nwb\")\n", - "orig_path.parent.mkdir(parents=True, exist_ok=True)\n", - "lindi_path = orig_path.with_suffix(\".lindi.json\")\n", - "h5_url = \"https://api.dandiarchive.org/api/assets/a6951f6e-b67e-4df3-a14f-07b7854b821c/download/\"\n", + "# import lindi\n", "\n", - "# Create the h5py-like client\n", - "zarr_store_opts = lindi.LindiH5ZarrStoreOpts(num_dataset_chunks_threshold=max_chunks_to_cache)\n", + "# DANDI:000717 sub-R6/sub-R6_behavior+ophys.nwb\n", + "# https://api.dandiarchive.org/api/dandisets/000717/versions/draft/assets/c15b3e06-f443-4964-a6b0-c44c367d830b/\n", + "remote_url = \"https://api.dandiarchive.org/api/assets/c15b3e06-f443-4964-a6b0-c44c367d830b/download/\"\n", + "local_path = \"/pscratch/sd/r/rly/nwb_bench/finished_conversions/sub-R6_ses-20200206T210000_behavior+ophys_rechunk.nwb\"\n", + "out_path = local_path + \".lindi.json\"\n", + "max_chunks_to_cache = None\n", "\n", - "# Use local refs\n", - "url_or_path = \"/Users/rly/Downloads/sub-R6_ses-20200206T210000_behavior+ophys.nwb\"\n", - "url = h5_url\n", - "local_cache = None\n", - "mode = \"r\"\n", - "\n", - "zarr_store = lindi.LindiH5ZarrStore.from_file(url_or_path, local_cache=local_cache, opts=zarr_store_opts, url=url)\n", - "client = lindi.LindiH5pyFile.from_zarr_store(\n", - " zarr_store=zarr_store,\n", - " mode=mode,\n", - " local_cache=local_cache\n", - ")\n", + "zarr_store_opts = lindi.LindiH5ZarrStoreOpts(num_dataset_chunks_threshold=max_chunks_to_cache)\n", + "zarr_store = lindi.LindiH5ZarrStore.from_file(local_path, url=remote_url, opts=zarr_store_opts)\n", + "client = lindi.LindiH5pyFile.from_zarr_store(zarr_store=zarr_store, mode=\"r\")\n", "\n", "# Generate a reference file system and write to JSON file\n", - "client.write_lindi_file(str(lindi_path))" + "client.write_lindi_file(str(out_path))\n", + "client.close()" ] }, { @@ -199,7 +165,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3 (ipykernel)", + "display_name": "test", "language": "python", "name": "python3" }, @@ -213,7 +179,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.11.9" + "version": "3.13.9" } }, "nbformat": 4,