Skip to content

Commit da18299

Browse files
rcpeenercpeene
authored
Merge pull request #59 from AllenInstitute/dev
Release LFP notebook
2 parents f70a927 + 6a2b1d0 commit da18299

File tree

8 files changed

+1735
-606
lines changed

8 files changed

+1735
-606
lines changed

‎docs/basics/download_nwb.ipynb‎

Lines changed: 40 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"id": "92b86eca",
66
"metadata": {},
77
"source": [
8-
"# Download an NWB File\n",
8+
"# Downloading an NWB File\n",
99
"In order to analyze some data, you'll need to have some data. The [DANDI Archive](https://dandiarchive.org/) is used to store NWB files in datasets called **dandisets**. Typically, an NWB file contains the data for just one experimental session, while a dandiset contains all the related data files yielded from a project. This notebook allows you to download from public dandisets or private dandisets (called **embargoed** dandisets) via the [DANDI Python API](https://dandi.readthedocs.io/en/latest/modref/index.html). To download embargoed dandisets from DANDI, you will need to make an account on the DANDI Archive and must be given access by the owner of the dandiset."
1010
]
1111
},
@@ -19,7 +19,7 @@
1919
},
2020
{
2121
"cell_type": "code",
22-
"execution_count": null,
22+
"execution_count": 1,
2323
"id": "8f34eecf",
2424
"metadata": {},
2525
"outputs": [],
@@ -38,23 +38,38 @@
3838
},
3939
{
4040
"cell_type": "code",
41-
"execution_count": null,
41+
"execution_count": 2,
4242
"id": "67536d37",
4343
"metadata": {},
4444
"outputs": [],
4545
"source": [
4646
"dandiset_id = \"000021\"\n",
47-
"data_loc = \"~/data\"\n",
47+
"download_loc = \".\"\n",
4848
"authenticate = False\n",
49-
"dandi_api_key = \"your_api_key_here\""
49+
"dandi_api_key = \"\""
5050
]
5151
},
5252
{
5353
"cell_type": "code",
54-
"execution_count": null,
54+
"execution_count": 3,
5555
"id": "a309c067",
5656
"metadata": {},
57-
"outputs": [],
57+
"outputs": [
58+
{
59+
"name": "stderr",
60+
"output_type": "stream",
61+
"text": [
62+
"A newer version (0.46.6) of dandi/dandi-cli is available. You are using 0.46.3\n"
63+
]
64+
},
65+
{
66+
"name": "stdout",
67+
"output_type": "stream",
68+
"text": [
69+
"Got dandiset DANDI:000021/draft\n"
70+
]
71+
}
72+
],
5873
"source": [
5974
"if authenticate:\n",
6075
" client = dandiapi.DandiAPIClient(token=dandi_api_key)\n",
@@ -70,13 +85,13 @@
7085
"id": "420ef8ac",
7186
"metadata": {},
7287
"source": [
73-
"### Download Just One File\n",
88+
"### Downloading Just One File\n",
7489
"Set `filepath` to the path of the file you want to download within the dandiset. You can get this by navigating to the file you want to download on the DANDI Archive website and pressing on the `i` icon. There, you can copy the filepath from the field labeled `path`. Don't include a leading `/`."
7590
]
7691
},
7792
{
7893
"cell_type": "code",
79-
"execution_count": null,
94+
"execution_count": 4,
8095
"id": "fe9aa40c",
8196
"metadata": {},
8297
"outputs": [],
@@ -86,25 +101,33 @@
86101
},
87102
{
88103
"cell_type": "code",
89-
"execution_count": null,
104+
"execution_count": 5,
90105
"id": "a110beeb",
91106
"metadata": {},
92-
"outputs": [],
107+
"outputs": [
108+
{
109+
"name": "stdout",
110+
"output_type": "stream",
111+
"text": [
112+
"Downloaded file to ./sub-699733573_ses-715093703.nwb\n"
113+
]
114+
}
115+
],
93116
"source": [
94-
"file = my_dandiset.get_asset_by_path(filepath)\n",
95117
"filename = filepath.split(\"/\")[-1]\n",
118+
"file = my_dandiset.get_asset_by_path(filepath)\n",
96119
"# this may take awhile, especially if the file to download is large\n",
97-
"file.download(f\"{data_loc}/{filename}\")\n",
120+
"file.download(f\"{download_loc}/{filename}\")\n",
98121
"\n",
99-
"print(f\"Downloaded file to {data_loc}/{filename}\")"
122+
"print(f\"Downloaded file to {download_loc}/{filename}\")"
100123
]
101124
},
102125
{
103126
"cell_type": "markdown",
104127
"id": "7a85a038",
105128
"metadata": {},
106129
"source": [
107-
"### Download Entire Dandiset\n",
130+
"### Downloading Entire Dandiset\n",
108131
"If you'd like to do a lot of work with the files in a dandiset, you might want to download the entire thing or some portion of the dandiset. Be prepared, though; This could take a significant amount of space on your drive and a significant amount of time. If you want to just download all the files within a directory of the dandiset, you can set the first argument of `download_directory` below to a more specific path within the dandiset."
109132
]
110133
},
@@ -116,9 +139,9 @@
116139
"outputs": [],
117140
"source": [
118141
"# patience isn't just a virtue, it's a requirement\n",
119-
"my_dandiset.download_directory(\"./\", f\"{data_loc}/{dandiset_id}\")\n",
142+
"my_dandiset.download_directory(\"./\", f\"{download_loc}/{dandiset_id}\")\n",
120143
"\n",
121-
"print(f\"Downloaded directory to {data_loc}/{dandiset_id}\")"
144+
"print(f\"Downloaded directory to {download_loc}/{dandiset_id}\")"
122145
]
123146
}
124147
],

‎docs/basics/read_nwb.ipynb‎

Lines changed: 84 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919
},
2020
{
2121
"cell_type": "code",
22-
"execution_count": null,
22+
"execution_count": 1,
2323
"id": "d76f08a6",
2424
"metadata": {},
2525
"outputs": [],
@@ -38,20 +38,99 @@
3838
},
3939
{
4040
"cell_type": "code",
41-
"execution_count": null,
41+
"execution_count": 2,
4242
"id": "400c411d",
4343
"metadata": {},
4444
"outputs": [],
4545
"source": [
46-
"nwb_filepath = \"~/data/sub-699733573_ses-715093703.nwb\""
46+
"nwb_filepath = \"./sub-699733573_ses-715093703.nwb\""
4747
]
4848
},
4949
{
5050
"cell_type": "code",
51-
"execution_count": null,
51+
"execution_count": 3,
5252
"id": "7628e758",
5353
"metadata": {},
54-
"outputs": [],
54+
"outputs": [
55+
{
56+
"name": "stderr",
57+
"output_type": "stream",
58+
"text": [
59+
"C:\\Users\\carter.peene\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\hdmf\\spec\\namespace.py:531: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.1 is already loaded.\n",
60+
" warn(\"Ignoring cached namespace '%s' version %s because version %s is already loaded.\"\n",
61+
"C:\\Users\\carter.peene\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\hdmf\\spec\\namespace.py:531: UserWarning: Ignoring cached namespace 'core' version 2.2.2 because version 2.5.0 is already loaded.\n",
62+
" warn(\"Ignoring cached namespace '%s' version %s because version %s is already loaded.\"\n"
63+
]
64+
},
65+
{
66+
"name": "stdout",
67+
"output_type": "stream",
68+
"text": [
69+
"root pynwb.file.NWBFile at 0x2647013427664\n",
70+
"Fields:\n",
71+
" acquisition: {\n",
72+
" raw_running_wheel_rotation <class 'pynwb.base.TimeSeries'>,\n",
73+
" running_wheel_signal_voltage <class 'pynwb.base.TimeSeries'>,\n",
74+
" running_wheel_supply_voltage <class 'pynwb.base.TimeSeries'>\n",
75+
" }\n",
76+
" devices: {\n",
77+
" probeA <class 'abc.EcephysProbe'>,\n",
78+
" probeB <class 'abc.EcephysProbe'>,\n",
79+
" probeC <class 'abc.EcephysProbe'>,\n",
80+
" probeD <class 'abc.EcephysProbe'>,\n",
81+
" probeE <class 'abc.EcephysProbe'>,\n",
82+
" probeF <class 'abc.EcephysProbe'>\n",
83+
" }\n",
84+
" electrode_groups: {\n",
85+
" probeA <class 'abc.EcephysElectrodeGroup'>,\n",
86+
" probeB <class 'abc.EcephysElectrodeGroup'>,\n",
87+
" probeC <class 'abc.EcephysElectrodeGroup'>,\n",
88+
" probeD <class 'abc.EcephysElectrodeGroup'>,\n",
89+
" probeE <class 'abc.EcephysElectrodeGroup'>,\n",
90+
" probeF <class 'abc.EcephysElectrodeGroup'>\n",
91+
" }\n",
92+
" electrodes: electrodes <class 'hdmf.common.table.DynamicTable'>\n",
93+
" file_create_date: [datetime.datetime(2020, 5, 26, 0, 53, 26, 986608, tzinfo=tzoffset(None, -25200))]\n",
94+
" identifier: 715093703\n",
95+
" institution: Allen Institute for Brain Science\n",
96+
" intervals: {\n",
97+
" drifting_gratings_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
98+
" flashes_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
99+
" gabors_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
100+
" invalid_times <class 'pynwb.epoch.TimeIntervals'>,\n",
101+
" natural_movie_one_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
102+
" natural_movie_three_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
103+
" natural_scenes_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
104+
" spontaneous_presentations <class 'pynwb.epoch.TimeIntervals'>,\n",
105+
" static_gratings_presentations <class 'pynwb.epoch.TimeIntervals'>\n",
106+
" }\n",
107+
" invalid_times: invalid_times <class 'pynwb.epoch.TimeIntervals'>\n",
108+
" processing: {\n",
109+
" eye_tracking_rig_metadata <class 'pynwb.base.ProcessingModule'>,\n",
110+
" optotagging <class 'pynwb.base.ProcessingModule'>,\n",
111+
" running <class 'pynwb.base.ProcessingModule'>,\n",
112+
" stimulus <class 'pynwb.base.ProcessingModule'>\n",
113+
" }\n",
114+
" session_description: Data and metadata for an Ecephys session\n",
115+
" session_id: 715093703\n",
116+
" session_start_time: 2019-01-19 00:54:18-08:00\n",
117+
" stimulus_notes: brain_observatory_1.1\n",
118+
" subject: subject abc.EcephysSpecimen at 0x2647012943904\n",
119+
"Fields:\n",
120+
" age: P118D\n",
121+
" age_in_days: 118.0\n",
122+
" genotype: Sst-IRES-Cre/wt;Ai32(RCL-ChR2(H134R)_EYFP)/wt\n",
123+
" sex: M\n",
124+
" species: Mus musculus\n",
125+
" specimen_name: Sst-IRES-Cre;Ai32-386129\n",
126+
" subject_id: 699733573\n",
127+
"\n",
128+
" timestamps_reference_time: 2019-01-19 00:54:18-08:00\n",
129+
" units: units <class 'pynwb.misc.Units'>\n",
130+
"\n"
131+
]
132+
}
133+
],
55134
"source": [
56135
"io = NWBHDF5IO(nwb_filepath, mode=\"r\", load_namespaces=True)\n",
57136
"nwb = io.read()\n",

‎docs/basics/stream_nwb.ipynb‎

Lines changed: 63 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
"id": "d84ee614",
66
"metadata": {},
77
"source": [
8-
"# Stream an NWB File with fsspec\n",
8+
"# Streaming an NWB File with fsspec\n",
99
"As you might have realized, NWB files are large. They take a lot of time to download and a lot of space on your drive. A convenient tool to mitigate this is **fsspec**. Fsspec allows you to *stream* the information from a file remotely without having to download it. This can be more efficient if you are only wanting to quickly examine a file or just need access to a portion of the file's contents. For more exensive analysis, it is still recommended that you download the file."
1010
]
1111
},
@@ -19,7 +19,7 @@
1919
},
2020
{
2121
"cell_type": "code",
22-
"execution_count": null,
22+
"execution_count": 1,
2323
"id": "df1c4cce",
2424
"metadata": {},
2525
"outputs": [],
@@ -47,7 +47,7 @@
4747
},
4848
{
4949
"cell_type": "code",
50-
"execution_count": null,
50+
"execution_count": 2,
5151
"id": "f3f97f13",
5252
"metadata": {},
5353
"outputs": [],
@@ -60,10 +60,25 @@
6060
},
6161
{
6262
"cell_type": "code",
63-
"execution_count": null,
63+
"execution_count": 3,
6464
"id": "a51caf90",
6565
"metadata": {},
66-
"outputs": [],
66+
"outputs": [
67+
{
68+
"name": "stderr",
69+
"output_type": "stream",
70+
"text": [
71+
"A newer version (0.46.6) of dandi/dandi-cli is available. You are using 0.46.3\n"
72+
]
73+
},
74+
{
75+
"name": "stdout",
76+
"output_type": "stream",
77+
"text": [
78+
"Got dandiset DANDI:000021/draft\n"
79+
]
80+
}
81+
],
6782
"source": [
6883
"if authenticate:\n",
6984
" client = dandiapi.DandiAPIClient(token=dandi_api_key)\n",
@@ -76,10 +91,18 @@
7691
},
7792
{
7893
"cell_type": "code",
79-
"execution_count": null,
94+
"execution_count": 4,
8095
"id": "d131ad56",
8196
"metadata": {},
82-
"outputs": [],
97+
"outputs": [
98+
{
99+
"name": "stdout",
100+
"output_type": "stream",
101+
"text": [
102+
"Retrieved file url https://dandiarchive.s3.amazonaws.com/blobs/f5f/175/f5f1752f-5227-47d5-8f75-cd71937878aa?response-content-disposition=attachment%3B%20filename%3D%22sub-699733573_ses-715093703.nwb%22&X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=AKIAUBRWC5GAEKH3223E%2F20221130%2Fus-east-2%2Fs3%2Faws4_request&X-Amz-Date=20221130T215329Z&X-Amz-Expires=3600&X-Amz-SignedHeaders=host&X-Amz-Signature=9edb7bb8f4263cd27cdcc37b27cc23cc7e79b8dbc3a0c9d6b4ae2a74da30c63a\n"
103+
]
104+
}
105+
],
83106
"source": [
84107
"file = my_dandiset.get_asset_by_path(filepath)\n",
85108
"base_url = file.client.session.head(file.base_download_url)\n",
@@ -93,22 +116,32 @@
93116
"id": "3df13a24",
94117
"metadata": {},
95118
"source": [
96-
"### Stream Your File\n",
119+
"### Streaming a File\n",
97120
"First, this creates a virtual filesystem based on the http protocol and specifies using caching to save accessed data to RAM. Then it opens the file remotely through the virtual filesystem."
98121
]
99122
},
100123
{
101124
"cell_type": "code",
102-
"execution_count": null,
125+
"execution_count": 5,
103126
"id": "d15db3bb",
104127
"metadata": {
105128
"scrolled": true
106129
},
107-
"outputs": [],
130+
"outputs": [
131+
{
132+
"name": "stderr",
133+
"output_type": "stream",
134+
"text": [
135+
"C:\\Users\\carter.peene\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\hdmf\\spec\\namespace.py:531: UserWarning: Ignoring cached namespace 'hdmf-common' version 1.1.3 because version 1.5.1 is already loaded.\n",
136+
" warn(\"Ignoring cached namespace '%s' version %s because version %s is already loaded.\"\n",
137+
"C:\\Users\\carter.peene\\AppData\\Local\\Programs\\Python\\Python39\\lib\\site-packages\\hdmf\\spec\\namespace.py:531: UserWarning: Ignoring cached namespace 'core' version 2.2.2 because version 2.5.0 is already loaded.\n",
138+
" warn(\"Ignoring cached namespace '%s' version %s because version %s is already loaded.\"\n"
139+
]
140+
}
141+
],
108142
"source": [
109143
"fs = CachingFileSystem(\n",
110-
" fs=fsspec.filesystem(\"http\"),\n",
111-
" cache_storage=\"nwb-cache\", # Local folder for the cache\n",
144+
" fs=fsspec.filesystem(\"http\")\n",
112145
")\n",
113146
"\n",
114147
"f = fs.open(file_url, \"rb\")\n",
@@ -128,7 +161,7 @@
128161
},
129162
{
130163
"cell_type": "code",
131-
"execution_count": null,
164+
"execution_count": 6,
132165
"id": "91031da2",
133166
"metadata": {},
134167
"outputs": [],
@@ -145,10 +178,25 @@
145178
},
146179
{
147180
"cell_type": "code",
148-
"execution_count": null,
181+
"execution_count": 7,
149182
"id": "3e06b964",
150183
"metadata": {},
151-
"outputs": [],
184+
"outputs": [
185+
{
186+
"data": {
187+
"application/vnd.jupyter.widget-view+json": {
188+
"model_id": "682abdb6aeef48f4a3164f618072cbc1",
189+
"version_major": 2,
190+
"version_minor": 0
191+
},
192+
"text/plain": [
193+
"VBox(children=(HBox(children=(Label(value='session_description:', layout=Layout(max_height='40px', max_width='…"
194+
]
195+
},
196+
"metadata": {},
197+
"output_type": "display_data"
198+
}
199+
],
152200
"source": [
153201
"nwb2widget(nwb)"
154202
]

0 commit comments

Comments
 (0)