From 56f00d05b53ba90b28c7141840fea5ce4da5348b Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 19 Oct 2026 09:28:54 -0700 Subject: [PATCH 1/2] Skip by-date DLS runs already converted from the subject's folder Co-Authored-By: Claude Opus 5.5 --- notebooks/dedupe_sessions.ipynb | 311 ++++++++++++++++++ .../seiler_2024_convert_dataset.py | 33 ++ tests/test_opto_session_deduplication.py | 70 ++++ 3 files changed, 414 insertions(+) create mode 100644 notebooks/dedupe_sessions.ipynb create mode 100644 tests/test_opto_session_deduplication.py diff --git a/notebooks/dedupe_sessions.ipynb b/notebooks/dedupe_sessions.ipynb new file mode 100644 index 0000000..b0e06c6 --- /dev/null +++ b/notebooks/dedupe_sessions.ipynb @@ -0,0 +1,311 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "f2fd347c", + "metadata": {}, + "source": [ + "# Duplicated DLS Excitatory sessions\n", + "\n", + "DANDI:000971 has 137 files with optogenetic treatment \"Unknown\". They come from the date-organized MED-PC files in `Opto Experiments/DLS Excitatory/`, which also hold runs that are already converted from the mice's own treatment folders. This notebook lists the sessions the fixed code converts and checks, for each published \"Unknown\" file, whether that run is still converted as \"Unknown\" or now only from the mouse's treatment folder. It then compares the events of each published duplicate pair." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "3ae8a478", + "metadata": { + "execution": { + "iopub.execute_input": "2026-10-19T16:28:17.845138Z", + "iopub.status.busy": "2026-10-19T16:28:17.844550Z", + "iopub.status.idle": "2026-10-19T16:28:20.372653Z", + "shell.execute_reply": "2026-10-19T16:28:20.372274Z" + } + }, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "import h5py\n", + "import numpy as np\n", + "import pandas as pd\n", + "\n", + "from lerner_lab_to_nwb.seiler_2024.seiler_2024_convert_dataset import opto_to_nwb, get_subject_session_start\n", + "\n", + "data_dir_path = Path(\"/Volumes/T7/CatalystNeuro/Lerner/_Raw_Data_for_future_reference\")\n", + "published_dir_path = Path(\"/Volumes/T7/CatalystNeuro/Lerner/000971\")" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2cddd766", + "metadata": { + "execution": { + "iopub.execute_input": "2026-10-19T16:28:20.374480Z", + "iopub.status.busy": "2026-10-19T16:28:20.374282Z", + "iopub.status.idle": "2026-10-19T16:28:29.161130Z", + "shell.execute_reply": "2026-10-19T16:28:29.160788Z" + } + }, + "outputs": [], + "source": [ + "session_to_nwb_kwargs_per_session = opto_to_nwb(\n", + " data_dir_path=data_dir_path,\n", + " output_dir_path=Path(\"unused\"),\n", + " start_variable=\"Start Date\",\n", + " verbose=False,\n", + ")\n", + "subject_session_start_to_treatments = {}\n", + "for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session:\n", + " subject_id, start_date, start_time = get_subject_session_start(session_to_nwb_kwargs)\n", + " start = pd.Timestamp(f\"{start_date} {start_time}\").isoformat().replace(\":\", \"-\")\n", + " subject_session_start_to_treatments.setdefault((subject_id, start), []).append(\n", + " session_to_nwb_kwargs[\"optogenetic_treatment\"]\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "4297585c", + "metadata": { + "execution": { + "iopub.execute_input": "2026-10-19T16:28:29.163148Z", + "iopub.status.busy": "2026-10-19T16:28:29.163011Z", + "iopub.status.idle": "2026-10-19T16:28:29.178600Z", + "shell.execute_reply": "2026-10-19T16:28:29.178315Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "137 published Unknown files\n" + ] + }, + { + "data": { + "text/plain": [ + "fixed_code_treatments\n", + "ChR2Scrambled 82\n", + "ChR2 53\n", + "Unknown 2\n", + "Name: count, dtype: int64" + ] + }, + "execution_count": 3, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def parse_file_name(file_path: Path) -> tuple[str, str, str]:\n", + " \"\"\"Return the subject ID, treatment and start (YYYY-MM-DDTHH-MM-SS) of a published opto file.\"\"\"\n", + " subject_part, session_part = file_path.stem.split(\"_ses-\")\n", + " subject_id = subject_part.removeprefix(\"sub-\").replace(\"-\", \".\")\n", + " session_id = session_part.split(\"_\")[0]\n", + " treatment = session_id.split(\"-\")[3]\n", + " start = session_id[-19:]\n", + " return subject_id, treatment, start\n", + "\n", + "\n", + "rows = []\n", + "for unknown_file_path in sorted(published_dir_path.glob(\"sub-*/*Opto-DLS-Excitatory-Unknown-*.nwb\")):\n", + " subject_id, _, start = parse_file_name(unknown_file_path)\n", + " rows.append(\n", + " dict(\n", + " subject_id=subject_id,\n", + " start=start,\n", + " fixed_code_treatments=\", \".join(subject_session_start_to_treatments[(subject_id, start)]),\n", + " )\n", + " )\n", + "unknown_df = pd.DataFrame(rows)\n", + "print(f\"{len(unknown_df)} published Unknown files\")\n", + "unknown_df[\"fixed_code_treatments\"].value_counts()" + ] + }, + { + "cell_type": "markdown", + "id": "3e6b1469", + "metadata": {}, + "source": [ + "The runs that still convert as \"Unknown\" exist only in the date-organized files:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2bcdacca", + "metadata": { + "execution": { + "iopub.execute_input": "2026-10-19T16:28:29.180095Z", + "iopub.status.busy": "2026-10-19T16:28:29.179986Z", + "iopub.status.idle": "2026-10-19T16:28:29.186877Z", + "shell.execute_reply": "2026-10-19T16:28:29.186425Z" + } + }, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
subject_idstartfixed_code_treatments
23291.4072020-09-23T12-41-13Unknown
24291.4072020-09-24T12-22-41Unknown
\n", + "
" + ], + "text/plain": [ + " subject_id start fixed_code_treatments\n", + "23 291.407 2020-09-23T12-41-13 Unknown\n", + "24 291.407 2020-09-24T12-22-41 Unknown" + ] + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "unknown_df[unknown_df[\"fixed_code_treatments\"] == \"Unknown\"]" + ] + }, + { + "cell_type": "markdown", + "id": "4c08eafe", + "metadata": {}, + "source": [ + "For the runs that are now converted only once, compare the events in the two published copies." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "14627b97", + "metadata": { + "execution": { + "iopub.execute_input": "2026-10-19T16:28:29.188323Z", + "iopub.status.busy": "2026-10-19T16:28:29.188215Z", + "iopub.status.idle": "2026-10-19T16:28:29.602885Z", + "shell.execute_reply": "2026-10-19T16:28:29.602605Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": [ + "named_treatment shared_datasets_identical only_in_unknown only_in_named\n", + "ChR2 True 48\n", + " footshock_times/timestamps 5\n", + "ChR2Scrambled True 74\n", + " footshock_times/timestamps 8\n", + "dtype: int64" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "def read_behavior_datasets(file_path: Path) -> dict[str, np.ndarray]:\n", + " datasets = {}\n", + " with h5py.File(file_path, \"r\") as file:\n", + " file[\"processing/behavior\"].visititems(\n", + " lambda name, item: datasets.__setitem__(name, item[()]) if isinstance(item, h5py.Dataset) else None\n", + " )\n", + " return datasets\n", + "\n", + "\n", + "comparison_rows = []\n", + "for row in unknown_df[unknown_df[\"fixed_code_treatments\"] != \"Unknown\"].itertuples():\n", + " subject_folder_path = published_dir_path / f\"sub-{row.subject_id.replace('.', '-')}\"\n", + " (unknown_file_path,) = subject_folder_path.glob(f\"*-Unknown-{row.start}*.nwb\")\n", + " (named_file_path,) = [\n", + " file_path for file_path in subject_folder_path.glob(f\"*{row.start}*.nwb\") if file_path != unknown_file_path\n", + " ]\n", + " unknown_datasets = read_behavior_datasets(unknown_file_path)\n", + " named_datasets = read_behavior_datasets(named_file_path)\n", + " shared_names = sorted(set(unknown_datasets) & set(named_datasets))\n", + " comparison_rows.append(\n", + " dict(\n", + " subject_id=row.subject_id,\n", + " start=row.start,\n", + " named_treatment=parse_file_name(named_file_path)[1],\n", + " shared_datasets_identical=all(\n", + " np.array_equal(unknown_datasets[name], named_datasets[name]) for name in shared_names\n", + " ),\n", + " only_in_unknown=\", \".join(sorted(set(unknown_datasets) - set(named_datasets))),\n", + " only_in_named=\", \".join(sorted(set(named_datasets) - set(unknown_datasets))),\n", + " )\n", + " )\n", + "comparison_df = pd.DataFrame(comparison_rows)\n", + "comparison_df.groupby([\"named_treatment\", \"shared_datasets_identical\", \"only_in_unknown\", \"only_in_named\"]).size()" + ] + }, + { + "cell_type": "markdown", + "id": "c76bd5f4", + "metadata": {}, + "source": [ + "Every duplicate pair has identical events in the datasets both files hold. In 13 pairs the \"Unknown\" copy also has `footshock_times`, because the named copy was converted from a CSV export and the CSV reader drops the shock column. That is fixed separately (see #43)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "lerner_lab_to_nwb_env", + "language": "python", + "name": "lerner_lab_to_nwb_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.13" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py index a38e1ca..21049a6 100644 --- a/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py +++ b/src/lerner_lab_to_nwb/seiler_2024/seiler_2024_convert_dataset.py @@ -52,6 +52,7 @@ split_western_blot, ) from lerner_lab_to_nwb.seiler_2024.medpc_helpers import get_medpc_variables, read_medpc_file +from lerner_lab_to_nwb.seiler_2024.seiler_2024csvbehaviorinterface import Seiler2024CSVBehaviorInterface def dataset_to_nwb( @@ -252,6 +253,32 @@ def get_session_key_from_kwargs(session_to_nwb_kwargs: dict): return session_key +def get_subject_session_start(session_to_nwb_kwargs: dict) -> tuple[str, str, str]: + """Get the subject ID, start date, and start time of a session. + + For CSV behavior files, the start date and time are read from the CSV itself. + + Parameters + ---------- + session_to_nwb_kwargs : dict + The arguments for session_to_nwb. + + Returns + ------- + tuple[str, str, str] + The subject ID, start date (mm/dd/yy), and start time (HH:MM:SS) of the session. + """ + behavior_file_path = session_to_nwb_kwargs["behavior_file_path"] + if behavior_file_path.suffix in (".csv", ".CSV"): + behavior_metadata = Seiler2024CSVBehaviorInterface(file_path=str(behavior_file_path)).get_metadata()["Behavior"] + start_date = behavior_metadata["start_date"] + start_time = behavior_metadata["start_time"] + else: + start_date = session_to_nwb_kwargs["session_conditions"]["Start Date"] + start_time = session_to_nwb_kwargs["session_conditions"]["Start Time"] + return session_to_nwb_kwargs["subject_id"], start_date, start_time + + def safe_session_to_nwb(*, session_to_nwb_kwargs: dict, exception_file_path: Union[Path, str]): """Convert a session to NWB while handling any errors by recording error messages to the exception_file_path. @@ -731,6 +758,10 @@ def opto_to_nwb( continue unique_session_keys.add(session_key) session_to_nwb_args_per_session.append(session_to_nwb_args) + subject_session_starts = { + get_subject_session_start(session_to_nwb_args) for session_to_nwb_args in session_to_nwb_args_per_session + } + # DLS Excitatory raw files by date raw_files_by_date_path = data_dir_path / "Opto Experiments" / "DLS Excitatory" start_dates, start_times, msns, file_paths, subjects, box_numbers = [], [], [], [], [], [] @@ -765,6 +796,8 @@ def opto_to_nwb( } if subject in partial_subject_ids_to_subject_id: subject = partial_subject_ids_to_subject_id[subject] + if (subject, start_date, start_time) in subject_session_starts: + continue # already converted from the subject's own folder, which also gives the treatment session_to_nwb_args = dict( data_dir_path=data_dir_path, output_dir_path=output_dir_path, diff --git a/tests/test_opto_session_deduplication.py b/tests/test_opto_session_deduplication.py new file mode 100644 index 0000000..8e1d93b --- /dev/null +++ b/tests/test_opto_session_deduplication.py @@ -0,0 +1,70 @@ +from pathlib import Path + +import pytest + +from lerner_lab_to_nwb.seiler_2024.seiler_2024_convert_dataset import opto_to_nwb + +TREATMENT_FOLDERS = [ + "DLS Excitatory/ChR2", + "DLS Excitatory/EYFP", + "DLS Excitatory/Scrambled", + "DMS Excitatory/ChR2", + "DMS Excitatory/EYFP", + "DMS Excitatory/Scrambled", + "DMS Inhibitory/Group 1/Halo", + "DMS Inhibitory/Group 1/EYFP", + "DMS Inhibitory/Group 1/Scrambled", + "DMS Inhibitory/Group 2/NpHr", + "DMS Inhibitory/Group 2/EYFP", + "DMS Inhibitory/Group 2/Scrambled", +] + + +def medpc_run(subject: str, box: str) -> str: + return ( + "Start Date: 09/29/20\n" + "End Date: 09/29/20\n" + f"Subject: {subject}\n" + "Experiment: \n" + "Group: \n" + f"Box: {box}\n" + "Start Time: 11:36:53\n" + "End Time: 12:36:45\n" + "MSN: Footshock Degradation Left\n" + ) + + +@pytest.fixture +def data_dir_path(tmp_path) -> Path: + opto_path = tmp_path / "Opto Experiments" + for treatment_folder in TREATMENT_FOLDERS: + (opto_path / treatment_folder).mkdir(parents=True) + + # 300.405's run exported to its own folder as a CSV + subject_folder_path = opto_path / "DLS Excitatory" / "Scrambled" / "300_405" + subject_folder_path.mkdir() + (subject_folder_path / "300.405_09-29-20.csv").write_text( + ",Start Date,End Date,Subject,Experiment,Group,Box,Start Time,End Time,MSN\n" + "0,09/29/20,09/29/20,300.405,,,5,11:36:53,12:36:45,Footshock Degradation Left\n" + ) + + # The same run and another mouse's run in the date-organized MED-PC file + (opto_path / "DLS Excitatory" / "2020-09-29").write_text( + "\n" + medpc_run(subject="418.404", box="1") + "\n" + medpc_run(subject="300.405", box="5") + ) + return tmp_path + + +class TestOptoToNwb: + def test_by_date_run_already_in_subject_folder_is_skipped(self, data_dir_path, tmp_path): + session_to_nwb_args_per_session = opto_to_nwb( + data_dir_path=data_dir_path, + output_dir_path=tmp_path / "nwb", + start_variable="Start Date", + verbose=False, + ) + subject_and_treatment = sorted( + (session_to_nwb_args["subject_id"], session_to_nwb_args["optogenetic_treatment"]) + for session_to_nwb_args in session_to_nwb_args_per_session + ) + assert subject_and_treatment == [("300.405", "ChR2Scrambled"), ("418.404", "Unknown")] From f8deae9a28abdd156c7f07e0b77aa1276bec467a Mon Sep 17 00:00:00 2001 From: pauladkisson Date: Mon, 19 Oct 2026 14:23:55 -0700 Subject: [PATCH 2/2] Move the audit notebook out of the repo Co-Authored-By: Claude Opus 5.5 --- notebooks/dedupe_sessions.ipynb | 311 -------------------------------- 1 file changed, 311 deletions(-) delete mode 100644 notebooks/dedupe_sessions.ipynb diff --git a/notebooks/dedupe_sessions.ipynb b/notebooks/dedupe_sessions.ipynb deleted file mode 100644 index b0e06c6..0000000 --- a/notebooks/dedupe_sessions.ipynb +++ /dev/null @@ -1,311 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "id": "f2fd347c", - "metadata": {}, - "source": [ - "# Duplicated DLS Excitatory sessions\n", - "\n", - "DANDI:000971 has 137 files with optogenetic treatment \"Unknown\". They come from the date-organized MED-PC files in `Opto Experiments/DLS Excitatory/`, which also hold runs that are already converted from the mice's own treatment folders. This notebook lists the sessions the fixed code converts and checks, for each published \"Unknown\" file, whether that run is still converted as \"Unknown\" or now only from the mouse's treatment folder. It then compares the events of each published duplicate pair." - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "id": "3ae8a478", - "metadata": { - "execution": { - "iopub.execute_input": "2026-10-19T16:28:17.845138Z", - "iopub.status.busy": "2026-10-19T16:28:17.844550Z", - "iopub.status.idle": "2026-10-19T16:28:20.372653Z", - "shell.execute_reply": "2026-10-19T16:28:20.372274Z" - } - }, - "outputs": [], - "source": [ - "from pathlib import Path\n", - "\n", - "import h5py\n", - "import numpy as np\n", - "import pandas as pd\n", - "\n", - "from lerner_lab_to_nwb.seiler_2024.seiler_2024_convert_dataset import opto_to_nwb, get_subject_session_start\n", - "\n", - "data_dir_path = Path(\"/Volumes/T7/CatalystNeuro/Lerner/_Raw_Data_for_future_reference\")\n", - "published_dir_path = Path(\"/Volumes/T7/CatalystNeuro/Lerner/000971\")" - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "id": "2cddd766", - "metadata": { - "execution": { - "iopub.execute_input": "2026-10-19T16:28:20.374480Z", - "iopub.status.busy": "2026-10-19T16:28:20.374282Z", - "iopub.status.idle": "2026-10-19T16:28:29.161130Z", - "shell.execute_reply": "2026-10-19T16:28:29.160788Z" - } - }, - "outputs": [], - "source": [ - "session_to_nwb_kwargs_per_session = opto_to_nwb(\n", - " data_dir_path=data_dir_path,\n", - " output_dir_path=Path(\"unused\"),\n", - " start_variable=\"Start Date\",\n", - " verbose=False,\n", - ")\n", - "subject_session_start_to_treatments = {}\n", - "for session_to_nwb_kwargs in session_to_nwb_kwargs_per_session:\n", - " subject_id, start_date, start_time = get_subject_session_start(session_to_nwb_kwargs)\n", - " start = pd.Timestamp(f\"{start_date} {start_time}\").isoformat().replace(\":\", \"-\")\n", - " subject_session_start_to_treatments.setdefault((subject_id, start), []).append(\n", - " session_to_nwb_kwargs[\"optogenetic_treatment\"]\n", - " )" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "id": "4297585c", - "metadata": { - "execution": { - "iopub.execute_input": "2026-10-19T16:28:29.163148Z", - "iopub.status.busy": "2026-10-19T16:28:29.163011Z", - "iopub.status.idle": "2026-10-19T16:28:29.178600Z", - "shell.execute_reply": "2026-10-19T16:28:29.178315Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "137 published Unknown files\n" - ] - }, - { - "data": { - "text/plain": [ - "fixed_code_treatments\n", - "ChR2Scrambled 82\n", - "ChR2 53\n", - "Unknown 2\n", - "Name: count, dtype: int64" - ] - }, - "execution_count": 3, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def parse_file_name(file_path: Path) -> tuple[str, str, str]:\n", - " \"\"\"Return the subject ID, treatment and start (YYYY-MM-DDTHH-MM-SS) of a published opto file.\"\"\"\n", - " subject_part, session_part = file_path.stem.split(\"_ses-\")\n", - " subject_id = subject_part.removeprefix(\"sub-\").replace(\"-\", \".\")\n", - " session_id = session_part.split(\"_\")[0]\n", - " treatment = session_id.split(\"-\")[3]\n", - " start = session_id[-19:]\n", - " return subject_id, treatment, start\n", - "\n", - "\n", - "rows = []\n", - "for unknown_file_path in sorted(published_dir_path.glob(\"sub-*/*Opto-DLS-Excitatory-Unknown-*.nwb\")):\n", - " subject_id, _, start = parse_file_name(unknown_file_path)\n", - " rows.append(\n", - " dict(\n", - " subject_id=subject_id,\n", - " start=start,\n", - " fixed_code_treatments=\", \".join(subject_session_start_to_treatments[(subject_id, start)]),\n", - " )\n", - " )\n", - "unknown_df = pd.DataFrame(rows)\n", - "print(f\"{len(unknown_df)} published Unknown files\")\n", - "unknown_df[\"fixed_code_treatments\"].value_counts()" - ] - }, - { - "cell_type": "markdown", - "id": "3e6b1469", - "metadata": {}, - "source": [ - "The runs that still convert as \"Unknown\" exist only in the date-organized files:" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "id": "2bcdacca", - "metadata": { - "execution": { - "iopub.execute_input": "2026-10-19T16:28:29.180095Z", - "iopub.status.busy": "2026-10-19T16:28:29.179986Z", - "iopub.status.idle": "2026-10-19T16:28:29.186877Z", - "shell.execute_reply": "2026-10-19T16:28:29.186425Z" - } - }, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
subject_idstartfixed_code_treatments
23291.4072020-09-23T12-41-13Unknown
24291.4072020-09-24T12-22-41Unknown
\n", - "
" - ], - "text/plain": [ - " subject_id start fixed_code_treatments\n", - "23 291.407 2020-09-23T12-41-13 Unknown\n", - "24 291.407 2020-09-24T12-22-41 Unknown" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "unknown_df[unknown_df[\"fixed_code_treatments\"] == \"Unknown\"]" - ] - }, - { - "cell_type": "markdown", - "id": "4c08eafe", - "metadata": {}, - "source": [ - "For the runs that are now converted only once, compare the events in the two published copies." - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "id": "14627b97", - "metadata": { - "execution": { - "iopub.execute_input": "2026-10-19T16:28:29.188323Z", - "iopub.status.busy": "2026-10-19T16:28:29.188215Z", - "iopub.status.idle": "2026-10-19T16:28:29.602885Z", - "shell.execute_reply": "2026-10-19T16:28:29.602605Z" - } - }, - "outputs": [ - { - "data": { - "text/plain": [ - "named_treatment shared_datasets_identical only_in_unknown only_in_named\n", - "ChR2 True 48\n", - " footshock_times/timestamps 5\n", - "ChR2Scrambled True 74\n", - " footshock_times/timestamps 8\n", - "dtype: int64" - ] - }, - "execution_count": 5, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "def read_behavior_datasets(file_path: Path) -> dict[str, np.ndarray]:\n", - " datasets = {}\n", - " with h5py.File(file_path, \"r\") as file:\n", - " file[\"processing/behavior\"].visititems(\n", - " lambda name, item: datasets.__setitem__(name, item[()]) if isinstance(item, h5py.Dataset) else None\n", - " )\n", - " return datasets\n", - "\n", - "\n", - "comparison_rows = []\n", - "for row in unknown_df[unknown_df[\"fixed_code_treatments\"] != \"Unknown\"].itertuples():\n", - " subject_folder_path = published_dir_path / f\"sub-{row.subject_id.replace('.', '-')}\"\n", - " (unknown_file_path,) = subject_folder_path.glob(f\"*-Unknown-{row.start}*.nwb\")\n", - " (named_file_path,) = [\n", - " file_path for file_path in subject_folder_path.glob(f\"*{row.start}*.nwb\") if file_path != unknown_file_path\n", - " ]\n", - " unknown_datasets = read_behavior_datasets(unknown_file_path)\n", - " named_datasets = read_behavior_datasets(named_file_path)\n", - " shared_names = sorted(set(unknown_datasets) & set(named_datasets))\n", - " comparison_rows.append(\n", - " dict(\n", - " subject_id=row.subject_id,\n", - " start=row.start,\n", - " named_treatment=parse_file_name(named_file_path)[1],\n", - " shared_datasets_identical=all(\n", - " np.array_equal(unknown_datasets[name], named_datasets[name]) for name in shared_names\n", - " ),\n", - " only_in_unknown=\", \".join(sorted(set(unknown_datasets) - set(named_datasets))),\n", - " only_in_named=\", \".join(sorted(set(named_datasets) - set(unknown_datasets))),\n", - " )\n", - " )\n", - "comparison_df = pd.DataFrame(comparison_rows)\n", - "comparison_df.groupby([\"named_treatment\", \"shared_datasets_identical\", \"only_in_unknown\", \"only_in_named\"]).size()" - ] - }, - { - "cell_type": "markdown", - "id": "c76bd5f4", - "metadata": {}, - "source": [ - "Every duplicate pair has identical events in the datasets both files hold. In 13 pairs the \"Unknown\" copy also has `footshock_times`, because the named copy was converted from a CSV export and the CSV reader drops the shock column. That is fixed separately (see #43)." - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "lerner_lab_to_nwb_env", - "language": "python", - "name": "lerner_lab_to_nwb_env" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.11.13" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}