From 7cfc720c5e59411f41bd57a95740340ed9e540e7 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 18 Jun 2024 13:20:31 -0400 Subject: [PATCH 01/39] Update(setup.py) element-interface branch to resolve installation conflicts --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 6a27897..c7bb1a0 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "ipywidgets", "plotly", "opencv-python", - "element-interface @ git+https://github.com/datajoint/element-interface.git", + "element-interface @ git+https://github.com/datajoint/element-interface.git@staging", ], extras_require={ "caiman_requirements": [caiman_requirements], From 589f3d315273784c7f82e21a808fc2613b795011 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 26 Jun 2024 23:21:13 -0400 Subject: [PATCH 02/39] Ingest Inscopix metadata --- element_miniscope/miniscope.py | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/element_miniscope/miniscope.py b/element_miniscope/miniscope.py index d5c34ba..9ab5aa9 100644 --- a/element_miniscope/miniscope.py +++ b/element_miniscope/miniscope.py @@ -8,6 +8,7 @@ import cv2 import datajoint as dj import numpy as np +import pandas as pd from element_interface.utils import dict_to_uuid, find_full_path, find_root_directory from . import miniscope_report @@ -116,7 +117,7 @@ class AcquisitionSoftware(dj.Lookup): definition = """ acq_software: varchar(24) """ - contents = zip(["Miniscope-DAQ-V3", "Miniscope-DAQ-V4"]) + contents = zip(["Miniscope-DAQ-V3", "Miniscope-DAQ-V4", "Inscopix"]) @schema @@ -241,8 +242,8 @@ def make(self, key): recording_filepaths = [ file_path.as_posix() for file_path in recording_path.glob("*.avi") - ] - + ] if acq_software != "Inscopix" else [ + file_path.as_posix() for file_path in recording_path.rglob("*.avi")] if not recording_filepaths: raise FileNotFoundError(f"No .avi files found in " f"{recording_directory}") @@ -299,6 +300,23 @@ def make(self, key): time_stamps = np.array( [list(map(int, time_stamps[i])) for i in range(1, len(time_stamps))] ) + + elif acq_software == "Inscopix": + session_metadata = list(recording_path.glob("apply_session.json"))[0] + timestamps_file = next(recording_path.glob("*/*timestamps.csv"))[0] + inscopix_metadata = json.load(open(session_metadata)) + recording_timestamps = pd.read_csv(timestamps_file) + + nchannels = len(inscopix_metadata["manual"]["mScope"]["ledMaxPower"]) + nframes = len(recording_timestamps) + fps = inscopix_metadata["microscope"]["fps"]["fps"] + gain = inscopix_metadata["microscope"]["gain"] + led_power = inscopix_metadata["microscope"]["led"]["exPower"] + time_stamps = (recording_timestamps[" time (ms)"] / 1000).values + px_height = None + px_width = None + spatial_downsample = None + else: raise NotImplementedError( f"Loading routine not implemented for {acq_software}" @@ -391,7 +409,6 @@ def insert_new_params( paramset_idx: int, paramset_desc: str, params: dict, - processing_method_desc: str = "", ): """Insert new parameter set. From 63dccf62b62c6f552a4c196984474f883f061ff7 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 27 Jun 2024 00:04:02 -0400 Subject: [PATCH 03/39] Fix(miniscopy.py) TypeError --- element_miniscope/miniscope.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope.py b/element_miniscope/miniscope.py index 9ab5aa9..681ebb8 100644 --- a/element_miniscope/miniscope.py +++ b/element_miniscope/miniscope.py @@ -303,7 +303,7 @@ def make(self, key): elif acq_software == "Inscopix": session_metadata = list(recording_path.glob("apply_session.json"))[0] - timestamps_file = next(recording_path.glob("*/*timestamps.csv"))[0] + timestamps_file = list(recording_path.glob("*/*timestamps.csv"))[0] inscopix_metadata = json.load(open(session_metadata)) recording_timestamps = pd.read_csv(timestamps_file) From 2149c8c4c33819409b1609326e52439903e20553 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 1 Jul 2024 17:13:00 -0400 Subject: [PATCH 04/39] Remove CaImAn install from setup.py --- setup.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/setup.py b/setup.py index c7bb1a0..6db2c22 100644 --- a/setup.py +++ b/setup.py @@ -12,14 +12,6 @@ with open(path.join(here, pkg_name, "version.py")) as f: exec(f.read()) -with urllib.request.urlopen( - "https://raw.githubusercontent.com/flatironinstitute/CaImAn/master/requirements.txt" -) as f: - caiman_requirements = f.read().decode("UTF-8").split("\n") - -caiman_requirements.remove("") -caiman_requirements.append("future") - setup( name=pkg_name.replace("_", "-"), version=__version__, # noqa: F821 @@ -42,8 +34,6 @@ "element-interface @ git+https://github.com/datajoint/element-interface.git@staging", ], extras_require={ - "caiman_requirements": [caiman_requirements], - "caiman": ["caiman @ git+https://github.com/datajoint/CaImAn.git"], "elements": [ "element-animal @ git+https://github.com/datajoint/element-animal.git", "element-event @ git+https://github.com/datajoint/element-event.git", From 2d7418b44602d12303d4a66649f8a7d70af32afb Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 16:06:31 -0400 Subject: [PATCH 05/39] Update tables in miniscope schema + update devcontainer for codespaces --- .devcontainer/devcontainer.json | 3 +- element_miniscope/miniscope.py | 649 +++++++++++++++++++------------- install_caiman.py | 39 ++ 3 files changed, 421 insertions(+), 270 deletions(-) create mode 100644 install_caiman.py diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 41ef274..2c07199 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,7 +7,8 @@ "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" }, "onCreateCommand": "mkdir -p ${MINISCOPE_ROOT_DATA_DIR} && pip install -e .", - "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${MINISCOPE_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", + "postStartCommand": ["docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", + "cd element-miniscope && python install_caiman.py"], "hostRequirements": { "cpus": 4, "memory": "8gb", diff --git a/element_miniscope/miniscope.py b/element_miniscope/miniscope.py index 681ebb8..8b89fba 100644 --- a/element_miniscope/miniscope.py +++ b/element_miniscope/miniscope.py @@ -1,11 +1,12 @@ import csv +import cv2 import importlib import inspect import json import pathlib from datetime import datetime +from typing import Union -import cv2 import datajoint as dj import numpy as np import pandas as pd @@ -13,10 +14,11 @@ from . import miniscope_report +logger = dj.logger + schema = dj.Schema() _linking_module = None -logger = dj.logger def activate( @@ -92,6 +94,25 @@ def get_miniscope_root_data_dir() -> list: return root_directories +def get_processed_root_data_dir() -> Union[str, pathlib.Path]: + """Retrieve the root directory for all processed data. + + All data paths and directories in DataJoint Elements are recommended to be stored as + relative paths (posix format), with respect to some user-configured "root" + directory, which varies from machine to machine (e.g. different mounted drive + locations). + + Returns: + dir (str| pathlib.Path): Absolute path of the processed miniscope root data + directory. + """ + + if hasattr(_linking_module, "get_processed_root_data_dir"): + return _linking_module.get_processed_root_data_dir() + else: + return get_miniscope_root_data_dir()[0] + + def get_session_directory(session_key: dict) -> str: """Pulls session directory information from database. @@ -111,8 +132,10 @@ def get_session_directory(session_key: dict) -> str: class AcquisitionSoftware(dj.Lookup): """Software used for miniscope acquisition. + Required to define a miniscope recording. + Attributes: - acq_software (varchar(24) ): Name of the miniscope acquisition software.""" + acq_software (str): Name of the miniscope acquisition software.""" definition = """ acq_software: varchar(24) @@ -136,21 +159,21 @@ class Channel(dj.Lookup): @schema class Recording(dj.Manual): - """Table for discrete recording sessions with the miniscope. + """Recording defined by a measurement done using a scanner and an acquisition software. Attributes: - Session (foreign key): Session primary key. - recording_id (foreign key, int): Unique recording ID. - Device: Lookup table for miniscope device information. - AcquisitionSoftware: Lookup table for miniscope acquisition software. - recording_notes (varchar(4095) ): notes about the recording session. + Session (foreign key): A primary key from Session. + recording_id (int): Unique recording ID. + Device (foreign key, optional): A primary key from Device. + AcquisitionSoftware (foreign key): A primary key from AcquisitionSoftware. + recording_notes (str, optional): notes about the recording session. """ definition = """ -> Session recording_id: int --- - -> Device + -> [nullable] Device -> AcquisitionSoftware recording_notes='' : varchar(4095) # free-notes """ @@ -161,8 +184,8 @@ class RecordingLocation(dj.Manual): """Brain location where the miniscope recording is acquired. Attributes: - Recording (foreign key): Recording primary key. - Anatomical Location: Select the anatomical region where recording was acquired. + Recording (foreign key): A primary key from Recording. + AnatomicalLocation (foreign key): A primary key from AnatomicalLocation. """ definition = """ @@ -175,10 +198,10 @@ class RecordingLocation(dj.Manual): @schema class RecordingInfo(dj.Imported): - """Automated table with recording metadata. + """Information about the recording extracted from the recorded files. Attributes: - Recording (foreign key): Recording primary key. + Recording (foreign key): A primary key from Recording. nchannels (tinyint): Number of recording channels. nframes (int): Number of recorded frames. px_height (smallint): Height in pixels. @@ -207,12 +230,29 @@ class RecordingInfo(dj.Imported): fps : float # (Hz) frames per second gain=null : float # recording gain spatial_downsample=1 : tinyint # e.g. 1, 2, 4, 8. 1 for no downsampling - led_power : float # LED power used in the given recording time_stamps : longblob # time stamps of each frame recording_datetime=null : datetime # datetime of the recording recording_duration=null : float # (seconds) duration of the recording """ + class Channel(dj.Part): + """Channel information for each recording. + + Attributes: + RecordingInfo (foreign key): A primary key from RecordingInfo. + channel_id (tinyint): Channel number. + channel_gain (float, optional): Channel gain. + led_power (float, optional): LED power used for the channel. + """ + + definition = """ + -> master + channel_id : tinyint + --- + channel_gain=null : float # channel gain + led_power=null : float # LED power used for the channel + """ + class File(dj.Part): """File path to recording file relative to root data directory. @@ -361,19 +401,17 @@ def make(self, key): @schema class ProcessingMethod(dj.Lookup): - """Method or analysis software to process miniscope acquisition. + """Package used for processing of miniscope data (e.g. CaImAn, etc.). Attributes: - processing_method (foreign key, varchar16): Recording processing method (e.g. CaImAn). - processing_method_desc (varchar(1000) ): Additional information about the processing method. + processing_method (str): Processing method. + processing_method_desc (str): Processing method description. """ - definition = """ - # Method, package, analysis software used for processing of miniscope data - # (e.g. CaImAn, etc.) + definition = """# Package used for processing of calcium imaging data (e.g. Suite2p, CaImAn, etc.). processing_method: varchar(16) --- - processing_method_desc='': varchar(1000) + processing_method_desc: varchar(1000) """ contents = [("caiman", "caiman analysis suite")] @@ -381,25 +419,27 @@ class ProcessingMethod(dj.Lookup): @schema class ProcessingParamSet(dj.Lookup): - """Parameters of the processing method. + """Parameter set used for the processing of miniscope recordings., + including both the analysis suite and its respective input parameters. + + A hash of the parameters of the analysis suite is also stored in order + to avoid duplicated entries. Attributes: - paramset_idx (foreign key, smallint): Unique parameter set ID. - ProcessingMethod (varchar(16) ): ProcessingMethod from the lookup table. - paramset_desc (varchar(128) ): Description of the parameter set. - paramset_set_hash (uuid): UUID hash for parameter set. - params (longblob): Dictionary of all parameters for the processing method. + paramset_idx (int): Unique parameter set ID. + ProcessingMethod (foreign key): A primary key from ProcessingMethod. + paramset_desc (str): Parameter set description. + paramset_set_hash (uuid): A universally unique identifier for the parameter set. + params (longblob): Parameter set, a dictionary of all applicable parameters to the analysis suite. """ - definition = """ - # Parameter set used for processing of miniscope data - paramset_idx: smallint + definition = """# Processing Parameter set + paramset_idx: smallint # Unique parameter set ID. --- -> ProcessingMethod - paramset_desc: varchar(128) - param_set_hash: uuid - unique index (param_set_hash) - params: longblob # dictionary of all applicable parameters + paramset_desc: varchar(1280) # Parameter set description + param_set_hash: uuid # A universally unique identifier for the parameter set unique index (param_set_hash) + params: longblob # Parameter set, a dictionary of all applicable parameters to the analysis suite. """ @classmethod @@ -464,17 +504,23 @@ class MaskType(dj.Lookup): @schema class ProcessingTask(dj.Manual): - """Table marking manual or automatic processing task. + """A pairing of processing params and recordings to be loaded or triggered. + + This table defines a miniscope recording processing task for a combination of a + `Recording` and a `ProcessingParamSet` entries, including all the inputs (recording, method, + method's parameters). The task defined here is then run in the downstream table + `Processing`. This table supports definitions of both loading of pre-generated results + and the triggering of new analysis for all supported analysis methods. Attributes: - RecordingInfo (foreign key): Recording info primary key. - ProcessingParamSet (foreign key): Processing param set primary key. - processing_output_dir (varchar(255) ): relative output data directory for processed files. - task_mode (enum): `Load` existing results or `trigger` new processing task. + RecordingInfo (foreign key): Primary key from RecordingInfo. + ProcessingParamSet (foreign key): Primary key from ProcessingParamSet. + processing_output_dir (str): Output directory of the processed scan relative to the root data directory. + task_mode (str): One of 'load' (load computed analysis results) or 'trigger' + (trigger computation). """ - definition = """ - # Manual table marking a processing task to be triggered or manually processed + definition = """# Manual table for defining a processing task ready to be run -> RecordingInfo -> ProcessingParamSet --- @@ -483,31 +529,144 @@ class ProcessingTask(dj.Manual): # 'trigger': trigger procedure """ + @classmethod + def infer_output_dir(cls, key, relative=False, mkdir=False): + """Infer an output directory for an entry in ProcessingTask table. + + Args: + key (dict): Primary key from the ProcessingTask table. + relative (bool): If True, processing_output_dir is returned relative to + imaging_root_dir. Default False. + mkdir (bool): If True, create the processing_output_dir directory. + Default True. + + Returns: + dir (str): A default output directory for the processed results (processed_output_dir + in ProcessingTask) based on the following convention: + processed_dir / scan_dir / {processing_method}_{paramset_idx} + e.g.: sub4/sess1/scan0/suite2p_0 + """ + acq_software = (Recording & key).fetch1("acq_software") + recording_dir = find_full_path( + get_miniscope_root_data_dir(), + get_session_directory(key)[0], + ) + root_dir = find_root_directory(get_miniscope_root_data_dir(), recording_dir) + + method = ( + (ProcessingParamSet & key).fetch1("processing_method").replace(".", "-") + ) + + processed_dir = pathlib.Path(get_processed_root_data_dir()) + output_dir = ( + processed_dir + / recording_dir.relative_to(root_dir) + / f'{method}_{key["paramset_idx"]}' + ) + + if mkdir: + output_dir.mkdir(parents=True, exist_ok=True) + + return output_dir.relative_to(processed_dir) if relative else output_dir + + @classmethod + def generate(cls, recording_key, paramset_idx=0): + """Generate a ProcessingTask for a Recording using an parameter ProcessingParamSet + + Generate an entry in the ProcessingTask table for a particular recording using an + existing parameter set from the ProcessingParamSet table. + + Args: + recording_key (dict): Primary key from Recording. + paramset_idx (int): Unique parameter set ID. + """ + key = {**recording_key, "paramset_idx": paramset_idx} + + processed_dir = get_processed_root_data_dir() + output_dir = cls.infer_output_dir(key, relative=False, mkdir=True) + + method = (ProcessingParamSet & {"paramset_idx": paramset_idx}).fetch1( + "processing_method" + ) + + try: + if method == "caiman": + from element_interface import caiman_loader + + caiman_loader.CaImAn(output_dir) + else: + raise NotImplementedError( + "Unknown/unimplemented method: {}".format(method) + ) + except FileNotFoundError: + task_mode = "trigger" + else: + task_mode = "load" + + cls.insert1( + { + **key, + "processing_output_dir": output_dir.relative_to( + processed_dir + ).as_posix(), + "task_mode": task_mode, + } + ) + + auto_generate_entries = generate + @schema class Processing(dj.Computed): - """Automatic table that beings the miniscope processing pipeline. + """Perform the computation of an entry (task) defined in the ProcessingTask table. + The computation is performed only on the recordings with RecordingInfo inserted. + Attributes: - ProcessingTask (foreign key): Processing task primary key. - processing_time (datetime): Generates time of the processed results. - package_version (varchar(16) ): Package version information. + ProcessingTask (foreign key): Primary key from ProcessingTask. + processing_time (datetime): Process completion datetime. + package_version (str, optional): Version of the analysis package used in processing the data. """ definition = """ -> ProcessingTask --- - processing_time : datetime # generation time of processed, segmented results + processing_time : datetime # generation time of processed results package_version='' : varchar(16) """ + @property + def key_source(self): + return ProcessingTask & RecordingInfo + def make(self, key): - """Triggers processing and populates Processing table.""" - task_mode = (ProcessingTask & key).fetch1("task_mode") - - output_dir = (ProcessingTask & key).fetch1("processing_output_dir") - output_dir = find_full_path(get_miniscope_root_data_dir(), output_dir) + """ + Execute the miniscope analysis defined by the ProcessingTask. + - task_mode: 'load', confirm that the results are already computed. + - task_mode: 'trigger' runs the analysis. + """ + task_mode, output_dir = (ProcessingTask & key).fetch1( + "task_mode", "processing_output_dir" + ) + if not output_dir: + output_dir = ProcessingTask.infer_output_dir(key, relative=True, mkdir=True) + # update processing_output_dir + ProcessingTask.update1( + {**key, "processing_output_dir": output_dir.as_posix()} + ) + try: + output_dir = find_full_path( + get_miniscope_root_data_dir(), output_dir + ).as_posix() + except FileNotFoundError as e: + if task_mode == "trigger": + processed_dir = pathlib.Path(get_processed_root_data_dir()) + output_dir = processed_dir / output_dir + output_dir.mkdir(parents=True, exist_ok=True) + else: + raise e + if task_mode == "load": method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": @@ -586,11 +745,11 @@ class Curation(dj.Manual): Attributes: Processing (foreign key): Processing primary key. - curation_id (foreign key, int): Unique curation ID. + curation_id (int): Unique curation ID. curation_time (datetime): Time of generation of curated results. - curation_output_dir (varchar(255) ): Output directory for curated results. + curation_output_dir (str): Output directory for curated results. manual_curation (bool): If True, manual curation has been performed. - curation_note (varchar(2000) ): Optional description of the curation procedure. + curation_note (str, optional): Optional description of the curation procedure. """ definition = """ @@ -660,100 +819,115 @@ class MotionCorrection(dj.Imported): """ class RigidMotionCorrection(dj.Part): - """Automated table with ridge motion correction data. + """Details of rigid motion correction performed on the imaging data. Attributes: - MotionCorrection (foreign key): MotionCorrection primary key. - outlier_frames (longblob): Mask with true for frames with outlier shifts. - y_shifts (longblob): y motion correction shifts, pixels. - x_shifts (longblob): x motion correction shifts, pixels. - y_std (float): Standard deviation of y shifts across all frames, pixels. - x_std (float): Standard deviation of x shifts across all frames, pixels. + MotionCorrection (foreign key): Primary key from MotionCorrection. + outlier_frames (longblob): Mask with true for frames with outlier shifts + (already corrected). + y_shifts (longblob): y motion correction shifts (pixels). + x_shifts (longblob): x motion correction shifts (pixels). + z_shifts (longblob, optional): z motion correction shifts (z-drift, pixels). + y_std (float): standard deviation of y shifts across all frames (pixels). + x_std (float): standard deviation of x shifts across all frames (pixels). + z_std (float, optional): standard deviation of z shifts across all frames + (pixels). """ - definition = """ + definition = """# Details of rigid motion correction performed on the imaging data -> master --- - outlier_frames=null : longblob # mask with true for frames with outlier shifts - # (already corrected) + outlier_frames=null : longblob # mask with true for frames with outlier shifts (already corrected) y_shifts : longblob # (pixels) y motion correction shifts x_shifts : longblob # (pixels) x motion correction shifts - y_std : float # (pixels) standard deviation of - # y shifts across all frames - x_std : float # (pixels) standard deviation of - # x shifts across all frames + z_shifts=null : longblob # (pixels) z motion correction shifts (z-drift) + y_std : float # (pixels) standard deviation of y shifts across all frames + x_std : float # (pixels) standard deviation of x shifts across all frames + z_std=null : float # (pixels) standard deviation of z shifts across all frames """ class NonRigidMotionCorrection(dj.Part): - """Automated table with piece-wise rigid motion correction data. + """Piece-wise rigid motion correction - tile the FOV into multiple 3D + blocks/patches. Attributes: - MotionCorrection (foreign key): MotionCorrection primary key. - outlier_frames (longblob): Mask with true for frames with outlier shifts (already corrected). - block_height (int): Height in pixels. - block_width (int): Width in pixels. + MotionCorrection (foreign key): Primary key from MotionCorrection. + outlier_frames (longblob, null): Mask with true for frames with outlier + shifts (already corrected). + block_height (int): Block height in pixels. + block_width (int): Block width in pixels. + block_depth (int): Block depth in pixels. block_count_y (int): Number of blocks tiled in the y direction. block_count_x (int): Number of blocks tiled in the x direction. + block_count_z (int): Number of blocks tiled in the z direction. """ - definition = """ + definition = """# Details of non-rigid motion correction performed on the imaging data -> master --- - outlier_frames=null : longblob # mask with true for frames with - # outlier shifts (already corrected) - block_height : int # (pixels) - block_width : int # (pixels) - block_count_y : int # number of blocks tiled in the - # y direction - block_count_x : int # number of blocks tiled in the - # x direction + outlier_frames=null : longblob # mask with true for frames with outlier shifts (already corrected) + block_height : int # (pixels) + block_width : int # (pixels) + block_depth : int # (pixels) + block_count_y : int # number of blocks tiled in the y direction + block_count_x : int # number of blocks tiled in the x direction + block_count_z : int # number of blocks tiled in the z direction """ class Block(dj.Part): - """Automated table with data for blocks used in non-rigid motion correction. + """FOV-tiled blocks used for non-rigid motion correction. Attributes: - master.NonRigidMotionCorrection (foreign key): NonRigidMotionCorrection primary key. - block_id (foreign key, int): Unique ID for each block. - block_y (longblob): y_start and y_end of this block in pixels. - block_x (longblob): x_start and x_end of this block in pixels. - y_shifts (longblob): y motion correction shifts for every frame in pixels. - x_shifts (longblob): x motion correction shifts for every frame in pixels. - y_std (float): standard deviation of y shifts across all frames in pixels. - x_std (float): standard deviation of x shifts across all frames in pixels. + NonRigidMotionCorrection (foreign key): Primary key from + NonRigidMotionCorrection. + block_id (int): Unique block ID. + block_y (longblob): y_start and y_end in pixels for this block + block_x (longblob): x_start and x_end in pixels for this block + block_z (longblob): z_start and z_end in pixels for this block + y_shifts (longblob): y motion correction shifts for every frame in pixels + x_shifts (longblob): x motion correction shifts for every frame in pixels + z_shift=null (longblob, optional): x motion correction shifts for every frame + in pixels + y_std (float): standard deviation of y shifts across all frames in pixels + x_std (float): standard deviation of x shifts across all frames in pixels + z_std=null (float, optional): standard deviation of z shifts across all frames + in pixels """ - definition = """ # FOV-tiled blocks used for non-rigid motion correction + definition = """# FOV-tiled blocks used for non-rigid motion correction -> master.NonRigidMotionCorrection - block_id : int + block_id : int --- - block_y : longblob # (y_start, y_end) in pixel of this block - block_x : longblob # (x_start, x_end) in pixel of this block - y_shifts : longblob # (pixels) y motion correction shifts for every frame - x_shifts : longblob # (pixels) x motion correction shifts for every frame - y_std : float # (pixels) standard deviation of y shifts across all frames - x_std : float # (pixels) standard deviation of x shifts across all frames + block_y : longblob # (y_start, y_end) in pixel of this block + block_x : longblob # (x_start, x_end) in pixel of this block + block_z : longblob # (z_start, z_end) in pixel of this block + y_shifts : longblob # (pixels) y motion correction shifts for every frame + x_shifts : longblob # (pixels) x motion correction shifts for every frame + z_shifts=null : longblob # (pixels) z motion correction shifts for every frame + y_std : float # (pixels) standard deviation of y shifts across all frames + x_std : float # (pixels) standard deviation of x shifts across all frames + z_std=null : float # (pixels) standard deviation of z shifts across all frames """ class Summary(dj.Part): - """A summary image for each field and channel after motion correction. + """Summary images for each field and channel after corrections. Attributes: - MotionCorrection (foreign key): MotionCorrection primary key. - ref_image (longblob): Image used as the alignment template. + MotionCorrection (foreign key): Primary key from MotionCorrection. + ref_image (longblob): Image used as alignment template. average_image (longblob): Mean of registered frames. - correlation_image (longblob): Correlation map computed during cell detection. - max_proj_image (longblob): Maximum of registered frames. + correlation_image (longblob, optional): Correlation map (computed during + cell detection). + max_proj_image (longblob, optional): Max of registered frames. """ - definition = """ # summary images for each field and channel after corrections + definition = """# Summary images for each field and channel after corrections -> master --- - ref_image=null : longblob # image used as alignment template - average_image : longblob # mean of registered frames - correlation_image=null : longblob # correlation map - # (computed during cell detection) - max_proj_image=null : longblob # max of registered frames + ref_image : longblob # image used as alignment template + average_image : longblob # mean of registered frames + correlation_image=null : longblob # correlation map (computed during cell detection) + max_proj_image=null : longblob # max of registered frames """ def make(self, key): @@ -761,104 +935,38 @@ def make(self, key): method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": - loaded_caiman = loaded_result + caiman_dataset = loaded_result self.insert1( - {**key, "motion_correct_channel": loaded_caiman.alignment_channel} + {**key, "motion_correct_channel": caiman_dataset.alignment_channel} ) # -- rigid motion correction -- - if not loaded_caiman.params.motion["pw_rigid"]: - rigid_correction = { - **key, - "x_shifts": loaded_caiman.motion_correction["shifts_rig"][:, 0], - "y_shifts": loaded_caiman.motion_correction["shifts_rig"][:, 1], - "x_std": np.nanstd( - loaded_caiman.motion_correction["shifts_rig"][:, 0] - ), - "y_std": np.nanstd( - loaded_caiman.motion_correction["shifts_rig"][:, 1] - ), - "outlier_frames": None, - } - - self.RigidMotionCorrection.insert1(rigid_correction) - - # -- non-rigid motion correction -- - else: - nonrigid_correction = { - **key, - "block_height": ( - loaded_caiman.params.motion["strides"][0] - + loaded_caiman.params.motion["overlaps"][0] - ), - "block_width": ( - loaded_caiman.params.motion["strides"][1] - + loaded_caiman.params.motion["overlaps"][1] - ), - "block_count_x": len( - set(loaded_caiman.motion_correction["coord_shifts_els"][:, 0]) - ), - "block_count_y": len( - set(loaded_caiman.motion_correction["coord_shifts_els"][:, 2]) - ), - "outlier_frames": None, - } - - nonrigid_blocks = [] - for b_id in range( - len(loaded_caiman.motion_correction["x_shifts_els"][0, :]) - ): - nonrigid_blocks.append( - { - **key, - "block_id": b_id, - "block_x": np.arange( - *loaded_caiman.motion_correction["coord_shifts_els"][ - b_id, 0:2 - ] - ), - "block_y": np.arange( - *loaded_caiman.motion_correction["coord_shifts_els"][ - b_id, 2:4 - ] - ), - "x_shifts": loaded_caiman.motion_correction["x_shifts_els"][ - :, b_id - ], - "y_shifts": loaded_caiman.motion_correction["y_shifts_els"][ - :, b_id - ], - "x_std": np.nanstd( - loaded_caiman.motion_correction["x_shifts_els"][:, b_id] - ), - "y_std": np.nanstd( - loaded_caiman.motion_correction["y_shifts_els"][:, b_id] - ), - } - ) - + if caiman_dataset.is_pw_rigid: + # -- non-rigid motion correction -- + ( + nonrigid_correction, + nonrigid_blocks, + ) = caiman_dataset.extract_pw_rigid_mc() + nonrigid_correction.update(**key) + nonrigid_blocks.update(**key) self.NonRigidMotionCorrection.insert1(nonrigid_correction) self.Block.insert(nonrigid_blocks) + else: + # -- rigid motion correction -- + rigid_correction = caiman_dataset.extract_rigid_mc() + rigid_correction.update(**key) + self.RigidMotionCorrection.insert1(rigid_correction) # -- summary images -- summary_images = { - **key, - "ref_image": loaded_caiman.motion_correction["reference_image"][...][ - np.newaxis, ... - ], - "average_image": loaded_caiman.motion_correction["average_image"][...][ - np.newaxis, ... - ], - "correlation_image": loaded_caiman.motion_correction[ - "correlation_image" - ][...][np.newaxis, ...], - "max_proj_image": loaded_caiman.motion_correction["max_image"][...][ - np.newaxis, ... - ], - } - - self.Summary.insert1(summary_images) + **key, + "ref_image": caiman_dataset.ref_image, + "average_image": caiman_dataset.mean_image, + "correlation_image": caiman_dataset.correlation_map, + "max_proj_image": caiman_dataset.max_proj_image, + } + self.Summary.insert(summary_images) else: raise NotImplementedError("Unknown/unimplemented method: {}".format(method)) @@ -880,31 +988,36 @@ class Segmentation(dj.Computed): """ class Mask(dj.Part): - """Image masks produced during segmentation. + """Details of the masks identified from the Segmentation procedure. Attributes: - Segmentation (foreign key): Segmentation primary key. - mask (smallint): Unique ID for each mask. - channel.proj(segmentation_channel='channel') (query): Channel to be used for segmentation. - mask_npix (int): Number of pixels in the mask. - mask_center_x (int): Center x coordinate in pixels. - mask_center_y (int): Center y coordinate in pixels. - mask_xpix (longblob): x coordinates of the mask in pixels. - mask_ypix (longblob): y coordinates of the mask in pixels. - mask_weights (longblob): weights of the mask at the indices above. + Segmentation (foreign key): Primary key from Segmentation. + mask (int): Unique mask ID. + Channel.proj(segmentation_channel='channel') (foreign key): Channel + used for segmentation. + mask_npix (int): Number of pixels in ROIs. + mask_center_x (int): Center x coordinate in pixel. + mask_center_y (int): Center y coordinate in pixel. + mask_center_z (int): Center z coordinate in pixel. + mask_xpix (longblob): X coordinates in pixels. + mask_ypix (longblob): Y coordinates in pixels. + mask_zpix (longblob): Z coordinates in pixels. + mask_weights (longblob): Weights of the mask at the indices above. """ definition = """ # A mask produced by segmentation. -> master - mask : smallint + mask : smallint --- -> Channel.proj(segmentation_channel='channel') # channel used for segmentation - mask_npix : int # number of pixels in this mask - mask_center_x=null : int # (pixels) center x coordinate - mask_center_y=null : int # (pixels) center y coordinate - mask_xpix=null : longblob # (pixels) x coordinates - mask_ypix=null : longblob # (pixels) y coordinates - mask_weights : longblob # weights of the mask at the indices above + mask_npix : int # number of pixels in ROIs + mask_center_x : int # center x coordinate in pixel + mask_center_y : int # center y coordinate in pixel + mask_center_z=null : int # center z coordinate in pixel + mask_xpix : longblob # x coordinates in pixels + mask_ypix : longblob # y coordinates in pixels + mask_zpix=null : longblob # z coordinates in pixels + mask_weights : longblob # weights of the mask at the indices above """ def make(self, key): @@ -912,53 +1025,50 @@ def make(self, key): method, loaded_result = get_loader_result(key, Curation) if method == "caiman": - loaded_caiman = loaded_result + caiman_dataset = loaded_result - # infer `segmentation_channel` from `params` if available, - # else from caiman loader + # infer "segmentation_channel" - from params if available, else from caiman loader params = (ProcessingParamSet * ProcessingTask & key).fetch1("params") segmentation_channel = params.get( - "segmentation_channel", loaded_caiman.segmentation_channel + "segmentation_channel", caiman_dataset.segmentation_channel ) masks, cells = [], [] - for mask in loaded_caiman.masks: - # Sample data had _id key, not mask. Permitting both - mask_id = mask.get("mask", mask["mask_id"]) + for mask in caiman_dataset.masks: masks.append( { **key, "segmentation_channel": segmentation_channel, - "mask": mask_id, + "mask": mask["mask_id"], "mask_npix": mask["mask_npix"], "mask_center_x": mask["mask_center_x"], "mask_center_y": mask["mask_center_y"], + "mask_center_z": mask["mask_center_z"], "mask_xpix": mask["mask_xpix"], "mask_ypix": mask["mask_ypix"], + "mask_zpix": mask["mask_zpix"], "mask_weights": mask["mask_weights"], } ) - - if loaded_caiman.cnmf.estimates.idx_components is not None: - if mask_id in loaded_caiman.cnmf.estimates.idx_components: - cells.append( - { - **key, - "mask_classification_method": "caiman_default_classifier", - "mask": mask_id, - "mask_type": "soma", - } - ) - - if not all([all(m.values()) for m in masks]): - logger.warning("Could not load all pixel values for at least one mask") + if mask["accepted"]: + cells.append( + { + **key, + "mask_classification_method": "caiman_default_classifier", + "mask": mask["mask_id"], + "mask_type": "soma", + } + ) self.insert1(key) self.Mask.insert(masks, ignore_extra_fields=True) if cells: MaskClassification.insert1( - {**key, "mask_classification_method": "caiman_default_classifier"}, + { + **key, + "mask_classification_method": "caiman_default_classifier", + }, allow_direct_insert=True, ) MaskClassification.MaskType.insert( @@ -1065,27 +1175,27 @@ def make(self, key): method, loaded_result = get_loader_result(key, Curation) if method == "caiman": - loaded_caiman = loaded_result + caiman_dataset = loaded_result - # infer `segmentation_channel` from `params` if available, - # else from caiman loader + # infer "segmentation_channel" - from params if available, else from caiman loader params = (ProcessingParamSet * ProcessingTask & key).fetch1("params") segmentation_channel = params.get( - "segmentation_channel", loaded_caiman.segmentation_channel + "segmentation_channel", caiman_dataset.segmentation_channel ) - self.insert1(key) - self.Trace.insert( - [ + fluo_traces = [] + for mask in caiman_dataset.masks: + fluo_traces.append( { **key, - "mask": mask.get("mask", mask["mask_id"]), - "fluorescence_channel": segmentation_channel, + "mask": mask["mask_id"], + "fluo_channel": segmentation_channel, "fluorescence": mask["inferred_trace"], } - for mask in loaded_caiman.masks - ] - ) + ) + + self.insert1(key) + self.Trace.insert(fluo_traces) else: raise NotImplementedError("Unknown/unimplemented method: {}".format(method)) @@ -1155,32 +1265,33 @@ def make(self, key): method, loaded_result = get_loader_result(key, Curation) if method == "caiman": - loaded_caiman = loaded_result - - if key["extraction_method"] in ("caiman_deconvolution", "caiman_dff"): - attr_mapper = {"caiman_deconvolution": "spikes", "caiman_dff": "dff"} + caiman_dataset = loaded_result + + if key["extraction_method"] in ( + "caiman_deconvolution", + "caiman_dff", + ): + attr_mapper = { + "caiman_deconvolution": "spikes", + "caiman_dff": "dff", + } - # infer `segmentation_channel` from `params` if available, - # else from caiman loader + # infer "segmentation_channel" - from params if available, else from caiman loader params = (ProcessingParamSet * ProcessingTask & key).fetch1("params") segmentation_channel = params.get( - "segmentation_channel", loaded_caiman.segmentation_channel + "segmentation_channel", caiman_dataset.segmentation_channel ) self.insert1(key) self.Trace.insert( - [ - { - **key, - "mask": mask.get("mask", mask["mask_id"]), - "fluorescence_channel": segmentation_channel, - "activity_trace": mask[ - attr_mapper[key["extraction_method"]] - ], - } - for mask in loaded_caiman.masks - ] + dict( + key, + mask=mask["mask_id"], + fluo_channel=segmentation_channel, + activity_trace=mask[attr_mapper[key["extraction_method"]]], + ) for mask in caiman_dataset.masks ) + else: raise NotImplementedError("Unknown/unimplemented method: {}".format(method)) diff --git a/install_caiman.py b/install_caiman.py new file mode 100644 index 0000000..91d5d44 --- /dev/null +++ b/install_caiman.py @@ -0,0 +1,39 @@ +import os +import subprocess +import sys + +def run_command(command): + """Run a system command and ensure it completes successfully.""" + result = subprocess.run(command, shell=True) + if result.returncode != 0: + print(f"Command failed with return code {result.returncode}: {command}") + sys.exit(result.returncode) + +def main(env_name="element-miniscope-env"): + conda_executable = 'conda' + mamba_executable = 'mamba' + + # Step 1: Create the Conda Environment + print(f"Creating conda environment: {env_name}") + run_command(f"{conda_executable} create -n {env_name} -y") + + # Step 2: Activate the Environment + print(f"Activating conda environment: {env_name}") + run_command(f"{conda_executable} activate base") + run_command(f"{conda_executable} activate C:/Users/kusha/miniconda3/envs/{env_name}") + + # Step 3: Install CaImAn and its dependencies + print("Installing CaImAn and its dependencies") + run_command(f"{conda_executable} install -c conda-forge mamba -y") + run_command(f"{mamba_executable} install -c conda-forge python==3.10 -y") + run_command(f"{mamba_executable} install -c conda-forge caiman -y") + run_command("pip install keras==2.15.0") + +if __name__ == "__main__": + import argparse + + parser = argparse.ArgumentParser(description="Install CaImAn and element-miniscope dependencies.") + parser.add_argument('-e', '--env', type=str, default="element-miniscope-env", help="Name of the conda environment to create and use. Default is 'element-miniscope-env'.") + args = parser.parse_args() + + main(args.env) From e9ba34fbcec175f62d3dc9a5e04f766ccdb17d49 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 16:15:56 -0400 Subject: [PATCH 06/39] Attempt caiman install from dockerfile during devcontainer build --- .devcontainer/Dockerfile | 6 +++--- .devcontainer/devcontainer.json | 3 +-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 7cb9a7b..f893736 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -32,9 +32,9 @@ COPY ./ /tmp/element-miniscope/ RUN \ # pipeline dependencies apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \ - pip install numpy Cython && \ - pip install --no-cache-dir -e /tmp/element-miniscope[elements,caiman_requirements,caiman] && \ - caimanmanager.py install && \ + pip install --no-cache-dir -e /tmp/element-miniscope[elements] +RUN cd /tmp/element-miniscope && \ + python install_caiman.py && \ # clean up rm -rf /tmp/element-miniscope && \ apt-get clean diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2c07199..e616754 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,8 +7,7 @@ "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" }, "onCreateCommand": "mkdir -p ${MINISCOPE_ROOT_DATA_DIR} && pip install -e .", - "postStartCommand": ["docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", - "cd element-miniscope && python install_caiman.py"], + "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", "hostRequirements": { "cpus": 4, "memory": "8gb", From 715fed79c55d1b38cbd42288cc4592f58723ba84 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 16:25:15 -0400 Subject: [PATCH 07/39] Clone and install caiman in image build --- .devcontainer/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index f893736..d3e121d 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,8 +33,8 @@ RUN \ # pipeline dependencies apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \ pip install --no-cache-dir -e /tmp/element-miniscope[elements] -RUN cd /tmp/element-miniscope && \ - python install_caiman.py && \ +RUN cd ./tmp && git clone git+https://github.com/datajoint/CaImAn.git && cd ./CaImAn && \ + pip install -e . && caimanmanager.py install --inplace && cd ~ && \ # clean up rm -rf /tmp/element-miniscope && \ apt-get clean From b954899ddad4a506a923b2428f3195a283f0b3bb Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 16:46:16 -0400 Subject: [PATCH 08/39] Install caiman in the devcontainer image --- .devcontainer/Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d3e121d..c526f03 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -33,10 +33,12 @@ RUN \ # pipeline dependencies apt-get install gcc g++ ffmpeg libsm6 libxext6 -y && \ pip install --no-cache-dir -e /tmp/element-miniscope[elements] -RUN cd ./tmp && git clone git+https://github.com/datajoint/CaImAn.git && cd ./CaImAn && \ - pip install -e . && caimanmanager.py install --inplace && cd ~ && \ +RUN cd ./tmp && git clone https://github.com/datajoint/CaImAn.git && cd ./CaImAn && \ + pip install -r requirements.txt && pip install -e . && \ + caimanmanager install --inplace && cd ~ && \ # clean up rm -rf /tmp/element-miniscope && \ + rm -rf /tmp/CaImAn && \ apt-get clean ENV DJ_HOST fakeservices.datajoint.io From dd4a3970804a562869ed5d387c15274c862582c6 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 17:01:54 -0400 Subject: [PATCH 09/39] Update devcontainer to python 3.10 --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c526f03..c27c0be 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3.9-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f +FROM python:3.10-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f ENV PATH /usr/local/bin:$PATH ENV PYTHON_VERSION 3.9.17 From d321be19657df7885854bbe1462751efcc6775d2 Mon Sep 17 00:00:00 2001 From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com> Date: Wed, 17 Jul 2024 21:07:45 +0000 Subject: [PATCH 10/39] Fix(devcontainer.json) imaging -> miniscope --- .devcontainer/devcontainer.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index e616754..41ef274 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -7,7 +7,7 @@ "LOCAL_WORKSPACE_FOLDER": "${localWorkspaceFolder}" }, "onCreateCommand": "mkdir -p ${MINISCOPE_ROOT_DATA_DIR} && pip install -e .", - "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${IMAGING_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", + "postStartCommand": "docker volume prune -f && s3fs ${DJ_PUBLIC_S3_LOCATION} ${MINISCOPE_ROOT_DATA_DIR} -o nonempty,multipart_size=530,endpoint=us-east-1,url=http://s3.amazonaws.com,public_bucket=1", "hostRequirements": { "cpus": 4, "memory": "8gb", From 914a55d9f3c91c862419f5c716beb71de4553b60 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 17 Jul 2024 17:11:13 -0400 Subject: [PATCH 11/39] Update `PYTHON_VERSION` env --- .devcontainer/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index c27c0be..6c34fc5 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.10-slim@sha256:5f0192a4f58a6ce99f732fe05e3b3d00f12ae62e183886bca3ebe3d202686c7f ENV PATH /usr/local/bin:$PATH -ENV PYTHON_VERSION 3.9.17 +ENV PYTHON_VERSION 3.10.13 RUN \ adduser --system --disabled-password --shell /bin/bash vscode && \ From a393a383b8020bac0510596213475d9ffa818cec Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 18 Jul 2024 10:26:37 -0400 Subject: [PATCH 12/39] Create `no_curation` module --- ...{miniscope.py => miniscope_no_curation.py} | 69 ++----------------- 1 file changed, 4 insertions(+), 65 deletions(-) rename element_miniscope/{miniscope.py => miniscope_no_curation.py} (95%) diff --git a/element_miniscope/miniscope.py b/element_miniscope/miniscope_no_curation.py similarity index 95% rename from element_miniscope/miniscope.py rename to element_miniscope/miniscope_no_curation.py index 8b89fba..c2a3be3 100644 --- a/element_miniscope/miniscope.py +++ b/element_miniscope/miniscope_no_curation.py @@ -739,66 +739,6 @@ def make(self, key): self.insert1(key) -@schema -class Curation(dj.Manual): - """Defines whether and how the results should be curated. - - Attributes: - Processing (foreign key): Processing primary key. - curation_id (int): Unique curation ID. - curation_time (datetime): Time of generation of curated results. - curation_output_dir (str): Output directory for curated results. - manual_curation (bool): If True, manual curation has been performed. - curation_note (str, optional): Optional description of the curation procedure. - """ - - definition = """ - # Different rounds of curation performed on the processing results of the data - # (no-curation can also be included here) - -> Processing - curation_id: int - --- - curation_time: datetime # time of generation of these curated results - curation_output_dir: varchar(255) # output directory of the curated results, - # relative to root data directory - manual_curation: bool # has manual curation been performed? - curation_note='': varchar(2000) - """ - - @classmethod - def create1_from_processing_task(self, key, is_curated=False, curation_note=""): - """Given a "ProcessingTask", create a new corresponding "Curation" """ - if key not in Processing(): - raise ValueError( - f"No corresponding entry in Processing available for: " - f"{key}; run `Processing.populate(key)`" - ) - - output_dir = (ProcessingTask & key).fetch1("processing_output_dir") - method, imaging_dataset = get_loader_result(key, ProcessingTask) - - if method == "caiman": - caiman_dataset = imaging_dataset - curation_time = caiman_dataset.creation_time - else: - raise NotImplementedError("Unknown method: {}".format(method)) - - # Synthesize curation_id - curation_id = ( - dj.U().aggr(self & key, n="ifnull(max(curation_id)+1,1)").fetch1("n") - ) - self.insert1( - { - **key, - "curation_id": curation_id, - "curation_time": curation_time, - "curation_output_dir": output_dir, - "manual_curation": is_curated, - "curation_note": curation_note, - } - ) - - # Motion Correction -------------------------------------------------------------------- @@ -807,12 +747,12 @@ class MotionCorrection(dj.Imported): """Automated table performing motion correction analysis. Attributes: - Curation (foreign key): Curation primary key. + Processing (foreign key): Processing primary key. Channel.proj(motion_correct_channel='channel'): Channel used for motion correction. """ definition = """ - -> Curation + -> Processing --- -> Channel.proj(motion_correct_channel='channel') # channel used for # motion correction @@ -980,11 +920,11 @@ class Segmentation(dj.Computed): """Automated table computes different mask segmentations. Attributes: - Curation (foreign key): Curation primary key. + Processing (foreign key): Processing primary key. """ definition = """ # Different mask segmentations. - -> Curation + -> Processing """ class Mask(dj.Part): @@ -1364,7 +1304,6 @@ def make(self, key): _table_attribute_mapper = { "ProcessingTask": "processing_output_dir", - "Curation": "curation_output_dir", } From b566fd9ccde60a3de0652038626c73cdb94299dc Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 29 Jul 2024 18:11:20 -0400 Subject: [PATCH 13/39] Update with changes from code review and testing --- element_miniscope/miniscope_no_curation.py | 111 +++++++++++---------- 1 file changed, 59 insertions(+), 52 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index c2a3be3..af29238 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -212,7 +212,6 @@ class RecordingInfo(dj.Imported): gain (float): Recording gain. spatial_downsample (tinyint): Amount of downsampling applied. led_power (float): LED power used for the recording. - time_stamps (longblob): Time stamps for each frame. recording_datetime (datetime): Datetime of the recording. recording_duration (float): Total recording duration (seconds). """ @@ -223,34 +222,40 @@ class RecordingInfo(dj.Imported): --- nchannels : tinyint # number of channels nframes : int # number of recorded frames - px_height=null : smallint # height in pixels - px_width=null : smallint # width in pixels - um_height=null : float # height in microns - um_width=null : float # width in microns + ndepths=1 : tinyint # number of depths + px_height : smallint # height in pixels + px_width : smallint # width in pixels fps : float # (Hz) frames per second - gain=null : float # recording gain - spatial_downsample=1 : tinyint # e.g. 1, 2, 4, 8. 1 for no downsampling - time_stamps : longblob # time stamps of each frame recording_datetime=null : datetime # datetime of the recording recording_duration=null : float # (seconds) duration of the recording """ - class Channel(dj.Part): - """Channel information for each recording. + class Config(dj.Part): + """Recording metadata and configuration. Attributes: - RecordingInfo (foreign key): A primary key from RecordingInfo. - channel_id (tinyint): Channel number. - channel_gain (float, optional): Channel gain. - led_power (float, optional): LED power used for the channel. + Recording (foreign key): A primary key from RecordingInfo. + config (longblob): Recording metadata and configuration. """ definition = """ -> master - channel_id : tinyint --- - channel_gain=null : float # channel gain - led_power=null : float # LED power used for the channel + config: longblob # recording metadata and configuration + """ + + class Timestamps(dj.Part): + """Recording timestamps for each frame. + + Attributes: + Recording (foreign key): A primary key from RecordingInfo. + timestamps (longblob): Recording timestamps for each frame. + """ + + definition = """ + -> master + --- + timestamps: longblob """ class File(dj.Part): @@ -280,10 +285,11 @@ def make(self, key): get_miniscope_root_data_dir(), recording_directory ) - recording_filepaths = [ - file_path.as_posix() for file_path in recording_path.glob("*.avi") - ] if acq_software != "Inscopix" else [ - file_path.as_posix() for file_path in recording_path.rglob("*.avi")] + recording_filepaths = ( + [file_path.as_posix() for file_path in recording_path.glob("*.avi")] + if acq_software != "Inscopix" + else [file_path.as_posix() for file_path in recording_path.rglob("*.avi")] + ) if not recording_filepaths: raise FileNotFoundError(f"No .avi files found in " f"{recording_directory}") @@ -311,14 +317,15 @@ def make(self, key): fps = video.get(cv2.CAP_PROP_FPS) elif acq_software == "Miniscope-DAQ-V4": - recording_metadata = list(recording_path.glob("metaData.json"))[0] - recording_timestamps = list(recording_path.glob("timeStamps.csv"))[0] - - if not recording_metadata.exists(): + try: + recording_metadata = next(recording_path.glob("metaData.json")) + except StopIteration: raise FileNotFoundError( f"No .json file found in " f"{recording_directory}" ) - if not recording_timestamps.exists(): + try: + recording_timestamps = next(recording_path.glob("timeStamps.csv")) + except StopIteration: raise FileNotFoundError( f"No timestamp (*.csv) file found in " f"{recording_directory}" ) @@ -334,29 +341,21 @@ def make(self, key): px_height = metadata["ROI"]["height"] px_width = metadata["ROI"]["width"] fps = int(metadata["frameRate"].replace("FPS", "")) - gain = metadata["gain"] - spatial_downsample = 1 # Assumes no spatial downsampling - led_power = metadata["led0"] - time_stamps = np.array( - [list(map(int, time_stamps[i])) for i in range(1, len(time_stamps))] - ) + time_stamps = np.array(time_stamps[1:], dtype=float)[:, 0] elif acq_software == "Inscopix": - session_metadata = list(recording_path.glob("apply_session.json"))[0] - timestamps_file = list(recording_path.glob("*/*timestamps.csv"))[0] - inscopix_metadata = json.load(open(session_metadata)) + inscopix_metadata = next(recording_path.glob("session.json")) + timestamps_file = next(recording_path.glob("*/*timestamps.csv")) + metadata = json.load(open(inscopix_metadata)) recording_timestamps = pd.read_csv(timestamps_file) - nchannels = len(inscopix_metadata["manual"]["mScope"]["ledMaxPower"]) + nchannels = len(metadata["manual"]["mScope"]["ledMaxPower"]) nframes = len(recording_timestamps) - fps = inscopix_metadata["microscope"]["fps"]["fps"] - gain = inscopix_metadata["microscope"]["gain"] - led_power = inscopix_metadata["microscope"]["led"]["exPower"] + fps = metadata["microscope"]["fps"]["fps"] time_stamps = (recording_timestamps[" time (ms)"] / 1000).values - px_height = None - px_width = None - spatial_downsample = None - + px_height = metadata["microscope"]["fov"]["height"] + px_width = metadata["microscope"]["fov"]["width"] + else: raise NotImplementedError( f"Loading routine not implemented for {acq_software}" @@ -372,10 +371,6 @@ def make(self, key): px_height=px_height, px_width=px_width, fps=fps, - gain=gain, - spatial_downsample=spatial_downsample, - led_power=led_power, - time_stamps=time_stamps, recording_duration=nframes / fps, ) ) @@ -395,6 +390,15 @@ def make(self, key): ] ) + if acq_software == "Inscopix" or acq_software == "Miniscope-DAQ-V4": + self.Timestamps.insert1(dict(**key, timestamps=time_stamps)) + self.Config.insert1( + dict( + **key, + config=metadata, + ) + ) + # Trigger a processing routine ------------------------------------------------- @@ -464,7 +468,10 @@ def insert_new_params( """ ProcessingMethod.insert1( - {"processing_method": processing_method}, skip_duplicates=True + { + "processing_method": processing_method, + "processing_method_desc": "caiman_analysis", + }, skip_duplicates=True ) param_dict = { "processing_method": processing_method, @@ -901,10 +908,10 @@ def make(self, key): # -- summary images -- summary_images = { **key, - "ref_image": caiman_dataset.ref_image, - "average_image": caiman_dataset.mean_image, - "correlation_image": caiman_dataset.correlation_map, - "max_proj_image": caiman_dataset.max_proj_image, + "ref_image": caiman_dataset.ref_image.transpose(2, 0, 1), + "average_image": caiman_dataset.mean_image.transpose(2, 0, 1), + "correlation_image": caiman_dataset.correlation_map.transpose(2, 0, 1), + "max_proj_image": caiman_dataset.max_proj_image.transpose(2, 0, 1), } self.Summary.insert(summary_images) From 80845ef36f505c632e3d0c4c5c7a3e2943cae694 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 29 Jul 2024 18:17:40 -0400 Subject: [PATCH 14/39] Apply Black formatting --- element_miniscope/miniscope_no_curation.py | 23 +++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index af29238..349e5ef 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -471,7 +471,8 @@ def insert_new_params( { "processing_method": processing_method, "processing_method_desc": "caiman_analysis", - }, skip_duplicates=True + }, + skip_duplicates=True, ) param_dict = { "processing_method": processing_method, @@ -645,7 +646,7 @@ class Processing(dj.Computed): @property def key_source(self): return ProcessingTask & RecordingInfo - + def make(self, key): """ Execute the miniscope analysis defined by the ProcessingTask. @@ -673,7 +674,7 @@ def make(self, key): output_dir.mkdir(parents=True, exist_ok=True) else: raise e - + if task_mode == "load": method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": @@ -907,12 +908,12 @@ def make(self, key): # -- summary images -- summary_images = { - **key, - "ref_image": caiman_dataset.ref_image.transpose(2, 0, 1), - "average_image": caiman_dataset.mean_image.transpose(2, 0, 1), - "correlation_image": caiman_dataset.correlation_map.transpose(2, 0, 1), - "max_proj_image": caiman_dataset.max_proj_image.transpose(2, 0, 1), - } + **key, + "ref_image": caiman_dataset.ref_image.transpose(2, 0, 1), + "average_image": caiman_dataset.mean_image.transpose(2, 0, 1), + "correlation_image": caiman_dataset.correlation_map.transpose(2, 0, 1), + "max_proj_image": caiman_dataset.max_proj_image.transpose(2, 0, 1), + } self.Summary.insert(summary_images) else: @@ -1236,9 +1237,9 @@ def make(self, key): mask=mask["mask_id"], fluo_channel=segmentation_channel, activity_trace=mask[attr_mapper[key["extraction_method"]]], - ) for mask in caiman_dataset.masks + ) + for mask in caiman_dataset.masks ) - else: raise NotImplementedError("Unknown/unimplemented method: {}".format(method)) From 9afedbb0dd2c64aa714c51cbda5097721b5fed9f Mon Sep 17 00:00:00 2001 From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com> Date: Mon, 29 Jul 2024 18:30:27 -0400 Subject: [PATCH 15/39] Update element_miniscope/miniscope_no_curation.py Co-authored-by: Thinh Nguyen --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 349e5ef..ee18857 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -390,7 +390,7 @@ def make(self, key): ] ) - if acq_software == "Inscopix" or acq_software == "Miniscope-DAQ-V4": + if acq_software in ("Inscopix", "Miniscope-DAQ-V4"): self.Timestamps.insert1(dict(**key, timestamps=time_stamps)) self.Config.insert1( dict( From e5358cbd72d0d167841245fdfe7900fce3e468f0 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 29 Jul 2024 18:32:10 -0400 Subject: [PATCH 16/39] Remove updated `key_source` for `Processing` table --- element_miniscope/miniscope_no_curation.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index ee18857..3dbbb1d 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -643,10 +643,6 @@ class Processing(dj.Computed): package_version='' : varchar(16) """ - @property - def key_source(self): - return ProcessingTask & RecordingInfo - def make(self, key): """ Execute the miniscope analysis defined by the ProcessingTask. From 42c52bd4ffc0b2b83e13e146f5cedc75341b8119 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Wed, 31 Jul 2024 12:57:16 -0400 Subject: [PATCH 17/39] Update foreign key reference in `miniscope_report` --- element_miniscope/miniscope_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_report.py b/element_miniscope/miniscope_report.py index 9620f16..8376f3b 100644 --- a/element_miniscope/miniscope_report.py +++ b/element_miniscope/miniscope_report.py @@ -35,7 +35,7 @@ def activate( @schema class QualityMetrics(dj.Imported): definition = """ - -> miniscope.Curation + -> miniscope.Processing --- r_values=null : longblob # space correlation for each component snr=null : longblob # trace SNR for each component From 05b36a79748e458b91aeb51f111175c75c86fdab Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 1 Aug 2024 14:45:36 -0400 Subject: [PATCH 18/39] Fix(miniscope_no_curation.py) session directory handling in `infer_output_dir()` --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 3dbbb1d..eb77f52 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -557,7 +557,7 @@ def infer_output_dir(cls, key, relative=False, mkdir=False): acq_software = (Recording & key).fetch1("acq_software") recording_dir = find_full_path( get_miniscope_root_data_dir(), - get_session_directory(key)[0], + get_session_directory(key), ) root_dir = find_root_directory(get_miniscope_root_data_dir(), recording_dir) From 1603fb76ed12e09d0217c5c4ae079428421a2018 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 5 Aug 2024 11:37:38 -0400 Subject: [PATCH 19/39] Remove input hash generation in `Processing` --- element_miniscope/miniscope_no_curation.py | 33 ++++++---------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index eb77f52..9114add 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -702,31 +702,14 @@ def make(self, key): ProcessingTask * Recording * RecordingInfo & key ).fetch1("fps") - input_hash = dict_to_uuid(dict(**key, **params)) - input_hash_fp = output_dir / f".{input_hash }.json" - - if not input_hash_fp.exists(): - start_time = datetime.utcnow() - run_caiman( - file_paths=avi_files, - parameters=params, - sampling_rate=sampling_rate, - output_dir=output_dir.as_posix(), - is3D=False, - ) - completion_time = datetime.utcnow() - with open(input_hash_fp, "w") as f: - json.dump( - { - "start_time": start_time, - "completion_time": completion_time, - "duration": ( - completion_time - start_time - ).total_seconds(), - }, - f, - default=str, - ) + run_caiman( + file_paths=avi_files, + parameters=params, + sampling_rate=sampling_rate, + output_dir=output_dir.as_posix(), + is3D=False, + ) + _, imaging_dataset = get_loader_result(key, ProcessingTask) caiman_dataset = imaging_dataset From 0fdd66facac5e565ce005ab5a11a2b9b9d12907b Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 5 Aug 2024 16:45:17 -0400 Subject: [PATCH 20/39] Remove redundant `as_posix()` for output_dir --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 9114add..8b4e43f 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -662,7 +662,7 @@ def make(self, key): try: output_dir = find_full_path( get_miniscope_root_data_dir(), output_dir - ).as_posix() + ) except FileNotFoundError as e: if task_mode == "trigger": processed_dir = pathlib.Path(get_processed_root_data_dir()) From 52fae0f11f91439040ceccc9beaddfd3de24268e Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 9 Aug 2024 16:34:55 -0400 Subject: [PATCH 21/39] Change `Curation` to `ProcessingTask` in `get_loader_result()` --- element_miniscope/miniscope_no_curation.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 8b4e43f..1441fd6 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -949,7 +949,7 @@ class Mask(dj.Part): def make(self, key): """Populates table with segmentation data.""" - method, loaded_result = get_loader_result(key, Curation) + method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": caiman_dataset = loaded_result @@ -1099,7 +1099,7 @@ class Trace(dj.Part): def make(self, key): """Populates table with fluorescence trace data.""" - method, loaded_result = get_loader_result(key, Curation) + method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": caiman_dataset = loaded_result @@ -1189,7 +1189,7 @@ def key_source(self): def make(self, key): """Populates table with activity trace data.""" - method, loaded_result = get_loader_result(key, Curation) + method, loaded_result = get_loader_result(key, ProcessingTask) if method == "caiman": caiman_dataset = loaded_result @@ -1298,9 +1298,9 @@ def get_loader_result(key, table) -> tuple: """Retrieve the loaded processed imaging results from the loader (e.g. caiman, etc.) Args: - key (dict): the `key` to one entry of ProcessingTask or Curation. + key (dict): the `key` to one entry of ProcessingTask. table (str): the class defining the table to retrieve - the loaded results from (e.g. ProcessingTask, Curation). + the loaded results from (e.g. ProcessingTask). Returns: method, loaded_output (tuple): method string and loader object with results (e.g. caiman.CaImAn, etc.) From a7d6b9c2d01ad8521dc4e4c4685141820fc77160 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 9 Aug 2024 16:42:00 -0400 Subject: [PATCH 22/39] `insert()` -> `insert1() for MotionCorrection.Summary --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 1441fd6..883d8bc 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -893,7 +893,7 @@ def make(self, key): "correlation_image": caiman_dataset.correlation_map.transpose(2, 0, 1), "max_proj_image": caiman_dataset.max_proj_image.transpose(2, 0, 1), } - self.Summary.insert(summary_images) + self.Summary.insert1(summary_images) else: raise NotImplementedError("Unknown/unimplemented method: {}".format(method)) From 0a4ecdcdb49f2caa487f223652c8c7d4bf3a111f Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 12 Aug 2024 10:39:31 -0400 Subject: [PATCH 23/39] Fix fluo_channel -> fluorescence_channel --- element_miniscope/miniscope_no_curation.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 883d8bc..2b1701c 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -1116,7 +1116,7 @@ def make(self, key): { **key, "mask": mask["mask_id"], - "fluo_channel": segmentation_channel, + "fluorescence_channel": segmentation_channel, "fluorescence": mask["inferred_trace"], } ) @@ -1214,7 +1214,7 @@ def make(self, key): dict( key, mask=mask["mask_id"], - fluo_channel=segmentation_channel, + fluorescence_channel=segmentation_channel, activity_trace=mask[attr_mapper[key["extraction_method"]]], ) for mask in caiman_dataset.masks From 72ee46b76390ffa3cf9fc3629a77a5418eb3d06c Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Mon, 18 Nov 2024 15:54:15 -0500 Subject: [PATCH 24/39] Add external store to Processing table + minor updates --- element_miniscope/miniscope_no_curation.py | 19 +++++++++++++++++++ setup.py | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 2b1701c..133e59d 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -643,6 +643,14 @@ class Processing(dj.Computed): package_version='' : varchar(16) """ + class File(dj.Part): + definition = """ + -> master + file_name: varchar(255) # file name + --- + file: filepath@miniscope-processed + """ + def make(self, key): """ Execute the miniscope analysis defined by the ProcessingTask. @@ -724,6 +732,17 @@ def make(self, key): raise ValueError(f"Unknown task mode: {task_mode}") self.insert1(key) + self.File.insert( + [ + { + **key, + "file_name": f.relative_to(output_dir).as_posix(), + "file": f, + } + for f in output_dir.rglob("*") + if f.is_file() + ] + ) # Motion Correction -------------------------------------------------------------------- diff --git a/setup.py b/setup.py index 6db2c22..ea7d9b1 100644 --- a/setup.py +++ b/setup.py @@ -31,7 +31,7 @@ "ipywidgets", "plotly", "opencv-python", - "element-interface @ git+https://github.com/datajoint/element-interface.git@staging", + "element-interface @ git+https://github.com/datajoint/element-interface.git", ], extras_require={ "elements": [ From 7018ae040d56861ba8fef31d34f1b6988673ec1d Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 16 Jan 2025 11:56:39 -0500 Subject: [PATCH 25/39] ignore extra fields for Processing.File --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 133e59d..910c073 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -741,7 +741,7 @@ def make(self, key): } for f in output_dir.rglob("*") if f.is_file() - ] + ], ignore_extra_fields=True, ) From 5c51e4282083fbe1185e9c3b766144e8b06dcdb5 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 23 Jan 2025 17:43:25 -0500 Subject: [PATCH 26/39] feat(cell_plot.py): Summary image plotting functions --- element_miniscope/plotting/cell_plot.py | 208 ++++++++++++++++++++++++ 1 file changed, 208 insertions(+) create mode 100644 element_miniscope/plotting/cell_plot.py diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py new file mode 100644 index 0000000..f37273c --- /dev/null +++ b/element_miniscope/plotting/cell_plot.py @@ -0,0 +1,208 @@ +from typing import Tuple + +import numpy as np +import plotly.graph_objects as go +from matplotlib import colors + + +def mask_overlayed_image( + image, mask_xpix, mask_ypix, cell_mask_ids, low_q=0, high_q=0.99 +): + """Overlay transparent cell masks on average image.""" + + q_min, q_max = np.quantile(image, [low_q, high_q]) + image = np.clip(image, q_min, q_max) + image = (image - q_min) / (q_max - q_min) + + SATURATION = 0.7 + image = image[:, :, None] * np.array([0, 0, 1]) + maskid_image = np.full(image.shape[:2], -1) + for xpix, ypix, roi_id in zip(mask_xpix, mask_ypix, cell_mask_ids): + image[ypix, xpix, :2] = [np.random.rand(), SATURATION] + maskid_image[ypix, xpix] = roi_id + image = (colors.hsv_to_rgb(image) * 255).astype(int) + return image, maskid_image + + +def get_tracelayout(key, width=600, height=600) -> dict: + """Returns a dictionary of layout settings for the trace figures.""" + text = f"Trace for Cell {key['mask']}" if isinstance(key, dict) else "Trace" + + return dict( + margin=dict(l=0, r=0, b=0, t=65, pad=0), + width=width, + height=height, + transition={"duration": 0}, + title={ + "text": text, + "xanchor": "center", + "yanchor": "top", + "y": 0.97, + "x": 0.5, + }, + xaxis={ + "title": "Time (sec)", + "visible": True, + "showticklabels": True, + "showgrid": True, + }, + yaxis={ + "title": "Fluorescence (a.u.)", + "visible": True, + "showticklabels": True, + "showgrid": True, + "anchor": "free", + "overlaying": "y", + "side": "left", + "position": 0, + }, + yaxis2={ + "title": "Calcium Event (a.u.)", + "visible": True, + "showticklabels": True, + "showgrid": True, + "anchor": "free", + "overlaying": "y", + "side": "right", + "position": 1, + }, + shapes=[ + go.layout.Shape( + type="rect", + xref="paper", + yref="paper", + x0=0, + y0=0, + x1=1.0, + y1=1.0, + line={"width": 1, "color": "black"}, + ) + ], + legend={ + "traceorder": "normal", + "yanchor": "top", + "y": 0.99, + "xanchor": "right", + "x": 0.99, + }, + plot_bgcolor="rgba(0,0,0,0.05)", + modebar_remove=[ + "zoom", + "resetScale", + "pan", + "select", + "zoomIn", + "zoomOut", + "autoScale2d", + ], + ) + + +def figure_data(miniscope_module, segmentation_key) -> Tuple[np.array, np.array]: + """Prepare the images for a given segmentation_key. + + Args: + miniscope_module (dj.Table): miniscope_module table. + segmentation_key (dict): A primary key from Segmentation table. + + Returns: + background_with_cells (np.array): Average image with transparently overlayed + cells. + cells_maskid_image (np.array): Mask ID image. + """ + + image = (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1( + "average_image" + ) + + cell_mask_ids, mask_xpix, mask_ypix = ( + miniscope_module.Segmentation.Mask * miniscope_module.MaskClassification.MaskType + & segmentation_key + ).fetch("mask", "mask_xpix", "mask_ypix") + + background_with_cells, cells_maskid_image = mask_overlayed_image( + image, mask_xpix, mask_ypix, cell_mask_ids, low_q=0, high_q=0.99 + ) + + return background_with_cells, cells_maskid_image + + +def plot_cell_overlayed_image(miniscope_module, segmentation_key) -> go.Figure: + """_summary_ + + Args: + miniscope_module (dj.Table): miniscope_module table. + segmentation_key (dict): A primary key from Segmentation table. + + Returns: + image_fig (plotly.Fig): Plotly figure object of the average image with + transparently overlayed cells. + """ + + background_with_cells, cells_maskid_image = figure_data(miniscope_module, segmentation_key) + + image_fig = go.Figure( + go.Image( + z=background_with_cells, + hovertemplate="x: %{x}
y: %{y}
mask_id: %{customdata}", + customdata=cells_maskid_image, + ) + ) + image_fig.update_layout( + title="Average Image with Cells", + xaxis={ + "title": "X (px)", + "visible": True, + "showticklabels": True, + "showgrid": False, + }, + yaxis={ + "title": "Y (px)", + "visible": True, + "showticklabels": True, + "showgrid": False, + }, + paper_bgcolor="rgba(0,0,0,0)", + plot_bgcolor="rgba(0,0,0,0)", + ) + + return image_fig + + +def plot_cell_traces(miniscope_module, cell_key) -> go.Figure: + """Prepare plotly trace figure. + + Args: + miniscope_module (dj.Table): miniscope_module table. + cell_key (dict): A primary key from miniscope_module.Activity.Trace table. + + Returns: + trace_fig: Plotly figure object of the traces. + """ + activity_trace = ( + miniscope_module.Activity.Trace & "extraction_method LIKE '%deconvolution'" & cell_key + ).fetch1("activity_trace") + fluorescence, fps = (miniscope_module.RecordingInfo * miniscope_module.Fluorescence.Trace & cell_key).fetch1( + "fluorescence", "fps" + ) + + trace_fig = go.Figure( + [ + go.Scatter( + x=np.arange(len(fluorescence)) / fps, + y=fluorescence, + name="Fluorescence", + yaxis="y1", + ), + go.Scatter( + x=np.arange(len(activity_trace)) / fps, + y=activity_trace, + name="Calcium Event", + yaxis="y2", + ), + ] + ) + + trace_fig.update_layout(get_tracelayout(cell_key)) + + return trace_fig From e360d192b48b2749984dad8c26444373fe5a29db Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 23 Jan 2025 18:30:45 -0500 Subject: [PATCH 27/39] Add __init__.py --- element_miniscope/plotting/__init__.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 element_miniscope/plotting/__init__.py diff --git a/element_miniscope/plotting/__init__.py b/element_miniscope/plotting/__init__.py new file mode 100644 index 0000000..e69de29 From 4859463bb6bd447c4fbdb7d211ebcda708c1f135 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Sat, 25 Jan 2025 16:16:20 -0500 Subject: [PATCH 28/39] feat(cell_plot.py): Simplify overlayed image function --- element_miniscope/plotting/cell_plot.py | 270 ++++++++---------------- 1 file changed, 83 insertions(+), 187 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index f37273c..18c13cb 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -1,208 +1,104 @@ -from typing import Tuple - -import numpy as np +import datajoint as dj import plotly.graph_objects as go -from matplotlib import colors - - -def mask_overlayed_image( - image, mask_xpix, mask_ypix, cell_mask_ids, low_q=0, high_q=0.99 -): - """Overlay transparent cell masks on average image.""" - - q_min, q_max = np.quantile(image, [low_q, high_q]) - image = np.clip(image, q_min, q_max) - image = (image - q_min) / (q_max - q_min) - - SATURATION = 0.7 - image = image[:, :, None] * np.array([0, 0, 1]) - maskid_image = np.full(image.shape[:2], -1) - for xpix, ypix, roi_id in zip(mask_xpix, mask_ypix, cell_mask_ids): - image[ypix, xpix, :2] = [np.random.rand(), SATURATION] - maskid_image[ypix, xpix] = roi_id - image = (colors.hsv_to_rgb(image) * 255).astype(int) - return image, maskid_image +import numpy as np -def get_tracelayout(key, width=600, height=600) -> dict: - """Returns a dictionary of layout settings for the trace figures.""" - text = f"Trace for Cell {key['mask']}" if isinstance(key, dict) else "Trace" +def plot_cell_overlayed_image( + miniscope_module, segmentation_key, fig_height=600, fig_width=600 +) -> go.Figure: - return dict( - margin=dict(l=0, r=0, b=0, t=65, pad=0), - width=width, - height=height, - transition={"duration": 0}, - title={ - "text": text, - "xanchor": "center", - "yanchor": "top", - "y": 0.97, - "x": 0.5, - }, - xaxis={ - "title": "Time (sec)", - "visible": True, - "showticklabels": True, - "showgrid": True, - }, - yaxis={ - "title": "Fluorescence (a.u.)", - "visible": True, - "showticklabels": True, - "showgrid": True, - "anchor": "free", - "overlaying": "y", - "side": "left", - "position": 0, - }, - yaxis2={ - "title": "Calcium Event (a.u.)", - "visible": True, - "showticklabels": True, - "showgrid": True, - "anchor": "free", - "overlaying": "y", - "side": "right", - "position": 1, - }, - shapes=[ - go.layout.Shape( - type="rect", - xref="paper", - yref="paper", - x0=0, - y0=0, - x1=1.0, - y1=1.0, - line={"width": 1, "color": "black"}, - ) - ], - legend={ - "traceorder": "normal", - "yanchor": "top", - "y": 0.99, - "xanchor": "right", - "x": 0.99, - }, - plot_bgcolor="rgba(0,0,0,0.05)", - modebar_remove=[ - "zoom", - "resetScale", - "pan", - "select", - "zoomIn", - "zoomOut", - "autoScale2d", - ], + average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix = figure_data( + miniscope_module, segmentation_key ) + fig = go.Figure() -def figure_data(miniscope_module, segmentation_key) -> Tuple[np.array, np.array]: - """Prepare the images for a given segmentation_key. - - Args: - miniscope_module (dj.Table): miniscope_module table. - segmentation_key (dict): A primary key from Segmentation table. - - Returns: - background_with_cells (np.array): Average image with transparently overlayed - cells. - cells_maskid_image (np.array): Mask ID image. - """ - - image = (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1( - "average_image" + # Add heatmaps for the average and max projection images + fig.add_trace( + go.Heatmap( + z=average_image, + colorscale=[[0, "black"], [1, "white"]], + showscale=False, + visible=True, # Initially visible + ) ) - cell_mask_ids, mask_xpix, mask_ypix = ( - miniscope_module.Segmentation.Mask * miniscope_module.MaskClassification.MaskType - & segmentation_key - ).fetch("mask", "mask_xpix", "mask_ypix") - - background_with_cells, cells_maskid_image = mask_overlayed_image( - image, mask_xpix, mask_ypix, cell_mask_ids, low_q=0, high_q=0.99 + fig.add_trace( + go.Heatmap( + z=max_projection_image, + colorscale=[[0, "black"], [1, "white"]], + showscale=False, + visible=False, # Initially hidden + ) ) - return background_with_cells, cells_maskid_image + roi_colors = [ + f"rgb({np.random.randint(0, 256)}, {np.random.randint(0, 256)}, {np.random.randint(0, 256)})" + for _ in range(mask_ids.size) + ] - -def plot_cell_overlayed_image(miniscope_module, segmentation_key) -> go.Figure: - """_summary_ - - Args: - miniscope_module (dj.Table): miniscope_module table. - segmentation_key (dict): A primary key from Segmentation table. - - Returns: - image_fig (plotly.Fig): Plotly figure object of the average image with - transparently overlayed cells. - """ - - background_with_cells, cells_maskid_image = figure_data(miniscope_module, segmentation_key) - - image_fig = go.Figure( - go.Image( - z=background_with_cells, - hovertemplate="x: %{x}
y: %{y}
mask_id: %{customdata}", - customdata=cells_maskid_image, + # Add scatter traces for ROI contours + for xpix, ypix, color, roi_id in zip(mask_xpix, mask_ypix, roi_colors, mask_ids): + fig.add_trace( + go.Scatter( + x=xpix, + y=ypix, + mode="lines", + line=dict(color=color, width=2), + hoverinfo="text", + text=[f"ROI {roi_id}"] * len(xpix), # Display ROI ID on hover + showlegend=False, # Hide legend for ROI contours + opacity=0.7, # Adjust line transparency + ) ) - ) - image_fig.update_layout( - title="Average Image with Cells", - xaxis={ - "title": "X (px)", - "visible": True, - "showticklabels": True, - "showgrid": False, - }, - yaxis={ - "title": "Y (px)", - "visible": True, - "showticklabels": True, - "showgrid": False, - }, - paper_bgcolor="rgba(0,0,0,0)", - plot_bgcolor="rgba(0,0,0,0)", - ) - return image_fig + fig.update_layout( + title=dict( + text="Summary Image", + x=0.5, + xanchor="center", + font=dict(size=18), + ), + updatemenus=[ + { + "buttons": [ + { + "label": "Average Projection", + "method": "update", + "args": [{"visible": [True, False] + [True] * mask_ids.size}], + }, + { + "label": "Max Projection", + "method": "update", + "args": [{"visible": [False, True] + [True] * mask_ids.size}], + }, + ], + "direction": "down", + "showactive": True, + "x": 0.5, + "xanchor": "center", + "y": 1.1, + } + ], + height=fig_height, + width=fig_width, + margin=dict(t=fig_height / 6, b=40), + ) + return fig -def plot_cell_traces(miniscope_module, cell_key) -> go.Figure: - """Prepare plotly trace figure. - Args: - miniscope_module (dj.Table): miniscope_module table. - cell_key (dict): A primary key from miniscope_module.Activity.Trace table. +def figure_data(miniscope_module, segmentation_key): + miniscope = dj.create_virtual_module("miniscope", miniscope_module) - Returns: - trace_fig: Plotly figure object of the traces. - """ - activity_trace = ( - miniscope_module.Activity.Trace & "extraction_method LIKE '%deconvolution'" & cell_key - ).fetch1("activity_trace") - fluorescence, fps = (miniscope_module.RecordingInfo * miniscope_module.Fluorescence.Trace & cell_key).fetch1( - "fluorescence", "fps" + average_image = np.squeeze( + (miniscope.MotionCorrection.Summary & segmentation_key).fetch1("average_image") ) - - trace_fig = go.Figure( - [ - go.Scatter( - x=np.arange(len(fluorescence)) / fps, - y=fluorescence, - name="Fluorescence", - yaxis="y1", - ), - go.Scatter( - x=np.arange(len(activity_trace)) / fps, - y=activity_trace, - name="Calcium Event", - yaxis="y2", - ), - ] + max_projection_image = np.squeeze( + (miniscope.MotionCorrection.Summary & segmentation_key).fetch1("max_proj_image") ) + mask_ids, mask_xpix, mask_ypix = ( + miniscope.Segmentation.Mask & segmentation_key + ).fetch("mask", "mask_xpix", "mask_ypix") - trace_fig.update_layout(get_tracelayout(cell_key)) - - return trace_fig + return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix From a2d054526f20e26cc888cdb963088759d236aab9 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Sat, 25 Jan 2025 16:25:16 -0500 Subject: [PATCH 29/39] Remove virtual module creation --- element_miniscope/plotting/cell_plot.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index 18c13cb..f51359d 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -89,16 +89,15 @@ def plot_cell_overlayed_image( def figure_data(miniscope_module, segmentation_key): - miniscope = dj.create_virtual_module("miniscope", miniscope_module) average_image = np.squeeze( - (miniscope.MotionCorrection.Summary & segmentation_key).fetch1("average_image") + (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1("average_image") ) max_projection_image = np.squeeze( - (miniscope.MotionCorrection.Summary & segmentation_key).fetch1("max_proj_image") + (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1("max_proj_image") ) mask_ids, mask_xpix, mask_ypix = ( - miniscope.Segmentation.Mask & segmentation_key + miniscope_module.Segmentation.Mask & segmentation_key ).fetch("mask", "mask_xpix", "mask_ypix") return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix From 3abe10ca277243444d66e837204fa3387239a042 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Sun, 26 Jan 2025 10:54:26 -0500 Subject: [PATCH 30/39] Fix names in figure dropdown --- element_miniscope/plotting/cell_plot.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index f51359d..d939449 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -63,12 +63,12 @@ def plot_cell_overlayed_image( { "buttons": [ { - "label": "Average Projection", + "label": "Average Image", "method": "update", "args": [{"visible": [True, False] + [True] * mask_ids.size}], }, { - "label": "Max Projection", + "label": "Max Projection Image", "method": "update", "args": [{"visible": [False, True] + [True] * mask_ids.size}], }, From 9608f76cacf92c46e40a0141ee74b75f68c6695b Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 30 Jan 2025 11:09:46 -0500 Subject: [PATCH 31/39] Use HSV masks --- element_miniscope/plotting/cell_plot.py | 56 +++++++++++-------------- 1 file changed, 24 insertions(+), 32 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index d939449..e29fb9d 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -1,54 +1,46 @@ import datajoint as dj +import plotly.express as px import plotly.graph_objects as go import numpy as np +import colorsys def plot_cell_overlayed_image( - miniscope_module, segmentation_key, fig_height=600, fig_width=600 + miniscope_module, segmentation_key, fig_height=600, fig_width=600, mask_saturation=0.7, mask_color_value=1 ) -> go.Figure: average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix = figure_data( miniscope_module, segmentation_key ) - fig = go.Figure() + roi_hsv_colors = [np.random.rand() for _ in range(mask_ids.size)] - # Add heatmaps for the average and max projection images - fig.add_trace( - go.Heatmap( - z=average_image, - colorscale=[[0, "black"], [1, "white"]], - showscale=False, - visible=True, # Initially visible - ) - ) + # Convert HSV colors to RGB for Plotly + roi_rgb_colors = [] + for hue in roi_hsv_colors: + rgb = colorsys.hsv_to_rgb(hue, mask_saturation, mask_color_value) + rgb_scaled = tuple(int(c * 255) for c in rgb) + roi_rgb_colors.append(f"rgb{rgb_scaled}") # Convert to Plotly-compatible format - fig.add_trace( - go.Heatmap( - z=max_projection_image, - colorscale=[[0, "black"], [1, "white"]], - showscale=False, - visible=False, # Initially hidden - ) + fig = px.imshow( + average_image, # Initial image (can toggle later) + color_continuous_scale="gray", + labels={"color": "Intensity"}, ) - - roi_colors = [ - f"rgb({np.random.randint(0, 256)}, {np.random.randint(0, 256)}, {np.random.randint(0, 256)})" - for _ in range(mask_ids.size) - ] - - # Add scatter traces for ROI contours - for xpix, ypix, color, roi_id in zip(mask_xpix, mask_ypix, roi_colors, mask_ids): + fig.update_coloraxes(showscale=False) + # Add scatter traces for ROI contours (keeping HSV logic internally) + for xpix, ypix, color, roi_id in zip(mask_xpix, mask_ypix, roi_rgb_colors, mask_ids): fig.add_trace( go.Scatter( x=xpix, y=ypix, mode="lines", - line=dict(color=color, width=2), + line=dict(color=color, width=2), # Use HSV-based color (converted to RGB) + name=f"ROI {roi_id}", hoverinfo="text", - text=[f"ROI {roi_id}"] * len(xpix), # Display ROI ID on hover - showlegend=False, # Hide legend for ROI contours - opacity=0.7, # Adjust line transparency + text=[f"ROI {roi_id}"] * len(xpix), + showlegend=False, + opacity=0.5, ) ) @@ -65,12 +57,12 @@ def plot_cell_overlayed_image( { "label": "Average Image", "method": "update", - "args": [{"visible": [True, False] + [True] * mask_ids.size}], + "args": [{"z": [average_image]}], }, { "label": "Max Projection Image", "method": "update", - "args": [{"visible": [False, True] + [True] * mask_ids.size}], + "args": [{"z": [max_projection_image]}], }, ], "direction": "down", From 29d5bfb78e0f03e81dda7c684922e3a364b4fcaa Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Thu, 30 Jan 2025 12:00:33 -0500 Subject: [PATCH 32/39] Add defailt value for `processing_output_dir` --- element_miniscope/miniscope_no_curation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index 910c073..e492312 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -532,7 +532,7 @@ class ProcessingTask(dj.Manual): -> RecordingInfo -> ProcessingParamSet --- - processing_output_dir : varchar(255) # relative to the root data directory + processing_output_dir='': varchar(255) # relative to the root data directory task_mode='load' : enum('load', 'trigger') # 'load': load existing results # 'trigger': trigger procedure """ From bb89d67771386955fb02770cb54da09c7f670d2f Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 31 Jan 2025 12:52:45 -0500 Subject: [PATCH 33/39] Revert to using heatmap for JSON compatibility --- element_miniscope/plotting/cell_plot.py | 66 ++++++++++++++++--------- 1 file changed, 43 insertions(+), 23 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index e29fb9d..da87097 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -1,41 +1,48 @@ -import datajoint as dj -import plotly.express as px -import plotly.graph_objects as go -import numpy as np import colorsys +import numpy as np +import plotly.graph_objects as go def plot_cell_overlayed_image( - miniscope_module, segmentation_key, fig_height=600, fig_width=600, mask_saturation=0.7, mask_color_value=1 + miniscope_module, + segmentation_key, + fig_height=600, + fig_width=600, + mask_saturation=0.7, + mask_color_value=1, + **kwargs, ) -> go.Figure: + """Generate a Plotly figure with an overlayed summary image and ROI masks.""" + # Fetch data average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix = figure_data( miniscope_module, segmentation_key ) + average_image = normalize_image(average_image, **kwargs) + max_projection_image = normalize_image(max_projection_image, **kwargs) + + # Generate random HSV colors and convert to RGB roi_hsv_colors = [np.random.rand() for _ in range(mask_ids.size)] + roi_rgb_colors = [ + f"rgb{tuple(int(c * 255) for c in colorsys.hsv_to_rgb(hue, mask_saturation, mask_color_value))}" + for hue in roi_hsv_colors + ] - # Convert HSV colors to RGB for Plotly - roi_rgb_colors = [] - for hue in roi_hsv_colors: - rgb = colorsys.hsv_to_rgb(hue, mask_saturation, mask_color_value) - rgb_scaled = tuple(int(c * 255) for c in rgb) - roi_rgb_colors.append(f"rgb{rgb_scaled}") # Convert to Plotly-compatible format + fig = go.Figure() - fig = px.imshow( - average_image, # Initial image (can toggle later) - color_continuous_scale="gray", - labels={"color": "Intensity"}, - ) - fig.update_coloraxes(showscale=False) - # Add scatter traces for ROI contours (keeping HSV logic internally) - for xpix, ypix, color, roi_id in zip(mask_xpix, mask_ypix, roi_rgb_colors, mask_ids): + # Use `go.Heatmap()` instead of `go.Image()` for better JSON handling + fig.add_trace(go.Heatmap(z=average_image, colorscale="gray", showscale=False)) + + for xpix, ypix, color, roi_id in zip( + mask_xpix, mask_ypix, roi_rgb_colors, mask_ids + ): fig.add_trace( go.Scatter( x=xpix, y=ypix, mode="lines", - line=dict(color=color, width=2), # Use HSV-based color (converted to RGB) + line=dict(color=color, width=2), name=f"ROI {roi_id}", hoverinfo="text", text=[f"ROI {roi_id}"] * len(xpix), @@ -44,6 +51,7 @@ def plot_cell_overlayed_image( ) ) + # Update layout fig.update_layout( title=dict( text="Summary Image", @@ -81,15 +89,27 @@ def plot_cell_overlayed_image( def figure_data(miniscope_module, segmentation_key): - + """Fetch data for generating a figure.""" average_image = np.squeeze( - (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1("average_image") + (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1( + "average_image" + ) ) max_projection_image = np.squeeze( - (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1("max_proj_image") + (miniscope_module.MotionCorrection.Summary & segmentation_key).fetch1( + "max_proj_image" + ) ) mask_ids, mask_xpix, mask_ypix = ( miniscope_module.Segmentation.Mask & segmentation_key ).fetch("mask", "mask_xpix", "mask_ypix") return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix + + +def normalize_image(image, low_q=0, high_q=1): + """Normalize image to [0,1] based on quantile clipping.""" + q_min, q_max = np.quantile(image, [low_q, high_q]) + image = np.clip(image, q_min, q_max) + + return ((image - q_min) / (q_max - q_min) * 255).astype(np.uint8) From ae1a47f2625ddc411d093d4e706d14c118a67356 Mon Sep 17 00:00:00 2001 From: Thinh Nguyen Date: Wed, 5 Feb 2025 15:05:14 -0600 Subject: [PATCH 34/39] feat(miniscope_report): `miniscope_report` no longer auto imported/activate if not used --- element_miniscope/miniscope_no_curation.py | 3 --- element_miniscope/miniscope_report.py | 13 ++++++------- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope_no_curation.py index e492312..734a324 100644 --- a/element_miniscope/miniscope_no_curation.py +++ b/element_miniscope/miniscope_no_curation.py @@ -12,8 +12,6 @@ import pandas as pd from element_interface.utils import dict_to_uuid, find_full_path, find_root_directory -from . import miniscope_report - logger = dj.logger schema = dj.Schema() @@ -69,7 +67,6 @@ def activate( create_tables=create_tables, add_objects=_linking_module.__dict__, ) - miniscope_report.activate(f"{miniscope_schema_name}_report", miniscope_schema_name) # Functions required by the element-miniscope ----------------------------------------- diff --git a/element_miniscope/miniscope_report.py b/element_miniscope/miniscope_report.py index 8376f3b..d4ea5d4 100644 --- a/element_miniscope/miniscope_report.py +++ b/element_miniscope/miniscope_report.py @@ -1,12 +1,12 @@ import datajoint as dj -schema = dj.schema() +from . import miniscope_no_curation as miniscope -miniscope = None +schema = dj.schema() def activate( - schema_name, miniscope_schema_name, *, create_schema=True, create_tables=True + schema_name, *, create_schema=True, create_tables=True ): """Activate this schema. @@ -15,15 +15,14 @@ def activate( Args: schema_name (str): schema name on the database server to activate the `miniscope_report` schema - miniscope_schema_name (str): schema name of the activated miniscope element for - which this miniscope_report schema will be downstream from create_schema (bool): when True (default), create schema in the database if it does not yet exist. create_tables (str): when True (default), create schema takes in the database if they do not yet exist. """ - global miniscope - miniscope = dj.create_virtual_module("miniscope", miniscope_schema_name) + if not miniscope.schema.is_activated(): + raise RuntimeError("Please activate the `miniscope` schema first.") + schema.activate( schema_name, create_schema=create_schema, From 3d49aadc7e61fa627563932be0a7cc97d4fb568d Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 7 Feb 2025 09:10:36 -0500 Subject: [PATCH 35/39] Fetch mask weights for figure data --- element_miniscope/plotting/cell_plot.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index da87097..94e6ff3 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -100,11 +100,11 @@ def figure_data(miniscope_module, segmentation_key): "max_proj_image" ) ) - mask_ids, mask_xpix, mask_ypix = ( + mask_ids, mask_xpix, mask_ypix, mask_weights = ( miniscope_module.Segmentation.Mask & segmentation_key - ).fetch("mask", "mask_xpix", "mask_ypix") + ).fetch("mask", "mask_xpix", "mask_ypix", "mask_weights") - return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix + return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix, mask_weights def normalize_image(image, low_q=0, high_q=1): From 23e454a3bf79635f4fc05570775919d60845b15b Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Fri, 7 Feb 2025 09:21:45 -0500 Subject: [PATCH 36/39] bugfix in `plot_cell_overlayed_image` --- element_miniscope/plotting/cell_plot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index 94e6ff3..bc1c83e 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -15,7 +15,7 @@ def plot_cell_overlayed_image( """Generate a Plotly figure with an overlayed summary image and ROI masks.""" # Fetch data - average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix = figure_data( + average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix, _ = figure_data( miniscope_module, segmentation_key ) From 5dc893a2fd792d56a7073472a058430ebb4da4dc Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 29 Jul 2025 10:29:37 -0400 Subject: [PATCH 37/39] Revert name to miniscope from miniscope_no_curation. Add alias --- element_miniscope/__init__.py | 3 +++ element_miniscope/{miniscope_no_curation.py => miniscope.py} | 0 2 files changed, 3 insertions(+) rename element_miniscope/{miniscope_no_curation.py => miniscope.py} (100%) diff --git a/element_miniscope/__init__.py b/element_miniscope/__init__.py index e69de29..bcaca71 100644 --- a/element_miniscope/__init__.py +++ b/element_miniscope/__init__.py @@ -0,0 +1,3 @@ +from . import miniscope + +miniscope_no_curation = miniscope # Alias for backwards compatibility \ No newline at end of file diff --git a/element_miniscope/miniscope_no_curation.py b/element_miniscope/miniscope.py similarity index 100% rename from element_miniscope/miniscope_no_curation.py rename to element_miniscope/miniscope.py From 3f8314718cbd0d449579fd2e74464067d766b8b8 Mon Sep 17 00:00:00 2001 From: kushalbakshi Date: Tue, 29 Jul 2025 10:36:26 -0400 Subject: [PATCH 38/39] Apply Black formatting --- element_miniscope/__init__.py | 2 +- element_miniscope/miniscope.py | 8 +++----- element_miniscope/miniscope_report.py | 4 +--- element_miniscope/plotting/cell_plot.py | 15 +++++++++++---- 4 files changed, 16 insertions(+), 13 deletions(-) diff --git a/element_miniscope/__init__.py b/element_miniscope/__init__.py index bcaca71..01224a9 100644 --- a/element_miniscope/__init__.py +++ b/element_miniscope/__init__.py @@ -1,3 +1,3 @@ from . import miniscope -miniscope_no_curation = miniscope # Alias for backwards compatibility \ No newline at end of file +miniscope_no_curation = miniscope # Alias for backwards compatibility diff --git a/element_miniscope/miniscope.py b/element_miniscope/miniscope.py index 734a324..58292a9 100644 --- a/element_miniscope/miniscope.py +++ b/element_miniscope/miniscope.py @@ -665,9 +665,7 @@ def make(self, key): {**key, "processing_output_dir": output_dir.as_posix()} ) try: - output_dir = find_full_path( - get_miniscope_root_data_dir(), output_dir - ) + output_dir = find_full_path(get_miniscope_root_data_dir(), output_dir) except FileNotFoundError as e: if task_mode == "trigger": processed_dir = pathlib.Path(get_processed_root_data_dir()) @@ -715,7 +713,6 @@ def make(self, key): is3D=False, ) - _, imaging_dataset = get_loader_result(key, ProcessingTask) caiman_dataset = imaging_dataset key["processing_time"] = caiman_dataset.creation_time @@ -738,7 +735,8 @@ def make(self, key): } for f in output_dir.rglob("*") if f.is_file() - ], ignore_extra_fields=True, + ], + ignore_extra_fields=True, ) diff --git a/element_miniscope/miniscope_report.py b/element_miniscope/miniscope_report.py index d4ea5d4..98dff3b 100644 --- a/element_miniscope/miniscope_report.py +++ b/element_miniscope/miniscope_report.py @@ -5,9 +5,7 @@ schema = dj.schema() -def activate( - schema_name, *, create_schema=True, create_tables=True -): +def activate(schema_name, *, create_schema=True, create_tables=True): """Activate this schema. The "activation" of miniscope_report should be evoked by the miniscope module diff --git a/element_miniscope/plotting/cell_plot.py b/element_miniscope/plotting/cell_plot.py index bc1c83e..73082dc 100644 --- a/element_miniscope/plotting/cell_plot.py +++ b/element_miniscope/plotting/cell_plot.py @@ -15,8 +15,8 @@ def plot_cell_overlayed_image( """Generate a Plotly figure with an overlayed summary image and ROI masks.""" # Fetch data - average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix, _ = figure_data( - miniscope_module, segmentation_key + average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix, _ = ( + figure_data(miniscope_module, segmentation_key) ) average_image = normalize_image(average_image, **kwargs) @@ -104,12 +104,19 @@ def figure_data(miniscope_module, segmentation_key): miniscope_module.Segmentation.Mask & segmentation_key ).fetch("mask", "mask_xpix", "mask_ypix", "mask_weights") - return average_image, max_projection_image, mask_ids, mask_xpix, mask_ypix, mask_weights + return ( + average_image, + max_projection_image, + mask_ids, + mask_xpix, + mask_ypix, + mask_weights, + ) def normalize_image(image, low_q=0, high_q=1): """Normalize image to [0,1] based on quantile clipping.""" q_min, q_max = np.quantile(image, [low_q, high_q]) image = np.clip(image, q_min, q_max) - + return ((image - q_min) / (q_max - q_min) * 255).astype(np.uint8) From 7f056fa50f3e4d0f2b63fe77ff9bef7be57a4549 Mon Sep 17 00:00:00 2001 From: Kushal Bakshi <52367253+kushalbakshi@users.noreply.github.com> Date: Tue, 29 Jul 2025 11:17:20 -0400 Subject: [PATCH 39/39] Update element_miniscope/miniscope_report.py Co-authored-by: Thinh Nguyen --- element_miniscope/miniscope_report.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/element_miniscope/miniscope_report.py b/element_miniscope/miniscope_report.py index 98dff3b..cbddba7 100644 --- a/element_miniscope/miniscope_report.py +++ b/element_miniscope/miniscope_report.py @@ -1,6 +1,6 @@ import datajoint as dj -from . import miniscope_no_curation as miniscope +from . import miniscope schema = dj.schema()