Register CLEM data in ISPyB (#689)

CLEM workflow changes
* Broke 'register_preprocessing_results' workflow down into smaller functions for the sake of readability
* Request for additional fluorescent-only and bright field-only composite images after data registration
* Added logic to create data collection group, atlas, and grid square table entries for CLEM workflow
* Included calculation of image dimensions (in pixels) as displayed on atlas
* Removed registration of composite image file path from 'register_align_and_merge_results' workflow

Other workflow changes
* Added new optional fields into 'GridSquareParameters' Pydantic model to stored scaled down information about the grid squares being registered
* Migrated logic for rescaling grid square values into the 'flush_spa_preprocess' workflow instead, so that the grid square registration and updating functions just register the values as-is

Database changes
* Linked 'CLEMImageSeries' to 'DataCollectionGroup' and 'GridSquare'
* Added new columns to 'CLEMImageSeries' to store glob search string for image stacks and composite images; information on data collection group, grid square, and atlas; and resolution, shape, size scale, and location information
* Removed deprecated columns from 'CLEMImageSeries' 

Tests
* Added tests for the 'register_preprocessing_results' workflow

Author: tieneupin

src/murfey/server/ispyb.py

@@ -153,7 +153,11 @@ def do_insert_atlas(self, record: Atlas):
         return {"success": False, "return_value": None}
 
     def do_update_atlas(
-        self, atlas_id: int, atlas_image: str, pixel_size: float, slot: int
+        self,
+        atlas_id: int,
+        atlas_image: str,
+        pixel_size: float,
+        slot: int | None,
     ):
         try:
             with ISPyBSession() as db:
@@ -190,34 +194,14 @@ def do_insert_grid_square(
                 grid_square_parameters.readout_area_x
                 / grid_square_parameters.thumbnail_size_x
             )
-        grid_square_parameters.height = (
-            int(grid_square_parameters.height / 7.8)
-            if grid_square_parameters.height
-            else None
-        )
-        grid_square_parameters.width = (
-            int(grid_square_parameters.width / 7.8)
-            if grid_square_parameters.width
-            else None
-        )
-        grid_square_parameters.x_location = (
-            int(grid_square_parameters.x_location / 7.8)
-            if grid_square_parameters.x_location
-            else None
-        )
-        grid_square_parameters.y_location = (
-            int(grid_square_parameters.y_location / 7.8)
-            if grid_square_parameters.y_location
-            else None
-        )
         record = GridSquare(
             atlasId=atlas_id,
             gridSquareLabel=grid_square_id,
             gridSquareImage=grid_square_parameters.image,
-            pixelLocationX=grid_square_parameters.x_location,
-            pixelLocationY=grid_square_parameters.y_location,
-            height=grid_square_parameters.height,
-            width=grid_square_parameters.width,
+            pixelLocationX=grid_square_parameters.x_location_scaled,
+            pixelLocationY=grid_square_parameters.y_location_scaled,
+            height=grid_square_parameters.height_scaled,
+            width=grid_square_parameters.width_scaled,
             angle=grid_square_parameters.angle,
             stageLocationX=grid_square_parameters.x_stage_position,
             stageLocationY=grid_square_parameters.y_stage_position,
@@ -242,7 +226,7 @@ def do_update_grid_square(
     ):
         try:
             with ISPyBSession() as db:
-                grid_square = (
+                grid_square: GridSquare = (
                     db.query(GridSquare)
                     .filter(GridSquare.gridSquareId == grid_square_id)
                     .one()
@@ -258,18 +242,18 @@ def do_update_grid_square(
                     )
                 if grid_square_parameters.image:
                     grid_square.gridSquareImage = grid_square_parameters.image
-                if grid_square_parameters.x_location:
-                    grid_square.pixelLocationX = int(
-                        grid_square_parameters.x_location / 7.8
+                if grid_square_parameters.x_location_scaled:
+                    grid_square.pixelLocationX = (
+                        grid_square_parameters.x_location_scaled
                     )
-                if grid_square_parameters.y_location:
-                    grid_square.pixelLocationY = int(
-                        grid_square_parameters.y_location / 7.8
+                if grid_square_parameters.y_location_scaled:
+                    grid_square.pixelLocationY = (
+                        grid_square_parameters.y_location_scaled
                     )
-                if grid_square_parameters.height is not None:
-                    grid_square.height = int(grid_square_parameters.height / 7.8)
-                if grid_square_parameters.width is not None:
-                    grid_square.width = int(grid_square_parameters.width / 7.8)
+                if grid_square_parameters.height_scaled is not None:
+                    grid_square.height = grid_square_parameters.height_scaled
+                if grid_square_parameters.width_scaled is not None:
+                    grid_square.width = grid_square_parameters.width_scaled
                 if grid_square_parameters.angle:
                     grid_square.angle = grid_square_parameters.angle
                 if grid_square_parameters.x_stage_position:

src/murfey/util/db.py

@@ -230,7 +230,7 @@ class CLEMImageMetadata(SQLModel, table=True):  # type: ignore
 
 class CLEMImageSeries(SQLModel, table=True):  # type: ignore
     """
-    Database recording the individual files associated with a series, which are to be
+    Database recording the files and metadata associated with a series, which are to be
     processed together as a group. These files could stem from a parent LIF file, or
     have been compiled together from individual TIFF files.
     """
@@ -239,6 +239,7 @@ class CLEMImageSeries(SQLModel, table=True):  # type: ignore
     series_name: str = Field(
         index=True
     )  # Name of the series, as determined from the metadata
+    search_string: Optional[str] = Field(default=None)  # Path for globbing with
 
     session: Optional["Session"] = Relationship(
         back_populates="image_series"
@@ -247,6 +248,22 @@ class CLEMImageSeries(SQLModel, table=True):  # type: ignore
         foreign_key="session.id", default=None, unique=False
     )
 
+    # Type of data (atlas/overview or grid square)
+    data_type: Optional[str] = Field(default=None)  # "atlas" or "grid_square"
+
+    # Link to data collection group
+    data_collection_group: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="clem_image_series"
+    )
+    dcg_id: Optional[int] = Field(foreign_key="datacollectiongroup.id", default=None)
+    dcg_name: Optional[str] = Field(default=None)
+
+    # Link to grid squares
+    grid_square: Optional["GridSquare"] = Relationship(
+        back_populates="clem_image_series"
+    )
+    grid_square_id: Optional[int] = Field(foreign_key="gridsquare.id", default=None)
+
     # The parent LIF file this series originates from, if any
     parent_lif: Optional["CLEMLIFFile"] = Relationship(
         back_populates="child_series",
@@ -270,18 +287,27 @@ class CLEMImageSeries(SQLModel, table=True):  # type: ignore
         default=None,
     )
 
-    # Databases of the image stacks that comprise this series
+    # Image stack entries that are part of this series
     child_stacks: List["CLEMImageStack"] = Relationship(
         back_populates="parent_series",
         sa_relationship_kwargs={"cascade": "delete"},
     )  # One to many
+    number_of_members: Optional[int] = Field(default=None)
 
-    # Process checklist for series
-    number_of_members: int = (
-        0  # Expected number of image stacks belonging to this series
-    )
+    # Shape and resolution information
+    pixels_x: Optional[int] = Field(default=None)
+    pixels_y: Optional[int] = Field(default=None)
+    pixel_size: Optional[float] = Field(default=None)
+    units: Optional[str] = Field(default=None)
+
+    # Extent of the imaged area in real space
+    x0: Optional[float] = Field(default=None)
+    x1: Optional[float] = Field(default=None)
+    y0: Optional[float] = Field(default=None)
+    y1: Optional[float] = Field(default=None)
+
+    # Composite images
     composite_created: bool = False  # Has a composite image been created?
-    composite_image: Optional[str] = None  # Full path to composite image
 
 
 class CLEMImageStack(SQLModel, table=True):  # type: ignore
@@ -389,6 +415,10 @@ class DataCollectionGroup(SQLModel, table=True):  # type: ignore
         back_populates="data_collection_group",
         sa_relationship_kwargs={"cascade": "delete"},
     )
+    clem_image_series: List["CLEMImageSeries"] = Relationship(
+        back_populates="data_collection_group",
+        sa_relationship_kwargs={"cascade": "delete"},
+    )
     notification_parameters: List["NotificationParameter"] = Relationship(
         back_populates="data_collection_group",
         sa_relationship_kwargs={"cascade": "delete"},
@@ -591,6 +621,9 @@ class GridSquare(SQLModel, table=True):  # type: ignore
     pixel_size: Optional[float] = None
     image: str = ""
     session: Optional[Session] = Relationship(back_populates="grid_squares")
+    clem_image_series: List["CLEMImageSeries"] = Relationship(
+        back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"}
+    )
     foil_holes: List["FoilHole"] = Relationship(
         back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"}
     )

src/murfey/util/models.py

@@ -128,15 +128,19 @@ class Base(BaseModel):
 class GridSquareParameters(BaseModel):
     tag: str
     x_location: Optional[float] = None
+    x_location_scaled: Optional[int] = None
     y_location: Optional[float] = None
+    y_location_scaled: Optional[int] = None
     x_stage_position: Optional[float] = None
     y_stage_position: Optional[float] = None
     readout_area_x: Optional[int] = None
     readout_area_y: Optional[int] = None
     thumbnail_size_x: Optional[int] = None
     thumbnail_size_y: Optional[int] = None
     height: Optional[int] = None
+    height_scaled: Optional[int] = None
     width: Optional[int] = None
+    width_scaled: Optional[int] = None
     pixel_size: Optional[float] = None
     image: str = ""
     angle: Optional[float] = None

src/murfey/workflows/clem/register_align_and_merge_results.py

@@ -87,7 +87,6 @@ def register_align_and_merge_result(
                 session_id=session_id,
                 series_name=result.series_name,
             )
-            clem_img_series.composite_image = str(result.composite_image)
             clem_img_series.composite_created = True
             murfey_db.add(clem_img_series)
             murfey_db.commit()