From 1b1cf5fe4faa6a154e912ff0f78983584f61d00c Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 1 Dec 2025 14:36:09 +0000 Subject: [PATCH 1/4] Add db components from ispyb --- src/murfey/util/db.py | 31 +++ src/murfey/util/processing_db.py | 316 +++++++++++++++++++++++++++++++ 2 files changed, 347 insertions(+) create mode 100644 src/murfey/util/processing_db.py diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 801ac5e4..d95f5bc3 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -9,6 +9,17 @@ import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine +from murfey.util.processing_db import ( + CTF, + Atlas, + MotionCorrection, + Movie as ProcessingMovie, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + Tomogram, +) + """ GENERAL """ @@ -429,6 +440,7 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) + Atlas: List["Atlas"] = Relationship(back_populates="DataCollectionGroup") class NotificationParameter(SQLModel, table=True): # type: ignore @@ -468,6 +480,11 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) + Movie: List["ProcessingMovie"] = Relationship(back_populates="DataCollection") + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") class ProcessingJob(SQLModel, table=True): # type: ignore @@ -569,6 +586,20 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") + CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="AutoProcProgram" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="AutoProcProgram" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="AutoProcProgram" + ) class MurfeyLedger(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py new file mode 100644 index 00000000..0c9841db --- /dev/null +++ b/src/murfey/util/processing_db.py @@ -0,0 +1,316 @@ +import datetime +from typing import List, Optional + +from sqlmodel import Enum, Field, Relationship, SQLModel + +from murfey.util.db import AutoProcProgram, DataCollection, DataCollectionGroup + + +class Atlas(SQLModel, table=True): # type: ignore + atlasId: int = Field(primary_key=True, unique=True) + dataCollectionGroupId: int = Field(foreign_key="DataCollectionGroup.id") + atlasImage: str + pixelSize: float + cassetteSlot: Optional[int] = None + DataCollectionGroup: Optional["DataCollectionGroup"] = Relationship( + back_populates="Atlas" + ) + GridSquare: List["GridSquare"] = Relationship(back_populates="Atlas") + + +class GridSquare(SQLModel, table=True): # type: ignore + gridSquareId: int = Field(primary_key=True, unique=True) + atlasId: int = Field(foreign_key="Atlas.atlasId") + gridSquareLabel: Optional[int] = None + gridSquareImage: Optional[str] = None + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + stageLocationX: Optional[float] = None + stageLocationY: Optional[float] = None + qualityIndicator: Optional[float] = None + pixelSize: Optional[float] = None + Atlas: Optional["Atlas"] = Relationship(back_populates="Atlas") + FoilHole: List["FoilHole"] = Relationship(back_populates="GridSquare") + Tomogram: List["Tomogram"] = Relationship(back_populates="GridSquare") + + +class FoilHole(SQLModel, table=True): # type: ignore + foilHoleId: int = Field(primary_key=True, unique=True) + gridSquareId: int = Field(foreign_key="GridSquare.gridSquareId") + foilHoleLabel: str + foilHoleImage: Optional[str] = None + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + diameter: Optional[int] = None + stageLocationX: Optional[float] = None + stageLocationY: Optional[float] = None + qualityIndicator: Optional[float] = None + pixelSize: Optional[float] = None + GridSquare: Optional["GridSquare"] = Relationship(back_populates="FoilHole") + Movie: List["Movie"] = Relationship(back_populates="FoilHole") + + +class Movie(SQLModel, table=True): # type: ignore + movieId: int = Field(primary_key=True, unique=True) + createdTimeStamp: datetime.datetime + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + movieNumber: Optional[int] = None + movieFullPath: Optional[str] = None + positionX: Optional[float] = None + positionY: Optional[float] = None + nominalDefocus: Optional[float] = None + angle: Optional[float] = None + fluence: Optional[float] = None + numberOfFrames: Optional[int] = None + foilHoleId: Optional[int] = Field(foreign_key="FoilHole.foilHoleId") + templateLabel: Optional[int] = None + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Movie") + FoilHole: Optional["FoilHole"] = Relationship(back_populates="Movie") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) + + +class MotionCorrection(SQLModel, table=True): # type: ignore + motionCorrectionId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + imageNumber: Optional[int] = None + firstFrame: Optional[int] = None + lastFrame: Optional[int] = None + dosePerFrame: Optional[float] = None + doseWeight: Optional[float] = None + totalMotion: Optional[float] = None + averageMotionPerFrame: Optional[float] = None + driftPlotFullPath: Optional[str] = None + micrographFullPath: Optional[str] = None + micrographSnapshotFullPath: Optional[str] = None + patchesUsedX: Optional[int] = None + patchesUsedY: Optional[int] = None + fftFullPath: Optional[str] = None + fftCorrectedFullPath: Optional[str] = None + comments: Optional[str] = None + movieId: Optional[int] = Field(foreign_key="Movie.movieId") + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="MotionCorrection" + ) + DataCollection: Optional["DataCollection"] = Relationship( + back_populates="MotionCorrection" + ) + Movie: Optional["Movie"] = Relationship(back_populates="MotionCorrection") + CTF: List["CTF"] = Relationship(back_populates="MotionCorrection") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="MotionCorrection" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="MotionCorrection" + ) + + +class Tomogram(SQLModel, table=True): # type: ignore + tomogramId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + volumeFile: Optional[str] = None + stackFile: Optional[str] = None + sizeX: Optional[int] = None + sizeY: Optional[int] = None + sizeZ: Optional[int] = None + pixelSpacing: Optional[float] = None + residualErrorMean: Optional[float] = None + residualErrorSD: Optional[float] = None + xAxisCorrection: Optional[float] = None + tiltAngleOffset: Optional[float] = None + zShift: Optional[float] = None + fileDirectory: Optional[str] = None + centralSliceImage: Optional[str] = None + tomogramMovie: Optional[str] = None + xyShiftPlot: Optional[str] = None + projXY: Optional[str] = None + projXZ: Optional[str] = None + recordTimeStamp: Optional[datetime.datetime] = None + globalAlignmentQuality: Optional[float] = None + gridSquareId: Optional[int] = Field(foreign_key="GridSquare.gridSquareId") + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="Tomogram" + ) + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") + GridSquare: Optional["GridSquare"] = Relationship(back_populates="Tomogram") + ProcessedTomogram: List["ProcessedTomogram"] = Relationship( + back_populates="Tomogram" + ) + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Tomogram" + ) + + +class CTF(SQLModel, table=True): # type: ignore + ctfId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + boxSizeX: Optional[float] = None + boxSizeY: Optional[float] = None + minResolution: Optional[float] = None + maxResolution: Optional[float] = None + minDefocus: Optional[float] = None + maxDefocus: Optional[float] = None + defocusStepSize: Optional[float] = None + astigmatism: Optional[float] = None + astigmatismAngle: Optional[float] = None + estimatedResolution: Optional[float] = None + estimatedDefocus: Optional[float] = None + amplitudeContrast: Optional[float] = None + ccValue: Optional[float] = None + fftTheoreticalFullPath: Optional[str] = None + comments: Optional[str] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship(back_populates="CTF") + MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF") + + +class ParticlePicker(SQLModel, table=True): # type: ignore + particlePickerId: int = Field(primary_key=True, unique=True) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + firstMotionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + particlePickingTemplate: Optional[str] = None + particleDiameter: Optional[float] = None + numberOfParticles: Optional[int] = None + summaryImageFullPath: Optional[str] = None + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="ParticlePicker" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticlePicker" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="ParticlePicker" + ) + + +class ProcessedTomogram(SQLModel, table=True): # type: ignore + processedTomogramId: int = Field(primary_key=True, unique=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId") + filePath: Optional[str] = None + processingType: Optional[str] = None + Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram") + + +class RelativeIceThickness(SQLModel, table=True): # type: ignore + relativeIceThicknessId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field( + foreign_key="AutoProcProgram.autoProcProgramId" + ) + minimum: Optional[float] = None + q1: Optional[float] = None + median: Optional[float] = None + q3: Optional[float] = None + maximum: Optional[float] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="RelativeIceThickness" + ) + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="RelativeIceThickness" + ) + + +class TiltImageAlignment(SQLModel, table=True): # type: ignore + movieId: int = Field(foreign_key="Movie.movieId", primary_key=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) + defocusU: Optional[float] = None + defocusV: Optional[float] = None + psdFile: Optional[str] = None + resolution: Optional[float] = None + fitQuality: Optional[float] = None + refinedMagnification: Optional[float] = None + refinedTiltAngle: Optional[float] = None + refinedTiltAxis: Optional[float] = None + residualError: Optional[float] = None + Movie: Optional["Movie"] = Relationship(back_populates="TiltImageAlignment") + Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment") + + +class ParticleClassificationGroup(SQLModel, table=True): # type: ignore + particleClassificationGroupId: int = Field(primary_key=True, unique=True) + particlePickerId: Optional[int] = Field( + foreign_key="ParticlePicker.particlePickerId" + ) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + type: Optional[str] = Enum("2D", "3D") + batchNumber: Optional[int] = None + numberOfParticlesPerBatch: Optional[int] = None + numberOfClassesPerBatch: Optional[int] = None + symmetry: Optional[str] = None + binnedPixelSize: Optional[float] = None + ParticlePicker: Optional["ParticlePicker"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + + +class ParticleClassification(SQLModel, table=True): # type: ignore + particleClassificationId: int = Field(primary_key=True, unique=True) + classNumber: Optional[int] = None + classImageFullPath: Optional[str] = None + particlesPerClass: Optional[int] = None + rotationAccuracy: Optional[float] = None + translationAccuracy: Optional[float] = None + estimatedResolution: Optional[float] = None + overallFourierCompleteness: Optional[float] = None + particleClassificationGroupId: Optional[int] = Field( + foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + ) + classDistribution: Optional[float] = None + selected: Optional[int] = None + bFactorFitIntercept: Optional[float] = None + bFactorFitLinear: Optional[float] = None + bFactorFitQuadratic: Optional[float] = None + angularEfficiency: Optional[float] = None + suggestedTilt: Optional[float] = None + CryoemInitialModel: List["CryoemInitialModel"] = Relationship( + back_populates="ParticleClassification" + ) + ParticleClassificationGroup: Optional["ParticleClassificationGroup"] = Relationship( + back_populates="ParticleClassification" + ) + BFactorFit: List["BFactorFit"] = Relationship( + back_populates="ParticleClassification" + ) + + +class BFactorFit(SQLModel, table=True): # type: ignore + bFactorFitId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + particleBatchSize: Optional[int] = None + ParticleClassification: Optional["ParticleClassification"] = Relationship( + back_populates="BFactorFit" + ) + + +class CryoemInitialModel(SQLModel, table=True): # type: ignore + cryoemInitialModelId: int = Field(primary_key=True, unique=True) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="CryoemInitialModel" + ) From ce045b685a22530fcbe779a6819a546d6a8b486e Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Mon, 1 Dec 2025 16:54:14 +0000 Subject: [PATCH 2/4] Merge together all similar tables --- src/murfey/util/db.py | 52 +++++++++++++++++++-- src/murfey/util/processing_db.py | 78 +++----------------------------- 2 files changed, 54 insertions(+), 76 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index d95f5bc3..ea75e63b 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -11,12 +11,11 @@ from murfey.util.processing_db import ( CTF, - Atlas, MotionCorrection, - Movie as ProcessingMovie, ParticleClassificationGroup, ParticlePicker, RelativeIceThickness, + TiltImageAlignment, Tomogram, ) @@ -440,7 +439,14 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) - Atlas: List["Atlas"] = Relationship(back_populates="DataCollectionGroup") + grid_squares: List["GridSquare"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + search_maps: List["SearchMap"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) class NotificationParameter(SQLModel, table=True): # type: ignore @@ -480,7 +486,9 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) - Movie: List["ProcessingMovie"] = Relationship(back_populates="DataCollection") + movies: List["Movie"] = Relationship( + back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} + ) MotionCorrection: List["MotionCorrection"] = Relationship( back_populates="DataCollection" ) @@ -639,6 +647,7 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") name: int tag: str x_location: Optional[float] @@ -650,6 +659,13 @@ class GridSquare(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" session: Optional[Session] = Relationship(back_populates="grid_squares") clem_image_series: List["CLEMImageSeries"] = Relationship( @@ -658,6 +674,9 @@ class GridSquare(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="grid_squares" + ) class FoilHole(SQLModel, table=True): # type: ignore @@ -674,6 +693,11 @@ class FoilHole(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + diameter: Optional[int] = None + quality_indicator: Optional[float] = None image: str = "" grid_square: Optional[GridSquare] = Relationship(back_populates="foil_holes") session: Optional[Session] = Relationship(back_populates="foil_holes") @@ -688,6 +712,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlasId: Optional[int] = Field(foreign_key="data_collection_group.id") name: str tag: str x_location: Optional[float] = None @@ -695,6 +720,13 @@ class SearchMap(SQLModel, table=True): # type: ignore x_stage_position: Optional[float] = None y_stage_position: Optional[float] = None pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + scaled_height: Optional[int] = None + scaled_width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" binning: Optional[float] = None reference_matrix_m11: Optional[float] = None @@ -715,17 +747,29 @@ class SearchMap(SQLModel, table=True): # type: ignore tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="search_maps" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) + data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") path: str image_number: int tag: str preprocessed: bool = False + createdTimeStamp: Optional[datetime] = None + movie_full_path: Optional[str] = None murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") + data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) class CtfParameters(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 0c9841db..89152d50 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -3,76 +3,7 @@ from sqlmodel import Enum, Field, Relationship, SQLModel -from murfey.util.db import AutoProcProgram, DataCollection, DataCollectionGroup - - -class Atlas(SQLModel, table=True): # type: ignore - atlasId: int = Field(primary_key=True, unique=True) - dataCollectionGroupId: int = Field(foreign_key="DataCollectionGroup.id") - atlasImage: str - pixelSize: float - cassetteSlot: Optional[int] = None - DataCollectionGroup: Optional["DataCollectionGroup"] = Relationship( - back_populates="Atlas" - ) - GridSquare: List["GridSquare"] = Relationship(back_populates="Atlas") - - -class GridSquare(SQLModel, table=True): # type: ignore - gridSquareId: int = Field(primary_key=True, unique=True) - atlasId: int = Field(foreign_key="Atlas.atlasId") - gridSquareLabel: Optional[int] = None - gridSquareImage: Optional[str] = None - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - height: Optional[int] = None - width: Optional[int] = None - angle: Optional[float] = None - stageLocationX: Optional[float] = None - stageLocationY: Optional[float] = None - qualityIndicator: Optional[float] = None - pixelSize: Optional[float] = None - Atlas: Optional["Atlas"] = Relationship(back_populates="Atlas") - FoilHole: List["FoilHole"] = Relationship(back_populates="GridSquare") - Tomogram: List["Tomogram"] = Relationship(back_populates="GridSquare") - - -class FoilHole(SQLModel, table=True): # type: ignore - foilHoleId: int = Field(primary_key=True, unique=True) - gridSquareId: int = Field(foreign_key="GridSquare.gridSquareId") - foilHoleLabel: str - foilHoleImage: Optional[str] = None - pixelLocationX: Optional[int] = None - pixelLocationY: Optional[int] = None - diameter: Optional[int] = None - stageLocationX: Optional[float] = None - stageLocationY: Optional[float] = None - qualityIndicator: Optional[float] = None - pixelSize: Optional[float] = None - GridSquare: Optional["GridSquare"] = Relationship(back_populates="FoilHole") - Movie: List["Movie"] = Relationship(back_populates="FoilHole") - - -class Movie(SQLModel, table=True): # type: ignore - movieId: int = Field(primary_key=True, unique=True) - createdTimeStamp: datetime.datetime - dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") - movieNumber: Optional[int] = None - movieFullPath: Optional[str] = None - positionX: Optional[float] = None - positionY: Optional[float] = None - nominalDefocus: Optional[float] = None - angle: Optional[float] = None - fluence: Optional[float] = None - numberOfFrames: Optional[int] = None - foilHoleId: Optional[int] = Field(foreign_key="FoilHole.foilHoleId") - templateLabel: Optional[int] = None - DataCollection: Optional["DataCollection"] = Relationship(back_populates="Movie") - FoilHole: Optional["FoilHole"] = Relationship(back_populates="Movie") - MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") - TiltImageAlignment: List["TiltImageAlignment"] = Relationship( - back_populates="Movie" - ) +from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap class MotionCorrection(SQLModel, table=True): # type: ignore @@ -134,14 +65,14 @@ class Tomogram(SQLModel, table=True): # type: ignore projXZ: Optional[str] = None recordTimeStamp: Optional[datetime.datetime] = None globalAlignmentQuality: Optional[float] = None - gridSquareId: Optional[int] = Field(foreign_key="GridSquare.gridSquareId") + gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") pixelLocationX: Optional[int] = None pixelLocationY: Optional[int] = None AutoProcProgram: Optional["AutoProcProgram"] = Relationship( back_populates="Tomogram" ) DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") - GridSquare: Optional["GridSquare"] = Relationship(back_populates="Tomogram") + SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") ProcessedTomogram: List["ProcessedTomogram"] = Relationship( back_populates="Tomogram" ) @@ -309,6 +240,9 @@ class BFactorFit(SQLModel, table=True): # type: ignore class CryoemInitialModel(SQLModel, table=True): # type: ignore cryoemInitialModelId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) resolution: Optional[float] = None numberOfParticles: Optional[int] = None ParticleClassification: List["ParticleClassification"] = Relationship( From e966b2a5a0134a893f816de548b6ee6d65142b10 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 2 Dec 2025 12:29:12 +0000 Subject: [PATCH 3/4] Register movies on the murfey side --- src/murfey/server/api/workflow.py | 12 +++++++++ src/murfey/server/feedback.py | 1 + src/murfey/util/db.py | 25 +++++++++---------- src/murfey/util/processing_db.py | 7 +++--- .../workflows/spa/flush_spa_preprocess.py | 1 + 5 files changed, 30 insertions(+), 16 deletions(-) diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index fcc793de..4f9744b8 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -443,6 +443,7 @@ async def request_spa_preprocessing( db.add(feedback_params) movie = Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=proc_file.path, image_number=proc_file.image_number, tag=proc_file.tag, @@ -695,6 +696,17 @@ async def request_tomography_preprocessing( 0 ].eer_fractionation_file + movie = Movie( + murfey_id=murfey_ids[0], + data_collection_id=dcid, + path=proc_file.path, + image_number=proc_file.image_number, + tag=proc_file.tag, + ) + db.add(movie) + db.commit() + db.close() + zocalo_message: dict = { "recipes": [recipe_name], "parameters": { diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 2e77928f..c487b46f 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -1488,6 +1488,7 @@ def _flush_tomography_preprocessing(message: dict, _db): p.parent.mkdir(parents=True) movie = db.Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=f.file_path, image_number=f.image_number, tag=f.tag, diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index ea75e63b..3bc4a289 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,20 +4,21 @@ """ from datetime import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine -from murfey.util.processing_db import ( - CTF, - MotionCorrection, - ParticleClassificationGroup, - ParticlePicker, - RelativeIceThickness, - TiltImageAlignment, - Tomogram, -) +if TYPE_CHECKING: + from murfey.util.processing_db import ( + CTF, + MotionCorrection, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + TiltImageAlignment, + Tomogram, + ) """ GENERAL @@ -712,7 +713,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlasId: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") name: str tag: str x_location: Optional[float] = None @@ -761,8 +762,6 @@ class Movie(SQLModel, table=True): # type: ignore image_number: int tag: str preprocessed: bool = False - createdTimeStamp: Optional[datetime] = None - movie_full_path: Optional[str] = None murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 89152d50..5a848eb3 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -1,9 +1,10 @@ import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional from sqlmodel import Enum, Field, Relationship, SQLModel -from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap +if TYPE_CHECKING: + from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap class MotionCorrection(SQLModel, table=True): # type: ignore @@ -25,7 +26,7 @@ class MotionCorrection(SQLModel, table=True): # type: ignore fftFullPath: Optional[str] = None fftCorrectedFullPath: Optional[str] = None comments: Optional[str] = None - movieId: Optional[int] = Field(foreign_key="Movie.movieId") + movieId: Optional[int] = Field(foreign_key="Movie.murfey_id") AutoProcProgram: Optional["AutoProcProgram"] = Relationship( back_populates="MotionCorrection" ) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 9f067f68..3cce2fdb 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -402,6 +402,7 @@ def flush_spa_preprocess( mrcp.parent.mkdir(parents=True) movie = Movie( murfey_id=murfey_ids[2 * i], + data_collection_id=collected_ids[1].id, path=f.file_path, image_number=f.image_number, tag=f.tag, From 7dad07ebe51904ff1ded2e89e8985bd95f506360 Mon Sep 17 00:00:00 2001 From: yxd92326 Date: Tue, 2 Dec 2025 13:18:52 +0000 Subject: [PATCH 4/4] Wrong naming --- src/murfey/util/db.py | 4 ++-- src/murfey/util/processing_db.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 3bc4a289..f1b0b9cf 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -648,7 +648,7 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: int tag: str x_location: Optional[float] @@ -713,7 +713,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") - atlas_id: Optional[int] = Field(foreign_key="data_collection_group.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: str tag: str x_location: Optional[float] = None diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py index 5a848eb3..37209928 100644 --- a/src/murfey/util/processing_db.py +++ b/src/murfey/util/processing_db.py @@ -158,7 +158,7 @@ class RelativeIceThickness(SQLModel, table=True): # type: ignore class TiltImageAlignment(SQLModel, table=True): # type: ignore - movieId: int = Field(foreign_key="Movie.movieId", primary_key=True) + movieId: int = Field(foreign_key="Movie.murfey_id", primary_key=True) tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) defocusU: Optional[float] = None defocusV: Optional[float] = None