diff --git a/src/murfey/server/api/workflow.py b/src/murfey/server/api/workflow.py index fcc793de..4f9744b8 100644 --- a/src/murfey/server/api/workflow.py +++ b/src/murfey/server/api/workflow.py @@ -443,6 +443,7 @@ async def request_spa_preprocessing( db.add(feedback_params) movie = Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=proc_file.path, image_number=proc_file.image_number, tag=proc_file.tag, @@ -695,6 +696,17 @@ async def request_tomography_preprocessing( 0 ].eer_fractionation_file + movie = Movie( + murfey_id=murfey_ids[0], + data_collection_id=dcid, + path=proc_file.path, + image_number=proc_file.image_number, + tag=proc_file.tag, + ) + db.add(movie) + db.commit() + db.close() + zocalo_message: dict = { "recipes": [recipe_name], "parameters": { diff --git a/src/murfey/server/feedback.py b/src/murfey/server/feedback.py index 2e77928f..c487b46f 100644 --- a/src/murfey/server/feedback.py +++ b/src/murfey/server/feedback.py @@ -1488,6 +1488,7 @@ def _flush_tomography_preprocessing(message: dict, _db): p.parent.mkdir(parents=True) movie = db.Movie( murfey_id=murfey_ids[0], + data_collection_id=detached_ids[1], path=f.file_path, image_number=f.image_number, tag=f.tag, diff --git a/src/murfey/util/db.py b/src/murfey/util/db.py index 801ac5e4..f1b0b9cf 100644 --- a/src/murfey/util/db.py +++ b/src/murfey/util/db.py @@ -4,11 +4,22 @@ """ from datetime import datetime -from typing import List, Optional +from typing import TYPE_CHECKING, List, Optional import sqlalchemy from sqlmodel import Field, Relationship, SQLModel, create_engine +if TYPE_CHECKING: + from murfey.util.processing_db import ( + CTF, + MotionCorrection, + ParticleClassificationGroup, + ParticlePicker, + RelativeIceThickness, + TiltImageAlignment, + Tomogram, + ) + """ GENERAL """ @@ -429,6 +440,14 @@ class DataCollectionGroup(SQLModel, table=True): # type: ignore sa_relationship_kwargs={"cascade": "delete"}, ) ) + grid_squares: List["GridSquare"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) + search_maps: List["SearchMap"] = Relationship( + back_populates="data_collection_group", + sa_relationship_kwargs={"cascade": "delete"}, + ) class NotificationParameter(SQLModel, table=True): # type: ignore @@ -468,6 +487,13 @@ class DataCollection(SQLModel, table=True): # type: ignore processing_jobs: List["ProcessingJob"] = Relationship( back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} ) + movies: List["Movie"] = Relationship( + back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"} + ) + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection") class ProcessingJob(SQLModel, table=True): # type: ignore @@ -569,6 +595,20 @@ class AutoProcProgram(SQLModel, table=True): # type: ignore murfey_ids: List["MurfeyLedger"] = Relationship( back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"} ) + MotionCorrection: List["MotionCorrection"] = Relationship( + back_populates="DataCollection" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram") + CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="AutoProcProgram" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="AutoProcProgram" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="AutoProcProgram" + ) class MurfeyLedger(SQLModel, table=True): # type: ignore @@ -608,6 +648,7 @@ class MurfeyLedger(SQLModel, table=True): # type: ignore class GridSquare(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: int tag: str x_location: Optional[float] @@ -619,6 +660,13 @@ class GridSquare(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + height: Optional[int] = None + width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" session: Optional[Session] = Relationship(back_populates="grid_squares") clem_image_series: List["CLEMImageSeries"] = Relationship( @@ -627,6 +675,9 @@ class GridSquare(SQLModel, table=True): # type: ignore foil_holes: List["FoilHole"] = Relationship( back_populates="grid_square", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="grid_squares" + ) class FoilHole(SQLModel, table=True): # type: ignore @@ -643,6 +694,11 @@ class FoilHole(SQLModel, table=True): # type: ignore thumbnail_size_x: Optional[int] thumbnail_size_y: Optional[int] pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + diameter: Optional[int] = None + quality_indicator: Optional[float] = None image: str = "" grid_square: Optional[GridSquare] = Relationship(back_populates="foil_holes") session: Optional[Session] = Relationship(back_populates="foil_holes") @@ -657,6 +713,7 @@ class FoilHole(SQLModel, table=True): # type: ignore class SearchMap(SQLModel, table=True): # type: ignore id: Optional[int] = Field(primary_key=True, default=None) session_id: int = Field(foreign_key="session.id") + atlas_id: Optional[int] = Field(foreign_key="datacollectiongroup.id") name: str tag: str x_location: Optional[float] = None @@ -664,6 +721,13 @@ class SearchMap(SQLModel, table=True): # type: ignore x_stage_position: Optional[float] = None y_stage_position: Optional[float] = None pixel_size: Optional[float] = None + scaled_pixel_size: Optional[float] = None + pixel_location_x: Optional[int] = None + pixel_location_y: Optional[int] = None + scaled_height: Optional[int] = None + scaled_width: Optional[int] = None + angle: Optional[float] = None + quality_indicator: Optional[float] = None image: str = "" binning: Optional[float] = None reference_matrix_m11: Optional[float] = None @@ -684,17 +748,27 @@ class SearchMap(SQLModel, table=True): # type: ignore tilt_series: List["TiltSeries"] = Relationship( back_populates="search_map", sa_relationship_kwargs={"cascade": "delete"} ) + data_collection_group: Optional["DataCollectionGroup"] = Relationship( + back_populates="search_maps" + ) + Tomogram: List["Tomogram"] = Relationship(back_populates="SearchMap") class Movie(SQLModel, table=True): # type: ignore murfey_id: int = Field(primary_key=True, foreign_key="murfeyledger.id") foil_hole_id: int = Field(foreign_key="foilhole.id", nullable=True, default=None) + data_collection_id: Optional[int] = Field(foreign_key="datacollection.id") path: str image_number: int tag: str preprocessed: bool = False murfey_ledger: Optional[MurfeyLedger] = Relationship(back_populates="movies") foil_hole: Optional[FoilHole] = Relationship(back_populates="movies") + data_collection: Optional["DataCollection"] = Relationship(back_populates="movies") + MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie") + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Movie" + ) class CtfParameters(SQLModel, table=True): # type: ignore diff --git a/src/murfey/util/processing_db.py b/src/murfey/util/processing_db.py new file mode 100644 index 00000000..37209928 --- /dev/null +++ b/src/murfey/util/processing_db.py @@ -0,0 +1,251 @@ +import datetime +from typing import TYPE_CHECKING, List, Optional + +from sqlmodel import Enum, Field, Relationship, SQLModel + +if TYPE_CHECKING: + from murfey.util.db import AutoProcProgram, DataCollection, Movie, SearchMap + + +class MotionCorrection(SQLModel, table=True): # type: ignore + motionCorrectionId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + imageNumber: Optional[int] = None + firstFrame: Optional[int] = None + lastFrame: Optional[int] = None + dosePerFrame: Optional[float] = None + doseWeight: Optional[float] = None + totalMotion: Optional[float] = None + averageMotionPerFrame: Optional[float] = None + driftPlotFullPath: Optional[str] = None + micrographFullPath: Optional[str] = None + micrographSnapshotFullPath: Optional[str] = None + patchesUsedX: Optional[int] = None + patchesUsedY: Optional[int] = None + fftFullPath: Optional[str] = None + fftCorrectedFullPath: Optional[str] = None + comments: Optional[str] = None + movieId: Optional[int] = Field(foreign_key="Movie.murfey_id") + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="MotionCorrection" + ) + DataCollection: Optional["DataCollection"] = Relationship( + back_populates="MotionCorrection" + ) + Movie: Optional["Movie"] = Relationship(back_populates="MotionCorrection") + CTF: List["CTF"] = Relationship(back_populates="MotionCorrection") + ParticlePicker: List["ParticlePicker"] = Relationship( + back_populates="MotionCorrection" + ) + RelativeIceThickness: List["RelativeIceThickness"] = Relationship( + back_populates="MotionCorrection" + ) + + +class Tomogram(SQLModel, table=True): # type: ignore + tomogramId: int = Field(primary_key=True, unique=True) + dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id") + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + volumeFile: Optional[str] = None + stackFile: Optional[str] = None + sizeX: Optional[int] = None + sizeY: Optional[int] = None + sizeZ: Optional[int] = None + pixelSpacing: Optional[float] = None + residualErrorMean: Optional[float] = None + residualErrorSD: Optional[float] = None + xAxisCorrection: Optional[float] = None + tiltAngleOffset: Optional[float] = None + zShift: Optional[float] = None + fileDirectory: Optional[str] = None + centralSliceImage: Optional[str] = None + tomogramMovie: Optional[str] = None + xyShiftPlot: Optional[str] = None + projXY: Optional[str] = None + projXZ: Optional[str] = None + recordTimeStamp: Optional[datetime.datetime] = None + globalAlignmentQuality: Optional[float] = None + gridSquareId: Optional[int] = Field(foreign_key="SearchMap.id") + pixelLocationX: Optional[int] = None + pixelLocationY: Optional[int] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="Tomogram" + ) + DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram") + SearchMap: Optional["SearchMap"] = Relationship(back_populates="Tomogram") + ProcessedTomogram: List["ProcessedTomogram"] = Relationship( + back_populates="Tomogram" + ) + TiltImageAlignment: List["TiltImageAlignment"] = Relationship( + back_populates="Tomogram" + ) + + +class CTF(SQLModel, table=True): # type: ignore + ctfId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id") + boxSizeX: Optional[float] = None + boxSizeY: Optional[float] = None + minResolution: Optional[float] = None + maxResolution: Optional[float] = None + minDefocus: Optional[float] = None + maxDefocus: Optional[float] = None + defocusStepSize: Optional[float] = None + astigmatism: Optional[float] = None + astigmatismAngle: Optional[float] = None + estimatedResolution: Optional[float] = None + estimatedDefocus: Optional[float] = None + amplitudeContrast: Optional[float] = None + ccValue: Optional[float] = None + fftTheoreticalFullPath: Optional[str] = None + comments: Optional[str] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship(back_populates="CTF") + MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF") + + +class ParticlePicker(SQLModel, table=True): # type: ignore + particlePickerId: int = Field(primary_key=True, unique=True) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + firstMotionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + particlePickingTemplate: Optional[str] = None + particleDiameter: Optional[float] = None + numberOfParticles: Optional[int] = None + summaryImageFullPath: Optional[str] = None + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="ParticlePicker" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticlePicker" + ) + ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship( + back_populates="ParticlePicker" + ) + + +class ProcessedTomogram(SQLModel, table=True): # type: ignore + processedTomogramId: int = Field(primary_key=True, unique=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId") + filePath: Optional[str] = None + processingType: Optional[str] = None + Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram") + + +class RelativeIceThickness(SQLModel, table=True): # type: ignore + relativeIceThicknessId: int = Field(primary_key=True, unique=True) + motionCorrectionId: Optional[int] = Field( + foreign_key="MotionCorrection.motionCorrectionId" + ) + autoProcProgramId: Optional[int] = Field( + foreign_key="AutoProcProgram.autoProcProgramId" + ) + minimum: Optional[float] = None + q1: Optional[float] = None + median: Optional[float] = None + q3: Optional[float] = None + maximum: Optional[float] = None + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="RelativeIceThickness" + ) + MotionCorrection: Optional["MotionCorrection"] = Relationship( + back_populates="RelativeIceThickness" + ) + + +class TiltImageAlignment(SQLModel, table=True): # type: ignore + movieId: int = Field(foreign_key="Movie.murfey_id", primary_key=True) + tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True) + defocusU: Optional[float] = None + defocusV: Optional[float] = None + psdFile: Optional[str] = None + resolution: Optional[float] = None + fitQuality: Optional[float] = None + refinedMagnification: Optional[float] = None + refinedTiltAngle: Optional[float] = None + refinedTiltAxis: Optional[float] = None + residualError: Optional[float] = None + Movie: Optional["Movie"] = Relationship(back_populates="TiltImageAlignment") + Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment") + + +class ParticleClassificationGroup(SQLModel, table=True): # type: ignore + particleClassificationGroupId: int = Field(primary_key=True, unique=True) + particlePickerId: Optional[int] = Field( + foreign_key="ParticlePicker.particlePickerId" + ) + programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId") + type: Optional[str] = Enum("2D", "3D") + batchNumber: Optional[int] = None + numberOfParticlesPerBatch: Optional[int] = None + numberOfClassesPerBatch: Optional[int] = None + symmetry: Optional[str] = None + binnedPixelSize: Optional[float] = None + ParticlePicker: Optional["ParticlePicker"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + AutoProcProgram: Optional["AutoProcProgram"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="ParticleClassificationGroup" + ) + + +class ParticleClassification(SQLModel, table=True): # type: ignore + particleClassificationId: int = Field(primary_key=True, unique=True) + classNumber: Optional[int] = None + classImageFullPath: Optional[str] = None + particlesPerClass: Optional[int] = None + rotationAccuracy: Optional[float] = None + translationAccuracy: Optional[float] = None + estimatedResolution: Optional[float] = None + overallFourierCompleteness: Optional[float] = None + particleClassificationGroupId: Optional[int] = Field( + foreign_key="ParticleClassificationGroup.particleClassificationGroupId" + ) + classDistribution: Optional[float] = None + selected: Optional[int] = None + bFactorFitIntercept: Optional[float] = None + bFactorFitLinear: Optional[float] = None + bFactorFitQuadratic: Optional[float] = None + angularEfficiency: Optional[float] = None + suggestedTilt: Optional[float] = None + CryoemInitialModel: List["CryoemInitialModel"] = Relationship( + back_populates="ParticleClassification" + ) + ParticleClassificationGroup: Optional["ParticleClassificationGroup"] = Relationship( + back_populates="ParticleClassification" + ) + BFactorFit: List["BFactorFit"] = Relationship( + back_populates="ParticleClassification" + ) + + +class BFactorFit(SQLModel, table=True): # type: ignore + bFactorFitId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + particleBatchSize: Optional[int] = None + ParticleClassification: Optional["ParticleClassification"] = Relationship( + back_populates="BFactorFit" + ) + + +class CryoemInitialModel(SQLModel, table=True): # type: ignore + cryoemInitialModelId: int = Field(primary_key=True, unique=True) + particleClassificationId: int = Field( + foreign_key="ParticleClassification.particleClassificationId" + ) + resolution: Optional[float] = None + numberOfParticles: Optional[int] = None + ParticleClassification: List["ParticleClassification"] = Relationship( + back_populates="CryoemInitialModel" + ) diff --git a/src/murfey/workflows/spa/flush_spa_preprocess.py b/src/murfey/workflows/spa/flush_spa_preprocess.py index 9f067f68..3cce2fdb 100644 --- a/src/murfey/workflows/spa/flush_spa_preprocess.py +++ b/src/murfey/workflows/spa/flush_spa_preprocess.py @@ -402,6 +402,7 @@ def flush_spa_preprocess( mrcp.parent.mkdir(parents=True) movie = Movie( murfey_id=murfey_ids[2 * i], + data_collection_id=collected_ids[1].id, path=f.file_path, image_number=f.image_number, tag=f.tag,