Add db components from ispyb for processing outputs

src/murfey/util/db.py

@@ -9,6 +9,17 @@
 import sqlalchemy
 from sqlmodel import Field, Relationship, SQLModel, create_engine
 
+from murfey.util.processing_db import (
+    CTF,
+    Atlas,
+    MotionCorrection,
+    Movie as ProcessingMovie,
+    ParticleClassificationGroup,
+    ParticlePicker,
+    RelativeIceThickness,
+    Tomogram,
+)
+
 """
 GENERAL
 """
@@ -429,6 +440,7 @@
             sa_relationship_kwargs={"cascade": "delete"},
         )
     )
+    Atlas: List["Atlas"] = Relationship(back_populates="DataCollectionGroup")
 
 
 class NotificationParameter(SQLModel, table=True):  # type: ignore
@@ -468,6 +480,11 @@
     processing_jobs: List["ProcessingJob"] = Relationship(
         back_populates="data_collection", sa_relationship_kwargs={"cascade": "delete"}
     )
+    Movie: List["ProcessingMovie"] = Relationship(back_populates="DataCollection")
+    MotionCorrection: List["MotionCorrection"] = Relationship(
+        back_populates="DataCollection"
+    )
+    Tomogram: List["Tomogram"] = Relationship(back_populates="DataCollection")
 
 
 class ProcessingJob(SQLModel, table=True):  # type: ignore
@@ -569,6 +586,20 @@
     murfey_ids: List["MurfeyLedger"] = Relationship(
         back_populates="auto_proc_program", sa_relationship_kwargs={"cascade": "delete"}
     )
+    MotionCorrection: List["MotionCorrection"] = Relationship(
+        back_populates="DataCollection"
+    )
+    Tomogram: List["Tomogram"] = Relationship(back_populates="AutoProcProgram")
+    CTF: List["CTF"] = Relationship(back_populates="AutoProcProgram")
+    ParticlePicker: List["ParticlePicker"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
+    RelativeIceThickness: List["RelativeIceThickness"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
+    ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship(
+        back_populates="AutoProcProgram"
+    )
 
 
 class MurfeyLedger(SQLModel, table=True):  # type: ignore

src/murfey/util/processing_db.py

@@ -0,0 +1,316 @@
+import datetime
+from typing import List, Optional
+
+from sqlmodel import Enum, Field, Relationship, SQLModel
+
+from murfey.util.db import AutoProcProgram, DataCollection, DataCollectionGroup
+
+
+class Atlas(SQLModel, table=True):  # type: ignore
+    atlasId: int = Field(primary_key=True, unique=True)
+    dataCollectionGroupId: int = Field(foreign_key="DataCollectionGroup.id")
+    atlasImage: str
+    pixelSize: float
+    cassetteSlot: Optional[int] = None
+    DataCollectionGroup: Optional["DataCollectionGroup"] = Relationship(
+        back_populates="Atlas"
+    )
+    GridSquare: List["GridSquare"] = Relationship(back_populates="Atlas")
+
+
+class GridSquare(SQLModel, table=True):  # type: ignore
+    gridSquareId: int = Field(primary_key=True, unique=True)
+    atlasId: int = Field(foreign_key="Atlas.atlasId")
+    gridSquareLabel: Optional[int] = None
+    gridSquareImage: Optional[str] = None
+    pixelLocationX: Optional[int] = None
+    pixelLocationY: Optional[int] = None
+    height: Optional[int] = None
+    width: Optional[int] = None
+    angle: Optional[float] = None
+    stageLocationX: Optional[float] = None
+    stageLocationY: Optional[float] = None
+    qualityIndicator: Optional[float] = None
+    pixelSize: Optional[float] = None
+    Atlas: Optional["Atlas"] = Relationship(back_populates="Atlas")
+    FoilHole: List["FoilHole"] = Relationship(back_populates="GridSquare")
+    Tomogram: List["Tomogram"] = Relationship(back_populates="GridSquare")
+
+
+class FoilHole(SQLModel, table=True):  # type: ignore
+    foilHoleId: int = Field(primary_key=True, unique=True)
+    gridSquareId: int = Field(foreign_key="GridSquare.gridSquareId")
+    foilHoleLabel: str
+    foilHoleImage: Optional[str] = None
+    pixelLocationX: Optional[int] = None
+    pixelLocationY: Optional[int] = None
+    diameter: Optional[int] = None
+    stageLocationX: Optional[float] = None
+    stageLocationY: Optional[float] = None
+    qualityIndicator: Optional[float] = None
+    pixelSize: Optional[float] = None
+    GridSquare: Optional["GridSquare"] = Relationship(back_populates="FoilHole")
+    Movie: List["Movie"] = Relationship(back_populates="FoilHole")
+
+
+class Movie(SQLModel, table=True):  # type: ignore
+    movieId: int = Field(primary_key=True, unique=True)
+    createdTimeStamp: datetime.datetime
+    dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id")
+    movieNumber: Optional[int] = None
+    movieFullPath: Optional[str] = None
+    positionX: Optional[float] = None
+    positionY: Optional[float] = None
+    nominalDefocus: Optional[float] = None
+    angle: Optional[float] = None
+    fluence: Optional[float] = None
+    numberOfFrames: Optional[int] = None
+    foilHoleId: Optional[int] = Field(foreign_key="FoilHole.foilHoleId")
+    templateLabel: Optional[int] = None
+    DataCollection: Optional["DataCollection"] = Relationship(back_populates="Movie")
+    FoilHole: Optional["FoilHole"] = Relationship(back_populates="Movie")
+    MotionCorrection: List["MotionCorrection"] = Relationship(back_populates="Movie")
+    TiltImageAlignment: List["TiltImageAlignment"] = Relationship(
+        back_populates="Movie"
+    )
+
+
+class MotionCorrection(SQLModel, table=True):  # type: ignore
+    motionCorrectionId: int = Field(primary_key=True, unique=True)
+    dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id")
+    autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id")
+    imageNumber: Optional[int] = None
+    firstFrame: Optional[int] = None
+    lastFrame: Optional[int] = None
+    dosePerFrame: Optional[float] = None
+    doseWeight: Optional[float] = None
+    totalMotion: Optional[float] = None
+    averageMotionPerFrame: Optional[float] = None
+    driftPlotFullPath: Optional[str] = None
+    micrographFullPath: Optional[str] = None
+    micrographSnapshotFullPath: Optional[str] = None
+    patchesUsedX: Optional[int] = None
+    patchesUsedY: Optional[int] = None
+    fftFullPath: Optional[str] = None
+    fftCorrectedFullPath: Optional[str] = None
+    comments: Optional[str] = None
+    movieId: Optional[int] = Field(foreign_key="Movie.movieId")
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(
+        back_populates="MotionCorrection"
+    )
+    DataCollection: Optional["DataCollection"] = Relationship(
+        back_populates="MotionCorrection"
+    )
+    Movie: Optional["Movie"] = Relationship(back_populates="MotionCorrection")
+    CTF: List["CTF"] = Relationship(back_populates="MotionCorrection")
+    ParticlePicker: List["ParticlePicker"] = Relationship(
+        back_populates="MotionCorrection"
+    )
+    RelativeIceThickness: List["RelativeIceThickness"] = Relationship(
+        back_populates="MotionCorrection"
+    )
+
+
+class Tomogram(SQLModel, table=True):  # type: ignore
+    tomogramId: int = Field(primary_key=True, unique=True)
+    dataCollectionId: Optional[int] = Field(foreign_key="DataCollection.id")
+    autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id")
+    volumeFile: Optional[str] = None
+    stackFile: Optional[str] = None
+    sizeX: Optional[int] = None
+    sizeY: Optional[int] = None
+    sizeZ: Optional[int] = None
+    pixelSpacing: Optional[float] = None
+    residualErrorMean: Optional[float] = None
+    residualErrorSD: Optional[float] = None
+    xAxisCorrection: Optional[float] = None
+    tiltAngleOffset: Optional[float] = None
+    zShift: Optional[float] = None
+    fileDirectory: Optional[str] = None
+    centralSliceImage: Optional[str] = None
+    tomogramMovie: Optional[str] = None
+    xyShiftPlot: Optional[str] = None
+    projXY: Optional[str] = None
+    projXZ: Optional[str] = None
+    recordTimeStamp: Optional[datetime.datetime] = None
+    globalAlignmentQuality: Optional[float] = None
+    gridSquareId: Optional[int] = Field(foreign_key="GridSquare.gridSquareId")
+    pixelLocationX: Optional[int] = None
+    pixelLocationY: Optional[int] = None
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(
+        back_populates="Tomogram"
+    )
+    DataCollection: Optional["DataCollection"] = Relationship(back_populates="Tomogram")
+    GridSquare: Optional["GridSquare"] = Relationship(back_populates="Tomogram")
+    ProcessedTomogram: List["ProcessedTomogram"] = Relationship(
+        back_populates="Tomogram"
+    )
+    TiltImageAlignment: List["TiltImageAlignment"] = Relationship(
+        back_populates="Tomogram"
+    )
+
+
+class CTF(SQLModel, table=True):  # type: ignore
+    ctfId: int = Field(primary_key=True, unique=True)
+    motionCorrectionId: Optional[int] = Field(
+        foreign_key="MotionCorrection.motionCorrectionId"
+    )
+    autoProcProgramId: Optional[int] = Field(foreign_key="AutoProcProgram.id")
+    boxSizeX: Optional[float] = None
+    boxSizeY: Optional[float] = None
+    minResolution: Optional[float] = None
+    maxResolution: Optional[float] = None
+    minDefocus: Optional[float] = None
+    maxDefocus: Optional[float] = None
+    defocusStepSize: Optional[float] = None
+    astigmatism: Optional[float] = None
+    astigmatismAngle: Optional[float] = None
+    estimatedResolution: Optional[float] = None
+    estimatedDefocus: Optional[float] = None
+    amplitudeContrast: Optional[float] = None
+    ccValue: Optional[float] = None
+    fftTheoreticalFullPath: Optional[str] = None
+    comments: Optional[str] = None
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(back_populates="CTF")
+    MotionCorrection: Optional["MotionCorrection"] = Relationship(back_populates="CTF")
+
+
+class ParticlePicker(SQLModel, table=True):  # type: ignore
+    particlePickerId: int = Field(primary_key=True, unique=True)
+    programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId")
+    firstMotionCorrectionId: Optional[int] = Field(
+        foreign_key="MotionCorrection.motionCorrectionId"
+    )
+    particlePickingTemplate: Optional[str] = None
+    particleDiameter: Optional[float] = None
+    numberOfParticles: Optional[int] = None
+    summaryImageFullPath: Optional[str] = None
+    MotionCorrection: Optional["MotionCorrection"] = Relationship(
+        back_populates="ParticlePicker"
+    )
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(
+        back_populates="ParticlePicker"
+    )
+    ParticleClassificationGroup: List["ParticleClassificationGroup"] = Relationship(
+        back_populates="ParticlePicker"
+    )
+
+
+class ProcessedTomogram(SQLModel, table=True):  # type: ignore
+    processedTomogramId: int = Field(primary_key=True, unique=True)
+    tomogramId: int = Field(foreign_key="Tomogram.tomogramId")
+    filePath: Optional[str] = None
+    processingType: Optional[str] = None
+    Tomogram: Optional["Tomogram"] = Relationship(back_populates="ProcessedTomogram")
+
+
+class RelativeIceThickness(SQLModel, table=True):  # type: ignore
+    relativeIceThicknessId: int = Field(primary_key=True, unique=True)
+    motionCorrectionId: Optional[int] = Field(
+        foreign_key="MotionCorrection.motionCorrectionId"
+    )
+    autoProcProgramId: Optional[int] = Field(
+        foreign_key="AutoProcProgram.autoProcProgramId"
+    )
+    minimum: Optional[float] = None
+    q1: Optional[float] = None
+    median: Optional[float] = None
+    q3: Optional[float] = None
+    maximum: Optional[float] = None
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(
+        back_populates="RelativeIceThickness"
+    )
+    MotionCorrection: Optional["MotionCorrection"] = Relationship(
+        back_populates="RelativeIceThickness"
+    )
+
+
+class TiltImageAlignment(SQLModel, table=True):  # type: ignore
+    movieId: int = Field(foreign_key="Movie.movieId", primary_key=True)
+    tomogramId: int = Field(foreign_key="Tomogram.tomogramId", primary_key=True)
+    defocusU: Optional[float] = None
+    defocusV: Optional[float] = None
+    psdFile: Optional[str] = None
+    resolution: Optional[float] = None
+    fitQuality: Optional[float] = None
+    refinedMagnification: Optional[float] = None
+    refinedTiltAngle: Optional[float] = None
+    refinedTiltAxis: Optional[float] = None
+    residualError: Optional[float] = None
+    Movie: Optional["Movie"] = Relationship(back_populates="TiltImageAlignment")
+    Tomogram: Optional["Tomogram"] = Relationship(back_populates="TiltImageAlignment")
+
+
+class ParticleClassificationGroup(SQLModel, table=True):  # type: ignore
+    particleClassificationGroupId: int = Field(primary_key=True, unique=True)
+    particlePickerId: Optional[int] = Field(
+        foreign_key="ParticlePicker.particlePickerId"
+    )
+    programId: Optional[int] = Field(foreign_key="AutoProcProgram.autoProcProgramId")
+    type: Optional[str] = Enum("2D", "3D")
+    batchNumber: Optional[int] = None
+    numberOfParticlesPerBatch: Optional[int] = None
+    numberOfClassesPerBatch: Optional[int] = None
+    symmetry: Optional[str] = None
+    binnedPixelSize: Optional[float] = None
+    ParticlePicker: Optional["ParticlePicker"] = Relationship(
+        back_populates="ParticleClassificationGroup"
+    )
+    AutoProcProgram: Optional["AutoProcProgram"] = Relationship(
+        back_populates="ParticleClassificationGroup"
+    )
+    ParticleClassification: List["ParticleClassification"] = Relationship(
+        back_populates="ParticleClassificationGroup"
+    )
+
+
+class ParticleClassification(SQLModel, table=True):  # type: ignore
+    particleClassificationId: int = Field(primary_key=True, unique=True)
+    classNumber: Optional[int] = None
+    classImageFullPath: Optional[str] = None
+    particlesPerClass: Optional[int] = None
+    rotationAccuracy: Optional[float] = None
+    translationAccuracy: Optional[float] = None
+    estimatedResolution: Optional[float] = None
+    overallFourierCompleteness: Optional[float] = None
+    particleClassificationGroupId: Optional[int] = Field(
+        foreign_key="ParticleClassificationGroup.particleClassificationGroupId"
+    )
+    classDistribution: Optional[float] = None
+    selected: Optional[int] = None
+    bFactorFitIntercept: Optional[float] = None
+    bFactorFitLinear: Optional[float] = None
+    bFactorFitQuadratic: Optional[float] = None
+    angularEfficiency: Optional[float] = None
+    suggestedTilt: Optional[float] = None
+    CryoemInitialModel: List["CryoemInitialModel"] = Relationship(
+        back_populates="ParticleClassification"
+    )
+    ParticleClassificationGroup: Optional["ParticleClassificationGroup"] = Relationship(
+        back_populates="ParticleClassification"
+    )
+    BFactorFit: List["BFactorFit"] = Relationship(
+        back_populates="ParticleClassification"
+    )
+
+
+class BFactorFit(SQLModel, table=True):  # type: ignore
+    bFactorFitId: int = Field(primary_key=True, unique=True)
+    particleClassificationId: int = Field(
+        foreign_key="ParticleClassification.particleClassificationId"
+    )
+    resolution: Optional[float] = None
+    numberOfParticles: Optional[int] = None
+    particleBatchSize: Optional[int] = None
+    ParticleClassification: Optional["ParticleClassification"] = Relationship(
+        back_populates="BFactorFit"
+    )
+
+
+class CryoemInitialModel(SQLModel, table=True):  # type: ignore
+    cryoemInitialModelId: int = Field(primary_key=True, unique=True)
+    resolution: Optional[float] = None
+    numberOfParticles: Optional[int] = None
+    ParticleClassification: List["ParticleClassification"] = Relationship(
+        back_populates="CryoemInitialModel"
+    )