Merge pull request #70 from ScilifelabDataCentre/dimensions-model-normalization

SQ-845: Normalize the dimensions table to sample and scaffold FK tables

Author: brinkdp

docker/benchmarking.dockerfile

@@ -1,6 +1,6 @@
 FROM python:3.12-slim
 
-RUN apt-get update && apt-get install -y git make && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y git make tabix && rm -rf /var/lib/apt/lists/*
 
 RUN git clone https://github.com/endast/fake-vcf.git /opt/fake-vcf
 WORKDIR /opt/fake-vcf

docs/development/database-migrations.md

@@ -38,7 +38,7 @@ docker compose -f docker/divbase_compose.yaml down && docker compose -f docker/d
 # Enter the running FastAPI container
 docker compose -f docker/divbase_compose.yaml exec -it fastapi sh
 
-# Generate migration (use descriptive names)
+# Generate migration (use descriptive names). The data will be prepended automatically so it does not need to be included in the name
 alembic revision --autogenerate -m "write_your_useful_slug_here"
 
 # Exit container - if needed - see NOTE below

packages/divbase-api/src/divbase_api/crud/vcf_dimensions.py

@@ -6,18 +6,34 @@
 
 from sqlalchemy import select
 from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.orm import selectinload
 
-from divbase_api.models.vcf_dimensions import SkippedVCFDB, VCFMetadataDB
+from divbase_api.models.vcf_dimensions import SkippedVCFDB, VCFMetadataDB, VCFMetadataSamplesDB, VCFMetadataScaffoldsDB
 
 logger = logging.getLogger(__name__)
 
 
 async def get_vcf_metadata_by_project_async(db: AsyncSession, project_id: int) -> dict:
     """
     FOR USER INTERACTIONS WITH API ONLY
-    Get all VCF metadata entries for a given project ID.
+    Get all VCF metadata entries for a given project ID, including the related sample and scaffold names.
+
+    VCFMetadataDB has a one-to-many relationship relationship with VCFMetadataSamplesDB and with VCFMetadataScaffoldsDB.
+
+    Eager load with selectinload is used to minimize the number of db queries (to 3 queries). It is used instead of table joins since these child tables can have a large amount of entries
+    per parent VCF file, which would result in a large amount of duplicated data being loaded if table joins were used and be inefficient.
+    In this eager load, 1 query is used to load all VCFMetadataDB entries (=VCF files) for the project, 1 query is used to load all related VCFMetadataSamplesDB entries (sample names)
+    for those VCFMetadataDB entries, and 1 query is used to load all related VCFMetadataScaffoldsDB entries (scaffold names) for those VCFMetadataDB entries.
+
+    Lazy load, which is not used here, would instead result in 1 + 2N queries for N VCFMetadataDB entries (=VCF files) for the project, since each VCFMetadataDB entry would be queried
+    separately for its related sample and scaffold names.
     """
-    stmt = select(VCFMetadataDB).where(VCFMetadataDB.project_id == project_id)
+
+    stmt = (
+        select(VCFMetadataDB)
+        .where(VCFMetadataDB.project_id == project_id)
+        .options(selectinload(VCFMetadataDB.samples), selectinload(VCFMetadataDB.scaffolds))
+    )
     result = await db.execute(stmt)
     entries = list(result.scalars().all())
 
@@ -28,8 +44,8 @@ async def get_vcf_metadata_by_project_async(db: AsyncSession, project_id: int) -
             {
                 "vcf_file_s3_key": entry.vcf_file_s3_key,
                 "s3_version_id": entry.s3_version_id,
-                "samples": entry.samples,
-                "scaffolds": entry.scaffolds,
+                "samples": [s.sample_name for s in entry.samples],
+                "scaffolds": [s.scaffold_name for s in entry.scaffolds],
                 "variant_count": entry.variant_count,
                 "sample_count": entry.sample_count,
                 "file_size_bytes": entry.file_size_bytes,
@@ -49,3 +65,41 @@ async def get_skipped_vcfs_by_project_async(db: AsyncSession, project_id: int) -
     stmt = select(SkippedVCFDB).where(SkippedVCFDB.project_id == project_id)
     result = await db.execute(stmt)
     return list(result.scalars().all())
+
+
+async def get_unique_samples_by_project_async(db: AsyncSession, project_id: int) -> list[str]:
+    """
+    Get unique sample names across all VCF files from a project's dimensions entries.
+    """
+
+    stmt = (
+        select(VCFMetadataSamplesDB.sample_name)
+        .join(VCFMetadataDB, VCFMetadataSamplesDB.vcf_metadata_id == VCFMetadataDB.id)
+        .where(VCFMetadataDB.project_id == project_id)
+        .distinct()
+        .order_by(VCFMetadataSamplesDB.sample_name)
+    )
+    result = await db.execute(stmt)
+    return list(result.scalars().all())
+
+
+async def get_unique_scaffolds_by_project_async(db: AsyncSession, project_id: int) -> list[str]:
+    """
+    Get unique scaffold names across all VCF files for a project.
+    """
+
+    stmt = (
+        select(VCFMetadataScaffoldsDB.scaffold_name)
+        .join(VCFMetadataDB, VCFMetadataScaffoldsDB.vcf_metadata_id == VCFMetadataDB.id)
+        .where(VCFMetadataDB.project_id == project_id)
+        .distinct()
+        .order_by(VCFMetadataScaffoldsDB.scaffold_name)
+    )
+    result = await db.execute(stmt)
+    scaffolds = list(result.scalars().all())
+
+    # Sort scaffold names in the same way as the dimensions show CLI does when returning all dimensions data: numeric first, then alphabetic
+    # Numeric sorting of name strings results means that 10 comes after 2 for scaffolds that have numeric names.
+    numeric = sorted([int(s) for s in scaffolds if s.isdigit()])
+    non_numeric = sorted([s for s in scaffolds if not s.isdigit()])
+    return [str(n) for n in numeric] + non_numeric

packages/divbase-api/src/divbase_api/migrations/versions/2026-02-20_normalize_dimensions_model.py

@@ -0,0 +1,83 @@
+"""2026-02-20_normalize_dimensions_model
+
+Revision ID: bde261f949a1
+Revises: faa3d0318fda
+Create Date: 2026-02-20 10:03:40.815406
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+from sqlalchemy.dialects import postgresql
+
+# revision identifiers, used by Alembic.
+revision: str = "bde261f949a1"
+down_revision: Union[str, Sequence[str], None] = "faa3d0318fda"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    """Upgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "vcf_metadata_samples",
+        sa.Column("vcf_metadata_id", sa.Integer(), nullable=False),
+        sa.Column("sample_name", sa.String(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(["vcf_metadata_id"], ["vcf_metadata.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_vcf_metadata_samples_id"), "vcf_metadata_samples", ["id"], unique=False)
+    op.create_index(op.f("ix_vcf_metadata_samples_sample_name"), "vcf_metadata_samples", ["sample_name"], unique=False)
+    op.create_index(
+        op.f("ix_vcf_metadata_samples_vcf_metadata_id"), "vcf_metadata_samples", ["vcf_metadata_id"], unique=False
+    )
+    op.create_table(
+        "vcf_metadata_scaffolds",
+        sa.Column("vcf_metadata_id", sa.Integer(), nullable=False),
+        sa.Column("scaffold_name", sa.String(), nullable=False),
+        sa.Column("id", sa.Integer(), autoincrement=True, nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False),
+        sa.ForeignKeyConstraint(["vcf_metadata_id"], ["vcf_metadata.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_vcf_metadata_scaffolds_id"), "vcf_metadata_scaffolds", ["id"], unique=False)
+    op.create_index(
+        op.f("ix_vcf_metadata_scaffolds_scaffold_name"), "vcf_metadata_scaffolds", ["scaffold_name"], unique=False
+    )
+    op.create_index(
+        op.f("ix_vcf_metadata_scaffolds_vcf_metadata_id"), "vcf_metadata_scaffolds", ["vcf_metadata_id"], unique=False
+    )
+    op.drop_index(op.f("ix_vcf_metadata_samples"), table_name="vcf_metadata")
+    op.drop_column(
+        "vcf_metadata", "scaffolds"
+    )  # NOTE! this drops previous data stored in the array column. The migration does not repopulate the FK tables. Run divbase-cli dimensions update instead
+    op.drop_column("vcf_metadata", "samples")  # NOTE! same here, run divbase-cli dimensions update instead
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    """Downgrade schema."""
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.add_column(
+        "vcf_metadata", sa.Column("samples", postgresql.ARRAY(sa.VARCHAR()), autoincrement=False, nullable=False)
+    )
+    op.add_column(
+        "vcf_metadata", sa.Column("scaffolds", postgresql.ARRAY(sa.VARCHAR()), autoincrement=False, nullable=False)
+    )
+    op.create_index(op.f("ix_vcf_metadata_samples"), "vcf_metadata", ["samples"], unique=False)
+    op.drop_index(op.f("ix_vcf_metadata_scaffolds_vcf_metadata_id"), table_name="vcf_metadata_scaffolds")
+    op.drop_index(op.f("ix_vcf_metadata_scaffolds_scaffold_name"), table_name="vcf_metadata_scaffolds")
+    op.drop_index(op.f("ix_vcf_metadata_scaffolds_id"), table_name="vcf_metadata_scaffolds")
+    op.drop_table("vcf_metadata_scaffolds")
+    op.drop_index(op.f("ix_vcf_metadata_samples_vcf_metadata_id"), table_name="vcf_metadata_samples")
+    op.drop_index(op.f("ix_vcf_metadata_samples_sample_name"), table_name="vcf_metadata_samples")
+    op.drop_index(op.f("ix_vcf_metadata_samples_id"), table_name="vcf_metadata_samples")
+    op.drop_table("vcf_metadata_samples")
+    # ### end Alembic commands ###

packages/divbase-api/src/divbase_api/models/__init__.py

@@ -29,4 +29,6 @@
     "ProjectVersionDB",
     "AnnouncementDB",
     "AnnouncementTarget",
+    "VCFMetadataSamplesDB",
+    "VCFMetadataScaffoldsDB",
 ]

packages/divbase-api/src/divbase_api/models/vcf_dimensions.py

@@ -5,7 +5,6 @@
 from typing import TYPE_CHECKING
 
 from sqlalchemy import BigInteger, ForeignKey, Integer, String, UniqueConstraint
-from sqlalchemy.dialects.postgresql import ARRAY
 from sqlalchemy.orm import Mapped, mapped_column, relationship
 
 from divbase_api.models.base import BaseDBModel
@@ -34,15 +33,20 @@ class VCFMetadataDB(BaseDBModel):
     )
     s3_version_id: Mapped[str] = mapped_column(String, nullable=False, index=True)
     file_size_bytes: Mapped[int] = mapped_column(BigInteger)
-    samples: Mapped[list[str]] = mapped_column(ARRAY(String), index=True)  # Sample names
-    scaffolds: Mapped[list[str]] = mapped_column(ARRAY(String))  # Scaffold/chromosome names
     variant_count: Mapped[int] = mapped_column(BigInteger)
     sample_count: Mapped[int] = mapped_column(Integer)
 
     __table_args__ = (UniqueConstraint("vcf_file_s3_key", "project_id", name="unique_vcf_per_project"),)
 
     project: Mapped["ProjectDB"] = relationship("ProjectDB", back_populates="vcf_metadata")
 
+    samples: Mapped[list["VCFMetadataSamplesDB"]] = relationship(
+        "VCFMetadataSamplesDB", back_populates="vcf_metadata", cascade="all, delete-orphan"
+    )
+    scaffolds: Mapped[list["VCFMetadataScaffoldsDB"]] = relationship(
+        "VCFMetadataScaffoldsDB", back_populates="vcf_metadata", cascade="all, delete-orphan"
+    )
+
 
 class SkippedVCFDB(BaseDBModel):
     """
@@ -63,3 +67,35 @@ class SkippedVCFDB(BaseDBModel):
     __table_args__ = (UniqueConstraint("vcf_file_s3_key", "project_id", name="unique_skipped_vcf_per_project"),)
 
     project: Mapped["ProjectDB"] = relationship("ProjectDB", back_populates="skipped_vcf_files")
+
+
+class VCFMetadataSamplesDB(BaseDBModel):
+    """
+    DB model for one-to-many relationship between VCF file metadata and the sample names in the VCF files.
+    """
+
+    __tablename__ = "vcf_metadata_samples"
+
+    vcf_metadata_id: Mapped[int] = mapped_column(
+        ForeignKey("vcf_metadata.id", ondelete="CASCADE"),
+        index=True,
+    )
+    sample_name: Mapped[str] = mapped_column(String, index=True)
+
+    vcf_metadata: Mapped["VCFMetadataDB"] = relationship("VCFMetadataDB", back_populates="samples")
+
+
+class VCFMetadataScaffoldsDB(BaseDBModel):
+    """
+    DB model for one-to-many relationship between VCF file metadata and the scaffold names in the VCF files.
+    """
+
+    __tablename__ = "vcf_metadata_scaffolds"
+
+    vcf_metadata_id: Mapped[int] = mapped_column(
+        ForeignKey("vcf_metadata.id", ondelete="CASCADE"),
+        index=True,
+    )
+    scaffold_name: Mapped[str] = mapped_column(String, index=True)
+
+    vcf_metadata: Mapped["VCFMetadataDB"] = relationship("VCFMetadataDB", back_populates="scaffolds")

packages/divbase-api/src/divbase_api/routes/queries.py

@@ -29,6 +29,7 @@
     SampleMetadataQueryRequest,
     SampleMetadataQueryTaskResult,
 )
+from divbase_lib.exceptions import DimensionsNotUpToDateWithBucketError
 
 logging.basicConfig(level=settings.api.log_level, handlers=[logging.StreamHandler(sys.stderr)])
 
@@ -84,6 +85,8 @@ async def sample_metadata_query(
     except VCFDimensionsEntryMissingError:
         # Catch and raise anew to avoid duplications in the error message
         raise VCFDimensionsEntryMissingError(project_name=project.name) from None
+    except DimensionsNotUpToDateWithBucketError as e:
+        raise DimensionsNotUpToDateWithBucketError(str(e)) from None
     except celery.exceptions.TimeoutError:  # type: ignore
         error_message = (
             f"The query is still being processed and has Task ID: {results.id}. \n"

packages/divbase-api/src/divbase_api/routes/vcf_dimensions.py

@@ -11,6 +11,8 @@
 from divbase_api.crud.task_history import create_task_history_entry
 from divbase_api.crud.vcf_dimensions import (
     get_skipped_vcfs_by_project_async,
+    get_unique_samples_by_project_async,
+    get_unique_scaffolds_by_project_async,
     get_vcf_metadata_by_project_async,
 )
 from divbase_api.db import get_db
@@ -19,7 +21,12 @@
 from divbase_api.models.projects import ProjectDB, ProjectRoles
 from divbase_api.models.users import UserDB
 from divbase_api.worker.tasks import update_vcf_dimensions_task
-from divbase_lib.api_schemas.vcf_dimensions import DimensionsShowResult, DimensionUpdateKwargs
+from divbase_lib.api_schemas.vcf_dimensions import (
+    DimensionsSamplesResult,
+    DimensionsScaffoldsResult,
+    DimensionsShowResult,
+    DimensionUpdateKwargs,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -102,3 +109,43 @@ async def update_vcf_dimensions_endpoint(
     )
 
     return job_id
+
+
+@vcf_dimensions_router.get(
+    "/projects/{project_name}/samples", status_code=status.HTTP_200_OK, response_model=DimensionsSamplesResult
+)
+async def list_unique_samples_endpoint(
+    project_name: str,
+    project_and_user_and_role: tuple[ProjectDB, UserDB, ProjectRoles] = Depends(get_project_member),
+    db: AsyncSession = Depends(get_db),
+) -> DimensionsSamplesResult:
+    """Get all unique sample names across project VCFs."""
+
+    project, current_user, role = project_and_user_and_role
+
+    if not has_required_role(role, ProjectRoles.READ):
+        raise AuthorizationError("You don't have permission to view VCF dimensions for this project.")
+
+    result = await get_unique_samples_by_project_async(db, project.id)
+
+    return DimensionsSamplesResult(unique_samples=result)
+
+
+@vcf_dimensions_router.get(
+    "/projects/{project_name}/scaffolds", status_code=status.HTTP_200_OK, response_model=DimensionsScaffoldsResult
+)
+async def list_unique_scaffolds_endpoint(
+    project_name: str,
+    project_and_user_and_role: tuple[ProjectDB, UserDB, ProjectRoles] = Depends(get_project_member),
+    db: AsyncSession = Depends(get_db),
+) -> DimensionsScaffoldsResult:
+    """Get all unique scaffold names across project VCFs."""
+
+    project, current_user, role = project_and_user_and_role
+
+    if not has_required_role(role, ProjectRoles.READ):
+        raise AuthorizationError("You don't have permission to view VCF dimensions for this project.")
+
+    result = await get_unique_scaffolds_by_project_async(db, project.id)
+
+    return DimensionsScaffoldsResult(unique_scaffolds=result)