Merge pull request #53 from ScilifelabDataCentre/project-versioning-to-db

SQ-791: Move bucket/project versioning store from s3 to db

Author: RMCrean

docs/using_divbase_cli.md

@@ -69,38 +69,61 @@ divbase-cli config show-default
 
 ## 3. Versioning the projects state
 
-The DivBase server uses S3 buckets to store project files. Each project has its own assigned bucket. These buckets support versioning of files natively via S3. So individual files can be restored to prior versions if needed.
+DivBase allows you to create named versions of your project's state at specific points in time. This is useful for tracking changes, ensuring reproducibility, and marking important milestones (e.g., when running analysis for a publication).
 
-On top of this we also support versioning of the entire project's state via a special file stored in the bucket. This allows users to create named versions of the entire project at a given timepoint.
+This means you can save the current state of your project as a named version, and later retrieve files as they were at that specific version (so you don't have to worry about having later updated files).
 
-One potential use case for this could be:
+### Add a new version
 
-- Marking a time point when you did analysis for the a publication.
+To save the current state of your project run:
 
-At a later date, you could then download/upload all files from/to the bucket as they were at that timepoint to ensure reproducibility of results.
+```bash
+divbase-cli version add NAME [OPTIONS]
+```
+
+- Replace `NAME` with a unique name for the version (e.g., v1.0.0).
+- You can optionally add a description using the --description flag.
+
+**Note:** Versions are project wide, so you share them with all other members of the same project.
 
-If you're in a new project, you can create the bucket versioning file by running
+### Listing Versions
+
+To see all existing versions of your project, run:
 
 ```bash
-divbase-cli version create
+divbase-cli version list
 ```
 
-This will create a file in the bucket called `.bucket_versions.yaml`
+This will display a list of all saved versions, including their names, descriptions, and creation dates.
+
+### Specific Version Details
 
-An already existing bucket likely has this file, we can see the contents of this file by running:
+To get detailed information about a specific version, use:
 
 ```bash
-divbase-cli version list
+divbase-cli version info NAME
 ```
 
-If after working with the bucket for a while you want to version the current state of the bucket, you can run:
+This view will also include all the files associated with that version.
+
+### Deleting Versions
+
+To delete a specific version from your project, run:
 
 ```bash
-divbase-cli version add [OPTIONS] NAME
+divbase-cli version delete NAME
 ```
 
-To download files from bucket at a specific bucket version/state, we can use the --bucket-version option and specify the version name we want to download from:
+This will delete the version entry from the project. Deleted versions older than 30 days will be permanently deleted. You can ask a DivBase admin to restore a deleted version within that time period.
+
+**NOTE:** The files associated with the version are never deleted by this operation.
+
+### Downloading files from a specific version
+
+To download files from your project as they were at a specific version, use the --project-version option:
 
 ```bash
-divbase-cli files download file1.vcf.gz file2.vcf.gz --bucket-version=v0.1.0
+divbase-cli files download file1.vcf.gz file2.vcf.gz --project-version=NAME
 ```
+
+Replacing `NAME` with the name of the version you want to download files from.

packages/divbase-api/src/divbase_api/admin_panel.py

@@ -24,6 +24,7 @@
     EmailField,
     HasOne,
     IntegerField,
+    JSONField,
     StringField,
     TextAreaField,
 )
@@ -35,6 +36,7 @@
 from divbase_api.db import get_db
 from divbase_api.deps import _authenticate_frontend_user_from_tokens
 from divbase_api.frontend_routes.auth import get_login, post_logout
+from divbase_api.models.project_versions import ProjectVersionDB
 from divbase_api.models.projects import ProjectDB, ProjectMembershipDB
 from divbase_api.models.revoked_tokens import RevokedTokenDB
 from divbase_api.models.task_history import CeleryTaskMeta, TaskHistoryDB, TaskStartedAtDB
@@ -100,6 +102,14 @@ class UserView(ModelView):
             disabled=True,
         ),
         "project_memberships",
+        DateTimeField(
+            "created_at", help_text="Timestamp when the entry was created. Value determined by system.", disabled=True
+        ),
+        DateTimeField(
+            "updated_at",
+            help_text="Timestamp when the entry was last updated. Value determined by system.",
+            disabled=True,
+        ),
     ]
 
     exclude_fields_from_list = ["hashed_password", "password"]
@@ -132,13 +142,11 @@ async def edit(self, request: Request, pk: Any, data: dict) -> Any:
         Override the default edit method to ensure that the `date_deleted` field is updated
         when/if a users soft deletion status is changed.
         """
-        logger.info(f"Editing user with pk={pk}, data={data}")
         if "is_deleted" in data:
             if data["is_deleted"]:
                 data["date_deleted"] = datetime.now(tz=timezone.utc)
             else:
                 data["date_deleted"] = None
-        logger.info(f"Editing user with pk={pk}, data={data}")
 
         return await super().edit(request=request, pk=pk, data=data)
 
@@ -202,6 +210,14 @@ class ProjectView(ModelView):
             help_text="Current storage usage for this project in bytes.",
         ),
         BooleanField("is_active", required=True, label="Is Active", help_text="Mark the project as active or not."),
+        DateTimeField(
+            "created_at", help_text="Timestamp when the entry was created. Value determined by system.", disabled=True
+        ),
+        DateTimeField(
+            "updated_at",
+            help_text="Timestamp when the entry was last updated. Value determined by system.",
+            disabled=True,
+        ),
     ]
 
     exclude_fields_from_list = ["description", "storage_used_bytes"]
@@ -293,6 +309,65 @@ async def serialize_field_value(self, value: Any, field: Any, action: RequestAct
         return await super().serialize_field_value(value, field, action, request)
 
 
+class ProjectVersionsView(ModelView):
+    """
+    Custom admin panel View for the ProjectVersionDB model.
+    """
+
+    fields = [
+        "id",
+        StringField("name", required=True, label="Version Name", help_text="Unique name for the version."),
+        TextAreaField("description", required=False, label="Description"),
+        HasOne("project", identity="project", label="Project"),
+        IntegerField(
+            "user_id", label="User ID"
+        ),  # No relationship created for this field in db model as this is for auditing only (can be null if user deleted)
+        BooleanField("is_deleted", required=True, label="Is Deleted", help_text="Mark the version as deleted or not."),
+        DateTimeField(
+            "date_deleted",
+            help_text="Timestamp when the version was soft deleted (else None). Value determined by system, cannot be edited.",
+            disabled=True,
+        ),
+        JSONField(
+            "files", required=True, label="Files", help_text="Mapping of file names to version IDs.", disabled=True
+        ),
+        DateTimeField(
+            "created_at", help_text="Timestamp when the entry was created. Value determined by system.", disabled=True
+        ),
+        DateTimeField(
+            "updated_at",
+            help_text="Timestamp when the entry was last updated. Value determined by system.",
+            disabled=True,
+        ),
+    ]
+
+    page_size_options = PAGINATION_DEFAULTS
+    exclude_fields_from_list = ["files"]
+    exclude_fields_from_edit = ["id", "created_at", "updated_at", "files", "project", "user_id"]
+    exclude_fields_from_detail = []
+
+    def can_delete(self, request: Request) -> bool:
+        """Disable deletion of project versions. Project versions can be soft deleted instead."""
+        return False
+
+    def can_create(self, request: Request) -> bool:
+        """Disable creation of project versions. This is something users can create instead."""
+        return False
+
+    async def edit(self, request: Request, pk: Any, data: dict) -> Any:
+        """
+        Override the default edit method to ensure that the `date_deleted` field is updated
+        when/if a version's soft deletion status is changed.
+        """
+        if "is_deleted" in data:
+            if data["is_deleted"]:
+                data["date_deleted"] = datetime.now(tz=timezone.utc)
+            else:
+                data["date_deleted"] = None
+
+        return await super().edit(request=request, pk=pk, data=data)
+
+
 class RevokedTokenView(ModelView):
     """
     Custom admin panel View for the RevokedTokenDB model.
@@ -337,70 +412,6 @@ async def serialize_field_value(self, value: Any, field: Any, action: RequestAct
         return await super().serialize_field_value(value, field, action, request)
 
 
-class DivBaseAuthProvider(AuthProvider):
-    """
-    This class enables starlette-admin to make use of DivBase's pre-existing auth system.
-
-    The methods below are overriding several existing methods in the AuthProvider class (and its parent BaseAuthProvider).
-    """
-
-    async def render_login(self, request: Request, admin: BaseAdmin) -> Response:
-        """Override the default starlette-admin login method to use our frontend get_login route/page."""
-        return await get_login(request)
-
-    async def render_logout(self, request: Request, admin: BaseAdmin) -> Response:
-        """Override the default starlette-admin logout to use our frontend post_logout function/route."""
-        # can't rely on dependency injection here like in FastAPI, so we manually obtain a db session
-        async for db in get_db():
-            logout_response = await post_logout(request, db)
-        return logout_response
-
-    async def is_authenticated(self, request: Request) -> bool:
-        """
-        Overrides the default implementation to use our pre-existing DivBase auth system.
-
-        In which we determine the user from their JWT tokens stored in httponly cookies.
-
-        As expected, we also confirm user.is_admin for access.
-        """
-        access_token = request.cookies.get("access_token")
-        refresh_token = request.cookies.get("refresh_token")
-        if not access_token and not refresh_token:
-            return False
-
-        try:
-            # Starlette does not support dependency injection like FastAPI,
-            # so we need to manually obtain the database session here.
-            async for db in get_db():
-                user = await _authenticate_frontend_user_from_tokens(
-                    access_token=access_token, refresh_token=refresh_token, db=db
-                )
-
-                if user and user.is_admin and user.is_active:
-                    # Store user info in the request state so it can be accessed by e.g. get_admin_user
-                    request.state.user = {"id": user.id, "name": user.name, "is_admin": user.is_admin}
-                    return True
-        except Exception as e:
-            logger.warning(
-                f"An error occurred while attempting to authenticate a user on the starlette-admin panel, details: {e}"
-            )
-            return False
-
-        return False
-
-    def get_admin_user(self, request: Request) -> AdminUser | None:
-        """
-        Retrieve the current (admin) user for display on the admin panel.
-
-        This controls the display of the user info in the top right of the admin panel and makes having a logout button possible.
-        """
-        user = request.state.user
-        if not user:
-            return None
-
-        return AdminUser(username=user["name"], photo_url=None)
-
-
 class TaskHistoryView(ModelView):
     page_size_options = PAGINATION_DEFAULTS
 
@@ -556,6 +567,70 @@ def can_delete(self, request: Request) -> bool:
         return False
 
 
+class DivBaseAuthProvider(AuthProvider):
+    """
+    This class enables starlette-admin to make use of DivBase's pre-existing auth system.
+
+    The methods below are overriding several existing methods in the AuthProvider class (and its parent BaseAuthProvider).
+    """
+
+    async def render_login(self, request: Request, admin: BaseAdmin) -> Response:
+        """Override the default starlette-admin login method to use our frontend get_login route/page."""
+        return await get_login(request)
+
+    async def render_logout(self, request: Request, admin: BaseAdmin) -> Response:
+        """Override the default starlette-admin logout to use our frontend post_logout function/route."""
+        # can't rely on dependency injection here like in FastAPI, so we manually obtain a db session
+        async for db in get_db():
+            logout_response = await post_logout(request, db)
+        return logout_response
+
+    async def is_authenticated(self, request: Request) -> bool:
+        """
+        Overrides the default implementation to use our pre-existing DivBase auth system.
+
+        In which we determine the user from their JWT tokens stored in httponly cookies.
+
+        As expected, we also confirm user.is_admin for access.
+        """
+        access_token = request.cookies.get("access_token")
+        refresh_token = request.cookies.get("refresh_token")
+        if not access_token and not refresh_token:
+            return False
+
+        try:
+            # Starlette does not support dependency injection like FastAPI,
+            # so we need to manually obtain the database session here.
+            async for db in get_db():
+                user = await _authenticate_frontend_user_from_tokens(
+                    access_token=access_token, refresh_token=refresh_token, db=db
+                )
+
+                if user and user.is_admin and user.is_active:
+                    # Store user info in the request state so it can be accessed by e.g. get_admin_user
+                    request.state.user = {"id": user.id, "name": user.name, "is_admin": user.is_admin}
+                    return True
+        except Exception as e:
+            logger.warning(
+                f"An error occurred while attempting to authenticate a user on the starlette-admin panel, details: {e}"
+            )
+            return False
+
+        return False
+
+    def get_admin_user(self, request: Request) -> AdminUser | None:
+        """
+        Retrieve the current (admin) user for display on the admin panel.
+
+        This controls the display of the user info in the top right of the admin panel and makes having a logout button possible.
+        """
+        user = request.state.user
+        if not user:
+            return None
+
+        return AdminUser(username=user["name"], photo_url=None)
+
+
 def register_admin_panel(app: FastAPI, engine: AsyncEngine) -> None:
     """
     Create and register an admin panel for the FastAPI app.
@@ -565,11 +640,12 @@ def register_admin_panel(app: FastAPI, engine: AsyncEngine) -> None:
     admin.add_view(UserView(UserDB, icon="fas fa-user", label="Users", identity="user"))
     admin.add_view(ProjectView(ProjectDB, icon="fas fa-folder", label="Projects", identity="project"))
     admin.add_view(ProjectMembershipView(ProjectMembershipDB, icon="fas fa-link", label="Project Memberships"))
+    admin.add_view(ProjectVersionsView(ProjectVersionDB, icon="fas fa-history", label="Project Versions"))
     admin.add_view(RevokedTokenView(RevokedTokenDB, icon="fas fa-ban", label="Revoked Tokens"))
     admin.add_view(TaskHistoryView(TaskHistoryDB, icon="fas fa-history", label="Task History"))
     admin.add_view(
         CeleryTaskMetaView(CeleryTaskMeta, icon="fas fa-tasks", label="Celery Task Meta", identity="celery-meta")
     )
-    admin.add_view(TaskStartedAtView(TaskStartedAtDB, icon="fas fa-clock", label="Task Started At"))
 
+    admin.add_view(TaskStartedAtView(TaskStartedAtDB, icon="fas fa-clock", label="Task Started At"))
     admin.mount_to(app)