DM-51337 : Release 0.5.0

.github/workflows/ci.yaml

@@ -12,6 +12,7 @@ name: "CI"
 
 env:
   UV_FROZEN: "1"
+  TEST__LOCAL_DB: "1"  # signals test fixtures to not use testcontainers
 
 jobs:
   rebase-checker:
@@ -30,7 +31,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
 
       - name: Run pre-commit
         uses: pre-commit/action@v3.0.1
@@ -39,7 +40,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     needs:
       - rebase-checker
     steps:
@@ -49,7 +50,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
-          version: "0.6.x"
+          version: "0.7.x"
           enable-cache: true
           python-version: ${{ matrix.python-version }}
 
@@ -63,7 +64,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ["3.11"]
+        python-version: ["3.12"]
     needs:
       - lint
       - mypy
@@ -75,7 +76,7 @@ jobs:
       - name: Install uv
         uses: astral-sh/setup-uv@v5
         with:
-          version: "0.6.x"
+          version: "0.7.x"
           enable-cache: true
           python-version: ${{ matrix.python-version }}
 

.pre-commit-config.yaml

@@ -11,7 +11,7 @@ repos:
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
-    rev: v0.9.2
+    rev: v0.11.13
     hooks:
       - id: ruff
       - id: ruff

.python-version

@@ -1 +1 @@
-3.11
+3.12

Makefile

@@ -44,6 +44,7 @@ update: update-deps init
 
 .PHONY: build
 build: export BUILDKIT_PROGRESS=plain
+build: export COMPOSE_BAKE=true
 build:
 	docker compose build cmservice
 	docker compose build cmworker
@@ -107,6 +108,7 @@ test: PGPORT=$(shell docker compose port postgresql 5432 | cut -d: -f2)
 test: export DB__URL=postgresql://cm-service@localhost:${PGPORT}/cm-service
 test: export DB__PASSWORD=INSECURE-PASSWORD
 test: export DB__TABLE_SCHEMA=cm_service_test
+test: export BPS__ARTIFACT_PATH=$(PWD)/output
 test: run-compose
 	alembic upgrade head
 	pytest -vvv --asyncio-mode=auto --cov=lsst.cmservice --cov-branch --cov-report=term --cov-report=html ${PYTEST_ARGS}

README.md

@@ -1,4 +1,5 @@
 # cm-service
+![Python](https://img.shields.io/python/required-version-toml?tomlFilePath=https%3A%2F%2Fraw.githubusercontent.com%2Flsst-dm%2Fcm-service%2Frefs%2Fheads%2Fmain%2Fpyproject.toml)
 [![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff)
 [![uv](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/uv/main/assets/badge/v0.json)](https://github.com/astral-sh/uv)
 
@@ -8,7 +9,7 @@ https://cm-service.lsst.io.
 
 ## Developer Quick Start
 
-You can build and run `cm-service` on any system which has Python 3.11 or greater, `uv`, `make`, and Docker w/ the
+You can build and run `cm-service` on any system which has Python 3.12 or greater, `uv`, `make`, and Docker w/ the
 Docker Compose V2 CLI plugin (this includes, in particular, recent MacOS with Docker Desktop).  Proceed as
 follows:
 

alembic/README.md

@@ -3,8 +3,6 @@
 Database migrations and schema evolution are handled by `alembic`, a database tool
 that is part of the `sqlalchemy` toolkit ecosystem.
 
-Alembic is included in the project's dependency graph via the Safir package.
-
 ## Running Alembic
 
 The `alembic` tool establishes an execution environment via the `env.py` file which

alembic/versions/1da92a1c740f_create_v2_tables.py

@@ -0,0 +1,250 @@
+"""create v2 tables
+
+Revision ID: 1da92a1c740f
+Revises: acf951c80750
+Create Date: 2025-06-13 14:56:31.238050+00:00
+
+"""
+
+from collections.abc import Sequence
+from enum import Enum
+from uuid import NAMESPACE_DNS
+
+import sqlalchemy as sa
+from sqlalchemy.dialects import postgresql
+
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "1da92a1c740f"
+down_revision: str | None = "acf951c80750"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+
+DEFAULT_CAMPAIGN_NAMESPACE = "dda54a0c-6878-5c95-ac4f-007f6808049e"
+"""UUID5 of name 'io.lsst.cmservice' in `uuid.NAMESPACE_DNS`."""
+
+# DB model uses mapped columns with Python Enum types, but we do not care
+# to use native enums in the database, so when we have such a column, this
+# definition will produce a VARCHAR instead.
+ENUM_COLUMN_AS_VARCHAR = sa.Enum(Enum, length=20, native_enum=False, check_constraint=False)
+
+
+def upgrade() -> None:
+    # Create table for machines v2
+    machines_v2 = op.create_table(
+        "machines_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("state", sa.PickleType, nullable=False),
+        sa.PrimaryKeyConstraint("id"),
+        if_not_exists=True,
+    )
+
+    # Create table for campaigns v2
+    campaigns_v2 = op.create_table(
+        "campaigns_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("name", postgresql.VARCHAR(), nullable=False),
+        sa.Column("namespace", postgresql.UUID(), nullable=False, default=DEFAULT_CAMPAIGN_NAMESPACE),
+        sa.Column("owner", postgresql.VARCHAR(), nullable=True),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column(
+            "configuration",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column("status", ENUM_COLUMN_AS_VARCHAR, nullable=False, default="waiting"),
+        sa.Column(
+            "machine", postgresql.UUID(), sa.ForeignKey(machines_v2.c.id, ondelete="CASCADE"), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "namespace"),
+        if_not_exists=True,
+    )
+
+    # Create node and edges tables for campaign digraph
+    nodes_v2 = op.create_table(
+        "nodes_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column(
+            "namespace",
+            postgresql.UUID(),
+            sa.ForeignKey(campaigns_v2.c.id, ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("name", postgresql.VARCHAR(), nullable=False),
+        sa.Column("version", postgresql.INTEGER(), nullable=False, default=1),
+        sa.Column("kind", ENUM_COLUMN_AS_VARCHAR, nullable=False, default="node"),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column(
+            "configuration",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column("status", ENUM_COLUMN_AS_VARCHAR, nullable=False, default="waiting"),
+        sa.Column(
+            "machine", postgresql.UUID(), sa.ForeignKey(machines_v2.c.id, ondelete="CASCADE"), nullable=True
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "version", "namespace"),
+        if_not_exists=True,
+    )
+
+    _ = op.create_table(
+        "edges_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("name", postgresql.VARCHAR(), nullable=False),
+        sa.Column(
+            "namespace",
+            postgresql.UUID(),
+            sa.ForeignKey(campaigns_v2.c.id, ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("source", postgresql.UUID(), sa.ForeignKey(nodes_v2.c.id), nullable=False),
+        sa.Column("target", postgresql.UUID(), sa.ForeignKey(nodes_v2.c.id), nullable=False),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column(
+            "configuration",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("source", "target", "namespace"),
+        if_not_exists=True,
+    )
+
+    # Create table for spec blocks v2 ("manifests")
+    _ = op.create_table(
+        "manifests_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("name", postgresql.VARCHAR(), nullable=False),
+        sa.Column(
+            "namespace",
+            postgresql.UUID(),
+            sa.ForeignKey(campaigns_v2.c.id, ondelete="CASCADE"),
+            nullable=False,
+        ),
+        sa.Column("version", postgresql.INTEGER(), nullable=False, default=1),
+        sa.Column("kind", ENUM_COLUMN_AS_VARCHAR, nullable=False, default="other"),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column(
+            "spec",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("name", "version", "namespace"),
+        if_not_exists=True,
+    )
+
+    # Create table for tasks v2
+    _ = op.create_table(
+        "tasks_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("namespace", postgresql.UUID(), nullable=False),
+        sa.Column("node", postgresql.UUID(), nullable=False),
+        sa.Column("priority", postgresql.INTEGER(), nullable=True),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), nullable=False),
+        sa.Column("submitted_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("finished_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("wms_id", postgresql.VARCHAR(), nullable=True),
+        sa.Column("site_affinity", postgresql.ARRAY(postgresql.VARCHAR()), nullable=True),
+        sa.Column("status", ENUM_COLUMN_AS_VARCHAR, nullable=False),
+        sa.Column("previous_status", ENUM_COLUMN_AS_VARCHAR, nullable=True),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        sa.ForeignKeyConstraint(["node"], ["nodes_v2.id"]),
+        sa.ForeignKeyConstraint(["namespace"], ["campaigns_v2.id"]),
+        if_not_exists=True,
+    )
+
+    _ = op.create_table(
+        "activity_log_v2",
+        sa.Column("id", postgresql.UUID(), nullable=False),
+        sa.Column("namespace", postgresql.UUID(), nullable=False),
+        sa.Column("node", postgresql.UUID(), sa.ForeignKey(nodes_v2.c.id), nullable=True),
+        sa.Column("operator", postgresql.VARCHAR(), nullable=False, default="root"),
+        sa.Column("created_at", postgresql.TIMESTAMP(timezone=True), nullable=False),
+        sa.Column("finished_at", postgresql.TIMESTAMP(timezone=True), nullable=True),
+        sa.Column("from_status", ENUM_COLUMN_AS_VARCHAR, nullable=False),
+        sa.Column("to_status", ENUM_COLUMN_AS_VARCHAR, nullable=False),
+        sa.Column(
+            "detail",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.Column(
+            "metadata",
+            postgresql.JSONB(),
+            nullable=False,
+            default=dict,
+            server_default=sa.text("'{}'::json"),
+        ),
+        sa.PrimaryKeyConstraint("id"),
+        if_not_exists=True,
+    )
+
+    # Insert default campaign (namespace) record
+    op.bulk_insert(
+        campaigns_v2,
+        [
+            {
+                "id": DEFAULT_CAMPAIGN_NAMESPACE,
+                "namespace": str(NAMESPACE_DNS),
+                "name": "DEFAULT",
+                "owner": "root",
+            }
+        ],
+    )
+
+
+def downgrade() -> None:
+    """Drop tables in the reverse order in which they were created."""
+    op.drop_table("activity_log_v2", if_exists=True)
+    op.drop_table("tasks_v2", if_exists=True)
+    op.drop_table("manifests_v2", if_exists=True)
+    op.drop_table("edges_v2", if_exists=True)
+    op.drop_table("nodes_v2", if_exists=True)
+    op.drop_table("campaigns_v2", if_exists=True)
+    op.drop_table("machines_v2", if_exists=True)

docker/Dockerfile

@@ -1,7 +1,7 @@
 # syntax=docker/dockerfile:1
 
-ARG PYTHON_VERSION="3.11"
-ARG UV_VERSION="0.6"
+ARG PYTHON_VERSION="3.12"
+ARG UV_VERSION="0.7"
 ARG ASGI_PORT="8080"
 
 #==============================================================================

docs/CONTRIBUTING.md

@@ -19,9 +19,7 @@ Releases are performed at an unspecified cadence, to be no shorter than 1 week a
 - Releases are named according to their semantic version (major.minor.patch).
 - Releases are made by adding a named tag to the trunk branch.
 - Each release will increment the minor version and set the patch level to 0, e.g., `1.0.12` -> `1.1.0`
-- If a bugfix commit needs to be added to a release, then a retroactive branch will be created from the
-  release tag; the commit is cherry-picked into the release branch and a new tag is written with an incremented
-  patch level, e.g., `1.23.0` -> `1.23.1`.
+- If a bugfix commit in the trunk needs to be added to a release, then a retroactive branch will be created from the affected release tag; any fix commits are cherry-picked into the release branch and a new tag is written with an incremented patch level, e.g., `1.23.0` -> `1.23.1`. This release branch is never merged to `main` (trunk) but is kept for subsequent cherry-picked fixes.
 - The major version is incremented only in the presence of user-facing breaking changes.
 
 This project uses `python-semantic-release` to manage releases. A release may be triggered by any ticket branch

docs/DEPLOYING.md

@@ -50,3 +50,18 @@ The CM Service consumes several secrets from the k8s application environment. Th
   - The AWS "default" profile is configured to use environment variables for its credentials source, so the appropriate secrets should be assigned to the `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` environment variables.
 
   - Authentication values for other profiles are stored in a secret mounted as a file at `/etc/aws/credentials`.
+
+## Dev / Staging Deployments
+Development or staging builds are deployed to the USDF Kubernetes vcluster `usdf-cm-dev`.
+
+Prior to deploying a new build using a prerelease tag, release tag, or ticket branch tag, it may be necessary to clear the database of all data so it can be migrated from scratch.
+
+### Clearing the Database
+After appropriately setting the Kubernetes context and namespace:
+
+1. Scale down the daemon deployment using `kubectl scale deployment cm-service-daemon --replicas=0`.
+1. Obtain a shell in an API server pod using `kubectl exec -it cm-service-server-<hash> -- bash`.
+1. Within this shell, downgrade the database migration using `alembic downgrade base`.
+
+> [!CAUTION]
+> This operation unconditionally destroys the database contents and all objects.