fix(release): repair the release pipeline and sign released artifacts

Release-pipeline fixes:
- Attach the DI container on the early-dispatch path so 'orb init' and
  'orb templates generate' work on a fresh checkout (they ran before the
  router attached it), unblocking the OpenAPI-export release step.
- Fold the web UI into the 'all' extra (and add monitoring-aws) so
  'pip install orb-py[all]' and the published container are fully functional;
  the container installs [all], and the release install syncs --extra all
  --no-dev (ui conflicts with the ci/dev groups by design).  A unit test guards
  that 'all' keeps covering every runtime extra.
- SBOM generation no longer fails the workflow (so a routine advisory can't
  block the release chain); a real SPDX SBOM is emitted.
- Manual workflow_dispatch can no longer publish to prod PyPI or move the
  'latest' container tag (publish/manifest/asset jobs gated to release events).
- OpenAPI-export readiness check polls the served endpoint instead of health.
- Serial-provider junit/coverage filenames match the Codecov upload pattern so
  the serial leg's results are counted.

Signing & provenance:
- GitHub release assets carry a Sigstore build-provenance attestation
  (keyless via OIDC), uploaded alongside the release.
- Container images are signed with cosign (keyless) and ship SLSA build
  provenance + an SBOM attestation (SLSA Build L2); images are scanned before
  publish.  PyPI publishing keeps its PEP 740 attestations.

Author: fgogolli

.github/workflows/dev-containers.yml

@@ -103,6 +103,10 @@ jobs:
       contents: read
       packages: write
       security-events: write
+      # Required for keyless cosign signing (ambient OIDC token) and
+      # actions/attest-build-provenance on pushed images.
+      id-token: write
+      attestations: write
 
     steps:
     - name: Checkout code
@@ -134,6 +138,7 @@ jobs:
       run: make build
 
     - name: Build and scan container image
+      id: build
       shell: bash
       env:
         REGISTRY: ${{ needs.config.outputs.container-registry }}
@@ -169,12 +174,23 @@ jobs:
             --build-arg "VCS_REF=$(git rev-parse --short HEAD)" \
             --build-arg "PYTHON_VERSION=$PYTHON_VERSION" \
             --build-arg "PACKAGE_NAME_SHORT=$PACKAGE_NAME_SHORT" \
+            --provenance=true \
+            --sbom=true \
+            --metadata-file metadata.json \
             --cache-from type=gha \
             --cache-to type=gha,mode=max \
             --push \
             -t "$REGISTRY/$IMAGE_NAME:$VERSION-python$PYTHON_VERSION" \
             .
 
+          # Extract the immutable digest for the signing steps below.
+          DIGEST=$(jq -r '."containerimage.digest"' metadata.json)
+          {
+            echo "digest=${DIGEST}"
+            echo "image=${REGISTRY}/${IMAGE_NAME}"
+            echo "pushed=true"
+          } >> "$GITHUB_OUTPUT"
+
           # Pull back a single-arch image for local Trivy scanning.
           # Multi-arch manifests cannot be loaded/scanned; we use the first
           # entry of build-platforms (SCAN_PLATFORM) rather than hardcoding amd64.
@@ -195,8 +211,29 @@ jobs:
             --load \
             -t "$REGISTRY/$IMAGE_NAME:$VERSION-python$PYTHON_VERSION" \
             .
+          echo "pushed=false" >> "$GITHUB_OUTPUT"
         fi
 
+    - name: Install cosign
+      # Only needed when an image was pushed; skip on PR / local-only builds.
+      if: steps.build.outputs.pushed == 'true'
+      uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6  # v4.1.2
+
+    - name: Sign container image (keyless / OIDC)
+      if: steps.build.outputs.pushed == 'true'
+      env:
+        DIGEST: ${{ steps.build.outputs.digest }}
+        IMAGE: ${{ steps.build.outputs.image }}
+      run: cosign sign --yes "$IMAGE@$DIGEST"
+
+    - name: Attach SLSA provenance attestation
+      if: steps.build.outputs.pushed == 'true'
+      uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373  # v4.1.1
+      with:
+        subject-name: ${{ steps.build.outputs.image }}
+        subject-digest: ${{ steps.build.outputs.digest }}
+        push-to-registry: true
+
     - name: Run Trivy security scan
       uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25  # v0.36.0
       with:

.github/workflows/dev-pypi.yml

@@ -80,7 +80,7 @@ jobs:
           echo "VERSION=${{ needs.config.outputs.package-version }}"
           echo "TARGET=testpypi"
         } >> "$GITHUB_ENV"
-        echo "Publishing to: TestPyPI with version: $VERSION"
+        echo "Publishing to: TestPyPI with version: ${{ needs.config.outputs.package-version }}"
 
     - name: Setup Python and UV
       uses: ./.github/actions/setup-uv-cached
@@ -156,6 +156,7 @@ jobs:
         name: reports-sbom-${{ needs.config.outputs.package-version }}
         path: |
           python-sbom-cyclonedx.json
+          python-sbom.json
           python-sbom-spdx.json
         retention-days: 180
 

.github/workflows/prod-release.yml

@@ -95,6 +95,10 @@ jobs:
       contents: read
       packages: write
       security-events: write
+      # Required for keyless cosign signing (ambient OIDC token) and
+      # actions/attest-build-provenance.
+      id-token: write
+      attestations: write
 
     steps:
     - name: Checkout code
@@ -119,6 +123,7 @@ jobs:
         path: dist/
 
     - name: Build and push container image
+      id: build
       env:
         REGISTRY: ${{ needs.config.outputs.container-registry }}
         IMAGE_NAME: ${{ needs.config.outputs.container-image }}
@@ -134,20 +139,78 @@ jobs:
           --build-arg "VCS_REF=$(git rev-parse --short HEAD)" \
           --build-arg "PYTHON_VERSION=$PYTHON_VERSION" \
           --build-arg "PACKAGE_NAME_SHORT=$PACKAGE_NAME_SHORT" \
+          --provenance=true \
+          --sbom=true \
+          --metadata-file metadata.json \
           --cache-from type=gha \
           --cache-to type=gha,mode=max \
           --push \
           -t "$REGISTRY/$IMAGE_NAME:$VERSION-python$PYTHON_VERSION" \
           .
 
+        # Extract the immutable digest from the BuildKit metadata output.
+        # containerimage.digest is the multi-arch index digest (sha256:…).
+        DIGEST=$(jq -r '."containerimage.digest"' metadata.json)
+        echo "digest=${DIGEST}" >> "$GITHUB_OUTPUT"
+        echo "image=${REGISTRY}/${IMAGE_NAME}" >> "$GITHUB_OUTPUT"
+
+        # Trivy cannot scan a multi-arch manifest — pull back a single-arch image.
+        # Use the first entry of build-platforms (same approach as dev-containers.yml).
+        SCAN_PLATFORM=$(echo "$BUILD_PLATFORMS" | cut -d, -f1)
+        docker pull --platform "$SCAN_PLATFORM" "$REGISTRY/$IMAGE_NAME:$VERSION-python$PYTHON_VERSION"
+
+    - name: Install cosign
+      uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6  # v4.1.2
+
+    - name: Sign container image (keyless / OIDC)
+      # Signs the immutable digest — not a mutable tag — using the GitHub OIDC
+      # ambient credential.  The Rekor transparency-log entry records the
+      # workflow identity, giving consumers a verifiable supply-chain link.
+      env:
+        DIGEST: ${{ steps.build.outputs.digest }}
+        IMAGE: ${{ steps.build.outputs.image }}
+      run: cosign sign --yes "$IMAGE@$DIGEST"
+
+    - name: Attach SLSA provenance attestation
+      # Generates a SLSA Build L2-conformant provenance attestation signed with
+      # the GitHub OIDC token and pushes it to the registry alongside the image.
+      # Complements the cosign signature: signature proves who signed; provenance
+      # records exactly what was built and how (workflow, inputs, commit SHA).
+      uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373  # v4.1.1
+      with:
+        subject-name: ${{ steps.build.outputs.image }}
+        subject-digest: ${{ steps.build.outputs.digest }}
+        push-to-registry: true
+
+    - name: Run Trivy security scan
+      uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25  # v0.36.0
+      with:
+        image-ref: "${{ needs.config.outputs.container-registry }}/${{ needs.config.outputs.container-image }}:${{ needs.config.outputs.package-version }}-python${{ matrix.python-version }}"
+        format: 'sarif'
+        output: 'trivy-results-py${{ matrix.python-version }}.sarif'
+        exit-code: '0'
+        severity: ${{ env.SCAN_LEVEL == 'full' && 'CRITICAL,HIGH,MEDIUM' || 'CRITICAL,HIGH' }}
+
+    - name: Upload Trivy scan results
+      uses: github/codeql-action/upload-sarif@99df26d4f13ea111d4ec1a7dddef6063f76b97e9  # v4.37.0
+      if: always()
+      with:
+        sarif_file: 'trivy-results-py${{ matrix.python-version }}.sarif'
+
   prod-manifests:
     name: Create Production Manifests
     needs: [config, prod-containers]
     runs-on: ubuntu-latest
     timeout-minutes: 15
+    # Only repoint `latest` / version manifests on a real GitHub Release.
+    # Manual workflow_dispatch builds images for testing but must not move prod tags.
+    if: github.event_name == 'release'
     permissions:
       contents: read
       packages: write
+      # Required for keyless cosign signing of the manifest-list digests.
+      id-token: write
+      attestations: write
 
     steps:
     - name: Checkout code
@@ -160,7 +223,11 @@ jobs:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
 
+    - name: Install cosign
+      uses: sigstore/cosign-installer@6f9f17788090df1f26f669e9d70d6ae9567deba6  # v4.1.2
+
     - name: Create production tag manifests
+      id: manifests
       shell: bash
       env:
         REGISTRY: ${{ needs.config.outputs.container-registry }}
@@ -183,6 +250,9 @@ jobs:
           fi
         done < <(make get-container-tags-env)
 
+        # Collect all created manifest-list digests so the signing step can sign them.
+        MANIFEST_REFS=""
+
         # Create primary tags (latest, version)
         if [[ -n "${primary_tags:-}" ]]; then
           IFS=',' read -ra TAGS <<< "$primary_tags"
@@ -191,6 +261,9 @@ jobs:
             docker buildx imagetools create \
               -t "$REGISTRY/$IMAGE:$tag" \
               "$REGISTRY/$IMAGE:$VERSION-python$DEFAULT_PYTHON"
+            # Resolve the digest of the newly-created manifest list.
+            DIGEST=$(docker buildx imagetools inspect --format '{{json .Manifest}}' "$REGISTRY/$IMAGE:$tag" | jq -r '.digest')
+            MANIFEST_REFS="${MANIFEST_REFS} $REGISTRY/$IMAGE@$DIGEST"
           done
         fi
 
@@ -203,14 +276,35 @@ jobs:
             docker buildx imagetools create \
               -t "$REGISTRY/$IMAGE:$tag" \
               "$REGISTRY/$IMAGE:$VERSION-python$py_ver"
+            DIGEST=$(docker buildx imagetools inspect --format '{{json .Manifest}}' "$REGISTRY/$IMAGE:$tag" | jq -r '.digest')
+            MANIFEST_REFS="${MANIFEST_REFS} $REGISTRY/$IMAGE@$DIGEST"
           done
         fi
 
+        # Deduplicate refs (multiple tags may resolve to the same digest).
+        UNIQUE_REFS=$(echo "$MANIFEST_REFS" | tr ' ' '\n' | sort -u | tr '\n' ' ')
+        echo "manifest-refs=${UNIQUE_REFS}" >> "$GITHUB_OUTPUT"
+
+    - name: Sign manifest-list images (keyless / OIDC)
+      # The per-python images were already signed in prod-containers; signing the
+      # manifest lists (latest, version, pythonX.Y tags) closes the chain so
+      # consumers of the convenience tags can also verify a signature.
+      env:
+        MANIFEST_REFS: ${{ steps.manifests.outputs.manifest-refs }}
+      run: |
+        for ref in $MANIFEST_REFS; do
+          echo "Signing manifest list: $ref"
+          cosign sign --yes "$ref"
+        done
+
   prod-pypi:
     name: Publish to PyPI
     needs: [config, build-wheel]
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    # Only publish to production PyPI on a real GitHub Release.
+    # Manual dispatch must not push a dev-build version to prod PyPI.
+    if: github.event_name == 'release'
     environment: pypi
     permissions:
       contents: read
@@ -309,8 +403,14 @@ jobs:
     needs: [config, build-wheel, prod-containers, prod-pypi]
     runs-on: ubuntu-latest
     timeout-minutes: 30
+    # Only upload assets and create gh release upload on a real GitHub Release.
+    # Manual dispatch would compute a dev version tag (1.x.y.devN) and fail on
+    # `gh release upload` because that tag doesn't exist as a release.
+    if: github.event_name == 'release'
     permissions:
       contents: write
+      id-token: write    # required by actions/attest-build-provenance (Sigstore OIDC)
+      attestations: write  # required to persist attestation to GitHub's attestations store
 
     steps:
     - name: Checkout code
@@ -331,15 +431,42 @@ jobs:
     - name: Generate SBOM
       run: make sbom-generate
 
+    - name: Attest build provenance for release artifacts
+      # Generates a Sigstore in-toto provenance attestation for the wheel and
+      # sdist.  The bundle is also uploaded to GitHub's attestations API so it
+      # is queryable via `gh attestation verify`.  We additionally attach it as
+      # a release asset (see next step) so OpenSSF Scorecard's Signed-Releases
+      # check — which scans release assets for .intoto.jsonl files — can
+      # discover it and score the release at 10/10.
+      #
+      # SHA pinned to actions/attest-build-provenance v4.1.1.
+      # Verify: https://github.com/actions/attest-build-provenance/releases/tag/v4.1.1
+      id: attest-artifacts
+      uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373  # v4.1.1
+      with:
+        subject-path: 'dist/*.whl, dist/*.tar.gz'
+
     - name: Upload release assets
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       run: |
         VERSION="${{ needs.config.outputs.package-version }}"
         RELEASE_TAG="v$VERSION"
+
+        # Upload wheel and sdist
         gh release upload "$RELEASE_TAG" dist/*.whl dist/*.tar.gz --clobber
+
+        # Upload SBOM files if present
         for f in python-sbom-cyclonedx.json python-sbom.json; do
           if [[ -f "$f" ]]; then
             gh release upload "$RELEASE_TAG" "$f" --clobber
           fi
         done
+
+        # Upload the Sigstore attestation bundle as a .intoto.jsonl release asset
+        # so OpenSSF Scorecard's Signed-Releases check (which looks for this
+        # extension on release assets) can detect provenance and score 10/10.
+        BUNDLE_SRC="${{ steps.attest-artifacts.outputs.bundle-path }}"
+        BUNDLE_DEST="attestation-v${VERSION}.intoto.jsonl"
+        cp "$BUNDLE_SRC" "$BUNDLE_DEST"
+        gh release upload "$RELEASE_TAG" "$BUNDLE_DEST" --clobber

.github/workflows/semantic-release.yml

@@ -85,7 +85,8 @@ jobs:
     permissions:
       contents: write
       packages: write
-      id-token: write
+      id-token: write      # required by actions/attest-build-provenance (Sigstore OIDC)
+      attestations: write  # required to persist attestation to GitHub's attestations store
 
     steps:
       - name: Checkout code
@@ -144,9 +145,47 @@ jobs:
           github_token: ${{ steps.cicd.outputs.token }}
           tag: ${{ steps.release.outputs.tag }}
 
+      - name: Attest build provenance for release artifacts
+        # python-semantic-release writes wheel + sdist to dist/ before publish-action
+        # uploads them.  We attest the same dist/* files here so every release asset
+        # has verifiable provenance.  The bundle is persisted to GitHub's attestations
+        # API AND uploaded as a .intoto.jsonl release asset so OpenSSF Scorecard's
+        # Signed-Releases check — which scans release assets for this extension —
+        # can detect it and score the release at 10/10.
+        #
+        # SHA pinned to actions/attest-build-provenance v4.1.1.
+        # Verify: https://github.com/actions/attest-build-provenance/releases/tag/v4.1.1
+        id: attest-artifacts
+        if: steps.release.outputs.released == 'true'
+        uses: actions/attest-build-provenance@0f67c3f4856b2e3261c31976d6725780e5e4c373  # v4.1.1
+        with:
+          subject-path: 'dist/*.whl, dist/*.tar.gz'
+
+      - name: Upload provenance bundle as release asset
+        # Attach the Sigstore attestation bundle with a .intoto.jsonl suffix so
+        # OpenSSF Scorecard's releasesHaveProvenance probe can discover it.
+        if: steps.release.outputs.released == 'true'
+        env:
+          GH_TOKEN: ${{ steps.cicd.outputs.token }}
+        run: |
+          TAG="${{ steps.release.outputs.tag }}"
+          VERSION="${{ steps.release.outputs.version }}"
+          BUNDLE_SRC="${{ steps.attest-artifacts.outputs.bundle-path }}"
+          BUNDLE_DEST="attestation-v${VERSION}.intoto.jsonl"
+          cp "$BUNDLE_SRC" "$BUNDLE_DEST"
+          gh release upload "$TAG" "$BUNDLE_DEST" --clobber
+
       - name: Install ORB
         if: steps.release.outputs.released == 'true'
-        run: uv sync --all-extras --group ci
+        # This step only boots the server to export the OpenAPI spec, so it needs
+        # the runtime `all` extra (api + providers + ui) but NOT the ci/dev test
+        # tooling.  --no-dev drops the default `dev` group; without it uv would
+        # reject `all` (which now includes `ui`) via the ui↔dev conflict.
+        # ORB_SKIP_UI_BUILD so building the wheel here doesn't invoke the
+        # (bun-less) SPA build hook.
+        env:
+          ORB_SKIP_UI_BUILD: "1"
+        run: uv sync --extra all --no-dev
 
       - name: Export OpenAPI spec for Go SDK
         if: steps.release.outputs.released == 'true'

Dockerfile

@@ -52,10 +52,17 @@ RUN pip install --no-cache-dir uv==0.8.12 \
     && uv venv /opt/venv
 ENV PATH="/opt/venv/bin:$PATH"
 
-# Copy pre-built wheel and install dependencies
+# Copy pre-built wheel and install with all runtime extras.
+#
+# [all] = cli + api (fastapi/uvicorn) + sql + monitoring + all-providers
+#         (aws, k8s) + ui (reflex).  The server needs api to serve, and the
+#         default UIConfig (enabled=True, mode="embedded") makes the entrypoint's
+#         `orb server start --foreground` import orb.ui.app, which needs reflex.
+# The ci/dev test tooling groups are not installed — this is a production image.
 COPY dist/*.whl /tmp/
-RUN uv pip install --no-cache /tmp/*.whl \
-    && rm -rf /tmp/*.whl
+RUN WHEEL=$(ls /tmp/*.whl) \
+    && uv pip install --no-cache "${WHEEL}[all]" \
+    && rm -f /tmp/*.whl
 
 # Copy only runtime files needed
 COPY config/ ./config/