pr-craft-k8s-tests.yml

# Runs the Craft Kubernetes integration tests against a Helm-installed kind
# cluster with the real API, Celery workers, sandbox proxy, and sandbox pods.
#
# Each test file owns a module-scoped sandbox pod and the single-node kind
# cluster fits one sandbox pod at a time, so the lane shards per test file, each
# shard on its own kind cluster (dynamic `fail-fast: false` matrix).
#
# Images are built once and pushed to the shared ECR repo, then each shard
# mirrors them into a local `localhost:5001` registry that kind pulls from
# unauthenticated — sandbox pods have no `imagePullSecrets`, so the image must
# come from an unauthenticated registry. (kind pulls from the local registry
# rather than `kind load`, which writes a multi-GB tar to tmpfs /tmp.)
# https://kind.sigs.k8s.io/docs/user/local-registry/
name: Craft Kubernetes Integration Tests
concurrency:
  group: Craft-K8s-Tests-${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: true

on:
  schedule:
    - cron: "0 7 * * *"   # 07:00 UTC nightly
  merge_group:
  pull_request:
    branches: [main]
    # NOTE: Intentionally no `paths:` filter. Trigger-level `paths:` is ignored
    # for `merge_group`, so we always trigger and let the `changes` job below
    # decide whether the real test job runs (matches the docker-compose lane
    # cadence; this lane is expensive and must not run on every PR).
  push:
    tags:
      - "v*.*.*"
  workflow_dispatch:

permissions:
  contents: read

env:
  S3_ENDPOINT_URL: "http://127.0.0.1:9004"
  S3_FILE_STORE_BUCKET_NAME: "onyx-file-store-bucket"
  S3_AWS_ACCESS_KEY_ID: "minioadmin"
  S3_AWS_SECRET_ACCESS_KEY: "minioadmin"

  SANDBOX_BACKEND: "kubernetes"
  SANDBOX_NAMESPACE: "onyx-sandboxes"
  SANDBOX_SERVICE_ACCOUNT_NAME: "sandbox"
  SANDBOX_CONTAINER_IMAGE: "localhost:5001/onyx-sandbox:ci"
  # Sandbox pod resource requests live in values-ci.yaml (configMap.SANDBOX_POD_*).
  ONYX_SERVER_URL: "http://onyx-api-service.onyx.svc.cluster.local:8080"

  # Proxy lives in release namespace `onyx`. SANDBOX_PROXY_HOST is set
  # later to the proxy Service ClusterIP — the runner has no cluster DNS,
  # so a Service FQDN here would fail _resolve_proxy_ip in the manager.
  HELM_RELEASE_NAMESPACE: "onyx"
  SANDBOX_PROXY_PORT: "8080"
  SANDBOX_PROXY_CA_SECRET: "sandbox-proxy-ca"
  SANDBOX_PROXY_CA_CONFIGMAP: "sandbox-proxy-ca-bundle"
  BACKEND_IMAGE: "localhost:5001/onyx-backend:ci"
  # Fail fast on opencode hangs (default is 900s) so a stuck turn surfaces logs
  # well within the job timeout instead of being killed with zero signal.
  SANDBOX_TURN_TIMEOUT_SECONDS: "120"
  KIND_REGISTRY_NAME: "kind-registry"
  KIND_REGISTRY_PORT: "5001"
  KIND_VERSION: "v0.31.0"
  KUBECTL_VERSION: "v1.35.0"
  CRAFT_ASSETS_ARTIFACT: "craft-k8s-assets"

  # The pytest process runs on the runner, so it reaches chart-managed
  # Postgres/Redis through kubectl port-forwards.
  POSTGRES_HOST: "127.0.0.1"
  POSTGRES_PORT: "5432"
  POSTGRES_USER: "postgres"
  POSTGRES_PASSWORD: "password"
  POSTGRES_DB: "postgres"
  REDIS_HOST: "127.0.0.1"
  REDIS_PORT: "6379"
  REDIS_PASSWORD: "password"
  API_SERVER_HOST: "127.0.0.1"
  API_SERVER_PORT: "8080"

  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

jobs:
  changes:
    # Gates the expensive test job. On pull_request / merge_group we use
    # paths-filter; on schedule / push (tags) / workflow_dispatch the filter is
    # skipped and the output defaults to `true` so the lane runs.
    runs-on: ubuntu-latest
    timeout-minutes: 5
    permissions:
      contents: read
      pull-requests: read
    outputs:
      craft_k8s: ${{ steps.filter.outputs.craft_k8s || 'true' }}
    steps:
      - name: Checkout code
        if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
        with:
          persist-credentials: false
      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
        id: filter
        if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
        with:
          filters: |
            craft_k8s:
              - 'backend/Dockerfile'
              - 'backend/onyx/sandbox_proxy/**'
              - 'backend/onyx/server/features/build/**'
              - 'backend/onyx/server/features/skill/**'
              - 'backend/onyx/skills/**'
              - 'backend/shared_configs/**'
              # Only the sandbox/craft-specific chart bits gate this lane. Generic
              # chart changes (workers, deps, version bumps) are install-tested by
              # the `Helm - Lint and Test Charts` lane (`ct install` into kind), so
              # they don't need the full sandbox suite here.
              - 'deployment/helm/charts/onyx/templates/sandbox-*'
              - 'deployment/helm/charts/onyx/templates/sandbox-proxy/**'
              - 'deployment/helm/charts/onyx/templates/network-policy-sandbox-*'
              - 'deployment/helm/charts/onyx/templates/craft-*'
              - 'deployment/helm/charts/onyx/templates/celery-worker-scheduled-tasks*'
              - 'deployment/helm/charts/onyx/charts/code-interpreter-*'
              - 'deployment/helm/charts/onyx/values-ci.yaml'
              - 'backend/tests/integration/tests/craft/*.py'
              - 'backend/tests/integration/tests/craft/k8s/**'
              - 'backend/tests/common/craft/**'
              - 'backend/tests/integration/conftest.py'
              - 'backend/tests/integration/common_utils/**'
              - '.github/workflows/pr-craft-k8s-tests.yml'
              - '.github/actions/setup-python-and-install-dependencies/**'
              - '.github/actions/setup-test-license/**'
              - '.github/actions/login-ecr-pullthrough-cache/**'

  discover-test-files:
    # One shard per craft k8s test file (module-scoped pool fixtures keep a file on one shard).
    needs: changes
    if: needs.changes.outputs.craft_k8s == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      test-files: ${{ steps.set-matrix.outputs.test-files }}
    steps:
      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Discover craft k8s test files
        id: set-matrix
        run: |
          set -eo pipefail
          # `name` = log-friendly shard id; `path` = repo-root-relative pytest target.
          entries=""
          for f in $(find backend/tests/integration/tests/craft/k8s \
              -maxdepth 1 -name 'test_*.py' -type f | sort); do
            base=$(basename "$f" .py)
            shard="${base#test_}"
            entries="${entries}{\"path\":\"${f}\",\"name\":\"${shard}\"},"
          done
          if [ -z "${entries}" ]; then
            echo "::error::no craft k8s test files discovered"
            exit 1
          fi
          echo "test-files=[${entries%,}]" >> "$GITHUB_OUTPUT"

  prepare-craft-assets:
    name: Prepare Craft test assets
    needs: changes
    if: needs.changes.outputs.craft_k8s == 'true'
    runs-on:
      - runs-on
      - runner=2cpu-linux-x64
      - spot=false
      - ${{ format('run-id={0}-craft-k8s-tools', github.run_id) }}
    timeout-minutes: 10
    permissions:
      contents: read
    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Download and verify kind tools
        run: |
          set -euo pipefail

          download() {
            curl --fail --location --silent --show-error \
              --retry 5 --retry-delay 2 --retry-all-errors \
              --output "$2" "$1"
          }

          cache_dir="${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64"
          kind_dir="${cache_dir}/kind/bin"
          kubectl_dir="${cache_dir}/kubectl/bin"
          mkdir -p "${kind_dir}" "${kubectl_dir}"

          kind_filename="kind-linux-amd64"
          kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}"
          download "${kind_url}/${kind_filename}" "${kind_dir}/${kind_filename}"
          download "${kind_url}/${kind_filename}.sha256sum" "${kind_dir}/${kind_filename}.sha256sum"
          (
            cd "${kind_dir}"
            grep "${kind_filename}" "${kind_filename}.sha256sum" | sha256sum --check -
            mv "${kind_filename}" kind
            rm "${kind_filename}.sha256sum"
            chmod +x kind
          )

          kubectl_url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64"
          download "${kubectl_url}/kubectl" "${kubectl_dir}/kubectl"
          download "${kubectl_url}/kubectl.sha256" "${kubectl_dir}/kubectl.sha256"
          (
            cd "${kubectl_dir}"
            echo "$(cat kubectl.sha256)  kubectl" | sha256sum --check -
            rm kubectl.sha256
            chmod +x kubectl
          )

      - name: Download Helm chart dependencies
        run: |
          set -euo pipefail

          retry() {
            local attempt
            for attempt in 1 2 3 4 5; do
              if "$@"; then
                return 0
              fi
              if [ "${attempt}" -eq 5 ]; then
                return 1
              fi
              echo "Command failed (attempt ${attempt}/5); retrying ..."
              sleep $((attempt * 5))
            done
          }

          retry helm repo add --force-update ingress-nginx https://kubernetes.github.io/ingress-nginx
          retry helm repo add --force-update opensearch https://opensearch-project.github.io/helm-charts
          retry helm repo add --force-update cloudnative-pg https://cloudnative-pg.github.io/charts
          retry helm repo add --force-update ot-container-kit https://ot-container-kit.github.io/helm-charts
          retry helm repo add --force-update minio https://charts.min.io/
          retry helm repo add --force-update code-interpreter https://onyx-dot-app.github.io/python-sandbox/
          retry helm repo update
          if ! retry helm dependency build --skip-refresh deployment/helm/charts/onyx; then
            echo "helm dependency build failed; pulling disabled code-interpreter dependency directly"
            code_interpreter_version=$(awk '
              $1 == "-" && $2 == "name:" && $3 == "code-interpreter" { found = 1 }
              found && $1 == "version:" { print $2; exit }
            ' deployment/helm/charts/onyx/Chart.yaml)
            test -n "${code_interpreter_version}"
            retry helm pull code-interpreter/code-interpreter \
              --version "${code_interpreter_version}" \
              --destination deployment/helm/charts/onyx/charts
          fi
          helm dependency list deployment/helm/charts/onyx
          helm dependency list deployment/helm/charts/onyx \
            | awk 'NR > 1 && NF && $4 != "ok" { bad = 1 } END { exit bad }'

      # Tar preserves executable permissions for kind and kubectl.
      - name: Package Craft test assets
        run: |
          set -euo pipefail
          assets_dir="${RUNNER_TEMP}/craft-assets"
          mkdir -p "${assets_dir}"
          tar -C "${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64" \
            -czf "${assets_dir}/kind-tools.tar.gz" .
          tar -C deployment/helm/charts/onyx/charts \
            -czf "${assets_dir}/helm-dependencies.tar.gz" .

      - name: Upload Craft test assets
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: ${{ env.CRAFT_ASSETS_ARTIFACT }}
          path: ${{ runner.temp }}/craft-assets
          if-no-files-found: error
          retention-days: 1
          compression-level: 0
          overwrite: true

  build-images:
    # Build both images in parallel (one matrix leg each) and push to the shared
    # ECR repo so each test shard pulls prebuilt images instead of cold-building.
    name: build-image (${{ matrix.image }})
    needs: changes
    if: needs.changes.outputs.craft_k8s == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          - image: sandbox
            context: ./backend/onyx/server/features/build/sandbox/image
            file: ./backend/onyx/server/features/build/sandbox/image/Dockerfile
            # CI tests don't exercise skill runtime deps (soffice/pdftoppm/pptxgenjs),
            # so skip LibreOffice et al. to keep the CI image lean. Browser stays
            # on (ENABLE_BROWSER defaults true) — chromium is lighter and the
            # browser integration test needs it.
            extra_build_args: "ENABLE_SKILLS=false"
            target: ""
          - image: backend
            context: ./backend
            file: ./backend/Dockerfile
            extra_build_args: ""
            # The production image; the backend Dockerfile's default (last) stage is the
            # dev variant.
            target: runtime
    runs-on:
      - runs-on
      - runner=8cpu-linux-x64
      - spot=false
      - volume=100gb
      - ${{ format('run-id={0}-craft-k8s-build-{1}', github.run_id, matrix.image) }}
      - extras=ecr-cache
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Log in to ECR pull-through cache
        uses: ./.github/actions/login-ecr-pullthrough-cache
        with:
          ecr-registry: ${{ vars.ECR_REGISTRY }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4

      - name: Build and push ${{ matrix.image }} image
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
        with:
          context: ${{ matrix.context }}
          file: ${{ matrix.file }}
          # Empty target means the Dockerfile's default (last) stage.
          target: ${{ matrix.target }}
          platforms: linux/amd64
          build-args: |
            BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}
            ${{ matrix.extra_build_args }}
          tags: ${{ env.RUNS_ON_ECR_CACHE }}:craft-k8s-${{ matrix.image }}-${{ github.run_id }}
          push: true
          # Attestations attach as ECR referrers to the image digest, which is
          # stable across runs and caps out at 100 per subject.
          provenance: false
          sbom: false
          cache-from: type=gha,scope=craft-${{ matrix.image }}
          cache-to: type=gha,scope=craft-${{ matrix.image }},mode=max

  craft-k8s-tests:
    name: craft-k8s (${{ matrix.test-file.name }})
    needs: [changes, discover-test-files, prepare-craft-assets, build-images]
    if: needs.changes.outputs.craft_k8s == 'true'
    # spot=false: this is a long lane (full kind cluster per shard); on-demand
    # avoids mid-run spot reclamation. Matches the compose lane.
    runs-on:
      - runs-on
      - runner=8cpu-linux-x64
      - spot=false
      - volume=100gb
      - ${{ format('run-id={0}-craft-k8s-tests-{1}', github.run_id, matrix.test-file.name) }}
      - extras=ecr-cache
    timeout-minutes: 40

    strategy:
      # fail-fast off so one shard's failure doesn't cancel the others.
      fail-fast: false
      matrix:
        test-file: ${{ fromJson(needs.discover-test-files.outputs.test-files) }}

    # id-token: OIDC for the setup-test-license step.
    permissions:
      contents: read
      id-token: write

    env:
      PYTHONPATH: ./backend
      MODEL_SERVER_HOST: "disabled"
      DISABLE_TELEMETRY: "true"
      DISABLE_VECTOR_DB: "false"

    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Setup Python and Install Dependencies
        uses: ./.github/actions/setup-python-and-install-dependencies
        with:
          requirements: |
            backend/requirements/default.txt
            backend/requirements/dev.txt
            backend/requirements/ee.txt

      - name: Download Craft test assets
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
        with:
          name: ${{ env.CRAFT_ASSETS_ARTIFACT }}
          path: ${{ runner.temp }}/craft-assets

      - name: Install Craft test assets
        run: |
          set -euo pipefail
          kind_cache_dir="${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64"
          charts_dir="deployment/helm/charts/onyx/charts"
          mkdir -p "${kind_cache_dir}" "${charts_dir}"
          tar -C "${kind_cache_dir}" \
            -xzf "${RUNNER_TEMP}/craft-assets/kind-tools.tar.gz"
          tar -C "${charts_dir}" \
            -xzf "${RUNNER_TEMP}/craft-assets/helm-dependencies.tar.gz"

      - name: Log in to ECR pull-through cache
        uses: ./.github/actions/login-ecr-pullthrough-cache
        with:
          ecr-registry: ${{ vars.ECR_REGISTRY }}

      - name: Start local image registry
        run: |
          docker run -d --restart=always \
            -p "127.0.0.1:${KIND_REGISTRY_PORT}:5000" \
            --name "${KIND_REGISTRY_NAME}" \
            "${BASE_IMAGE_REGISTRY:-docker.io}/library/registry:2"

      # Re-push the prebuilt ECR images into this shard's local registry under the
      # tags kind pulls (unauthenticated); the blob copy is cheap vs a cold build.
      - name: Mirror prebuilt images into local registry
        env:
          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
          RUN_ID: ${{ github.run_id }}
        run: |
          set -eo pipefail
          mirror() {
            local src="$1" dst="$2"
            docker pull "$src"
            docker tag "$src" "$dst"
            docker push "$dst"
          }
          mirror "${ECR_CACHE}:craft-k8s-sandbox-${RUN_ID}" "${SANDBOX_CONTAINER_IMAGE}"
          mirror "${ECR_CACHE}:craft-k8s-backend-${RUN_ID}" "${BACKEND_IMAGE}"

      - name: Write kind config
        run: |
          cat > "${RUNNER_TEMP}/kind-config.yaml" <<'EOF'
          kind: Cluster
          apiVersion: kind.x-k8s.io/v1alpha4
          containerdConfigPatches:
            - |-
              [plugins."io.containerd.grpc.v1.cri".registry]
                config_path = "/etc/containerd/certs.d"
          EOF

      - name: Create kind cluster
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # ratchet:helm/kind-action@v1.14.0
        with:
          version: ${{ env.KIND_VERSION }}
          kubectl_version: ${{ env.KUBECTL_VERSION }}
          cluster_name: onyx-craft-ci
          node_image: kindest/node:v1.33.1
          config: ${{ runner.temp }}/kind-config.yaml

      - name: Connect local registry to kind network
        run: |
          docker network connect "kind" "${KIND_REGISTRY_NAME}" || true
          REGISTRY_DIR="/etc/containerd/certs.d/localhost:${KIND_REGISTRY_PORT}"
          docker exec onyx-craft-ci-control-plane mkdir -p "${REGISTRY_DIR}"
          docker exec onyx-craft-ci-control-plane bash -c "cat > ${REGISTRY_DIR}/hosts.toml <<EOF
          [host.\"http://${KIND_REGISTRY_NAME}:5000\"]
          EOF"

      # Pod spec sets nodeSelector onyx.app/workload=sandbox
      # (kubernetes_sandbox_manager.py). In prod this matches a dedicated
      # node pool; in kind we just label the single control-plane node.
      - name: Label kind node for sandbox workload
        run: kubectl label node onyx-craft-ci-control-plane onyx.app/workload=sandbox

      - name: Validate Helm chart dependencies
        run: |
          helm dependency list deployment/helm/charts/onyx
          helm dependency list deployment/helm/charts/onyx \
            | awk 'NR > 1 && NF && $4 != "ok" { bad = 1 } END { exit bad }'

      - name: Generate sandbox push key
        run: |
          python - <<'PY' >> "$GITHUB_ENV"
          import base64
          from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
          from cryptography.hazmat.primitives.serialization import Encoding
          from cryptography.hazmat.primitives.serialization import NoEncryption
          from cryptography.hazmat.primitives.serialization import PrivateFormat

          key = Ed25519PrivateKey.generate()
          raw = key.private_bytes(
              encoding=Encoding.Raw,
              format=PrivateFormat.Raw,
              encryption_algorithm=NoEncryption(),
          )
          print(f"ONYX_SANDBOX_PUSH_PRIVATE_KEY={base64.b64encode(raw).decode('ascii')}")
          PY

      - name: Install Onyx chart into kind
        run: |
          set -eo pipefail
          kubectl create namespace "${HELM_RELEASE_NAMESPACE}" \
            --dry-run=client -o yaml | kubectl apply -f -
          kubectl config set-context --current --namespace="${HELM_RELEASE_NAMESPACE}"

          for attempt in 1 2 3; do
            if helm upgrade --install onyx deployment/helm/charts/onyx \
              -n "${HELM_RELEASE_NAMESPACE}" \
              -f deployment/helm/charts/onyx/values-ci.yaml \
              --set "api.image.repository=localhost:5001/onyx-backend" \
              --set "api.image.tag=ci" \
              --set "celery_shared.image.repository=localhost:5001/onyx-backend" \
              --set "celery_shared.image.tag=ci" \
              --set-string "auth.sandboxPushSecret.values.private_key=${ONYX_SANDBOX_PUSH_PRIVATE_KEY}" \
              --timeout 10m; then
              exit 0
            fi
            if [ "$attempt" -lt 3 ]; then
              echo "helm install failed (attempt ${attempt}/3); waiting before retry ..."
              kubectl -n "${HELM_RELEASE_NAMESPACE}" get pods || true
              sleep 20
            fi
          done

          helm status onyx -n "${HELM_RELEASE_NAMESPACE}" || true
          kubectl get pods -A || true
          exit 1

      - name: Wait for chart Postgres ready
        run: |
          set -eo pipefail
          kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
            deployment/onyx-cloudnative-pg --timeout=180s
          # Don't add a `pod -l cnpg.io/cluster` wait — it also matches the
          # transient initdb pod (deleted mid-wait → NotFound).
          kubectl -n "${HELM_RELEASE_NAMESPACE}" wait \
            --for=condition=Ready cluster/onyx-pg \
            --timeout=300s

      - name: Forward chart Postgres to runner
        run: |
          set -eo pipefail
          # setsid survives the step-boundary process-group cleanup; the loop
          # survives kubectl port-forward's mid-run drops.
          setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-pg-rw \"$POSTGRES_PORT\":5432; sleep 1; done" \
            > "${RUNNER_TEMP}/postgres-port-forward.log" 2>&1 &
          echo "$!" > "${RUNNER_TEMP}/postgres-port-forward.pid"
          for _ in $(seq 1 60); do
            if python -c 'import os, psycopg2; conn = psycopg2.connect(dbname=os.environ["POSTGRES_DB"], user=os.environ["POSTGRES_USER"], password=os.environ["POSTGRES_PASSWORD"], host=os.environ["POSTGRES_HOST"], port=os.environ["POSTGRES_PORT"], connect_timeout=2); conn.close()'; then
              echo "chart Postgres is reachable at ${POSTGRES_HOST}:${POSTGRES_PORT}"
              exit 0
            fi
            sleep 2
          done
          cat "${RUNNER_TEMP}/postgres-port-forward.log" || true
          exit 1

      - name: Wait for API startup migrations
        run: |
          kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
            deployment/onyx-api-server --timeout=360s
          kubectl -n "${HELM_RELEASE_NAMESPACE}" exec deployment/onyx-api-server -- \
            alembic current --check-heads

      - name: Wait for chart OpenSearch ready
        run: |
          kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
            statefulset/onyx-opensearch-master --timeout=420s

      - name: Wait for chart Redis ready
        run: |
          set -eo pipefail
          kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
            deployment/redis-operator --timeout=180s
          for _ in $(seq 1 120); do
            endpoints=$(kubectl -n "${HELM_RELEASE_NAMESPACE}" get endpoints onyx \
              -o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null || true)
            if [ -n "${endpoints}" ]; then
              exit 0
            fi
            sleep 2
          done
          kubectl -n "${HELM_RELEASE_NAMESPACE}" get redis,pods,svc || true
          exit 1

      - name: Forward chart Redis to runner
        run: |
          set -eo pipefail
          setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx \"$REDIS_PORT\":6379; sleep 1; done" \
            > "${RUNNER_TEMP}/redis-port-forward.log" 2>&1 &
          echo "$!" > "${RUNNER_TEMP}/redis-port-forward.pid"
          for _ in $(seq 1 60); do
            if python -c 'import os; from redis import Redis; Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), password=os.environ["REDIS_PASSWORD"], socket_connect_timeout=2).ping()'; then
              echo "chart Redis is reachable at ${REDIS_HOST}:${REDIS_PORT}"
              exit 0
            fi
            sleep 2
          done
          cat "${RUNNER_TEMP}/redis-port-forward.log" || true
          exit 1

      - name: Wait for chart MinIO ready
        run: |
          kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
            deployment/onyx-minio --timeout=180s
          if kubectl -n "${HELM_RELEASE_NAMESPACE}" get job/onyx-minio-post-job >/dev/null 2>&1; then
            kubectl -n "${HELM_RELEASE_NAMESPACE}" wait \
              --for=condition=complete job/onyx-minio-post-job \
              --timeout=180s
          else
            echo "MinIO post-install hook job already completed and was deleted by Helm"
          fi

      - name: Forward chart MinIO to runner
        run: |
          set -eo pipefail
          setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-minio 9004:9000; sleep 1; done" \
            > "${RUNNER_TEMP}/minio-port-forward.log" 2>&1 &
          echo "$!" > "${RUNNER_TEMP}/minio-port-forward.pid"
          for _ in $(seq 1 30); do
            if curl -fsS "${S3_ENDPOINT_URL}/minio/health/ready"; then
              echo "chart MinIO is reachable at ${S3_ENDPOINT_URL}"
              exit 0
            fi
            sleep 1
          done
          cat "${RUNNER_TEMP}/minio-port-forward.log" || true
          exit 1

      - name: Wait for chart app runtime ready
        run: |
          set -eo pipefail
          for deployment in \
            onyx-sandbox-proxy \
            onyx-api-server \
            onyx-celery-beat \
            onyx-celery-worker-primary \
            onyx-celery-worker-light \
            onyx-celery-worker-heavy \
            onyx-celery-worker-scheduled-tasks; do
            kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
              "deployment/${deployment}" --timeout=360s
          done

      # Tier-gated EE routes (test_skill_push user-groups) need a license.
      - name: Fetch dev license
        uses: ./.github/actions/setup-test-license
        with:
          aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}

      - name: Seed dev license
        # The api-server can briefly restart if Postgres/cnpg flaps right after
        # rollout, so re-assert readiness and retry the exec.
        run: |
          set -eo pipefail
          for attempt in $(seq 1 5); do
            kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
              deployment/onyx-api-server --timeout=120s || true
            if kubectl -n "${HELM_RELEASE_NAMESPACE}" exec deployment/onyx-api-server -- \
                env ONYX_DEV_LICENSE="${ONYX_DEV_LICENSE}" python -m scripts.seed_dev_license; then
              echo "seeded dev license (attempt ${attempt})"
              exit 0
            fi
            echo "seed dev license failed (attempt ${attempt}); retrying ..."
            sleep 10
          done
          exit 1

      - name: Forward chart API to runner
        run: |
          set -eo pipefail
          setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-api-service \"$API_SERVER_PORT\":8080; sleep 1; done" \
            > "${RUNNER_TEMP}/api-port-forward.log" 2>&1 &
          echo "$!" > "${RUNNER_TEMP}/api-port-forward.pid"
          for _ in $(seq 1 60); do
            if curl -fsS "http://${API_SERVER_HOST}:${API_SERVER_PORT}/health"; then
              echo "chart API reachable at ${API_SERVER_HOST}:${API_SERVER_PORT}"
              exit 0
            fi
            sleep 2
          done
          cat "${RUNNER_TEMP}/api-port-forward.log" || true
          exit 1

      # The test process runs on the CI runner, which has no cluster DNS.
      # `socket.gethostbyname(IP)` returns the IP unchanged, so resolving
      # the Service ClusterIP here lets _resolve_proxy_ip in the manager
      # succeed without any DNS resolution. host_aliases on sandbox pods
      # then pins `sandbox-proxy` to this same IP.
      - name: Resolve proxy Service IP and export to env
        run: |
          PROXY_IP=$(kubectl -n "${HELM_RELEASE_NAMESPACE}" get svc \
            onyx-sandbox-proxy -o jsonpath='{.spec.clusterIP}')
          echo "SANDBOX_PROXY_HOST=${PROXY_IP}" >> "$GITHUB_ENV"
          echo "resolved proxy ClusterIP: ${PROXY_IP}"

      # Sandbox pods are torn down on failure before `kind export logs` runs,
      # so kubelet GCs their opencode-serve logs. Stream them to disk live (one
      # follower per pod, retrying the attach) so failures leave a trail.
      - name: Start sandbox pod log capture
        run: |
          mkdir -p sandbox-logs
          # The inner `bash -c` body is single-quoted on purpose: its $vars
          # expand at runtime inside that shell, not here.
          # shellcheck disable=SC2016
          nohup bash -c '
            while true; do
              for name in $(kubectl -n onyx-sandboxes get pods \
                  -o jsonpath="{.items[*].metadata.name}" 2>/dev/null); do
                if mkdir "sandbox-logs/.lock-${name}" 2>/dev/null; then
                  (
                    for _ in $(seq 1 120); do
                      kubectl -n onyx-sandboxes logs --all-containers=true \
                        --prefix=true -f "$name" \
                        >> "sandbox-logs/${name}.log" 2>&1 && break
                      kubectl -n onyx-sandboxes get pod "$name" \
                        >/dev/null 2>&1 || break
                      sleep 1
                    done
                  ) &
                fi
              done
              sleep 1
            done
          ' > sandbox-logs/_watcher.log 2>&1 &
          echo "$!" > "${RUNNER_TEMP}/sandbox-log-watcher.pid"
          disown || true
          echo "sandbox pod log capture started (pid $!)"

      - name: Run Craft K8s tests (${{ matrix.test-file.name }})
        shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
        run: |
          py.test \
            --durations=8 \
            -o junit_family=xunit2 \
            -v \
            --ff \
            -m "not nightly" \
            "${{ matrix.test-file.path }}"

      - name: Collect diagnostics on failure
        if: failure()
        run: |
          mkdir -p kind-logs

          echo "::group::sandbox pod logs"
          if compgen -G "sandbox-logs/*.log" > /dev/null; then
            for f in sandbox-logs/*.log; do echo "--- ${f} ---"; cat "${f}" || true; done
          else
            echo "no sandbox pod logs were captured"
          fi
          echo "::endgroup::"

          echo "::group::disk and docker status"
          df -h || true; docker system df || true; docker images || true; ls -lah /tmp || true
          echo "::endgroup::"

          echo "::group::MinIO diagnostics"
          kubectl -n "${HELM_RELEASE_NAMESPACE}" get pods,svc,job -l release=onyx || true
          kubectl -n "${HELM_RELEASE_NAMESPACE}" logs deployment/onyx-minio --tail=500 || true
          kubectl -n "${HELM_RELEASE_NAMESPACE}" logs job/onyx-minio-post-job --all-containers --tail=500 || true
          python - <<'PY' || true
          import os, boto3
          from botocore.config import Config
          s3 = boto3.client(
              "s3",
              endpoint_url=os.environ.get("S3_ENDPOINT_URL"),
              aws_access_key_id=os.environ.get("S3_AWS_ACCESS_KEY_ID"),
              aws_secret_access_key=os.environ.get("S3_AWS_SECRET_ACCESS_KEY"),
              region_name=os.environ.get("AWS_REGION_NAME") or "us-east-1",
              config=Config(s3={"addressing_style": "path"}),
          )
          print("buckets:", [b["Name"] for b in s3.list_buckets().get("Buckets", [])])
          for b in s3.list_buckets().get("Buckets", []):
              objs = s3.list_objects_v2(Bucket=b["Name"]).get("Contents", [])
              print(b["Name"], "->", [o["Key"] for o in objs][:20])
          PY
          echo "::endgroup::"

          echo "::group::chart resources and app logs"
          helm status onyx -n "${HELM_RELEASE_NAMESPACE}" || true
          for dep in onyx-api-server onyx-celery-worker-primary \
              onyx-celery-worker-heavy onyx-celery-worker-scheduled-tasks \
              onyx-cloudnative-pg redis-operator; do
            kubectl -n "${HELM_RELEASE_NAMESPACE}" logs "deployment/${dep}" --all-containers --tail=500 || true
          done
          kubectl -n "${HELM_RELEASE_NAMESPACE}" logs statefulset/onyx-opensearch-master --all-containers --tail=500 || true
          kubectl -n "${HELM_RELEASE_NAMESPACE}" get cluster,redis,statefulset,pods,svc,pvc || true
          echo "::endgroup::"

          echo "::group::port-forward logs"
          for f in postgres redis minio api; do
            echo "--- ${f} ---"; cat "${RUNNER_TEMP}/${f}-port-forward.log" 2>/dev/null || true
          done
          echo "::endgroup::"

          kind export logs ./kind-logs --name onyx-craft-ci || true
          kubectl get pods -A -o wide > kind-logs/pods.txt 2>&1 || true
          kubectl get svc -A -o wide > kind-logs/services.txt 2>&1 || true
          kubectl describe pods -n onyx-sandboxes > kind-logs/sandbox-pods-describe.txt 2>&1 || true
          kubectl describe pods -n "${HELM_RELEASE_NAMESPACE}" > kind-logs/release-ns-pods-describe.txt 2>&1 || true
          kubectl -n "${HELM_RELEASE_NAMESPACE}" logs -l app.kubernetes.io/component=sandbox-proxy --tail=500 > kind-logs/sandbox-proxy.log 2>&1 || true
          cp "${RUNNER_TEMP}"/*-port-forward.log kind-logs/ 2>/dev/null || true

      - name: Upload logs
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: craft-k8s-logs-${{ matrix.test-file.name }}
          path: |
            kind-logs/
            sandbox-logs/
          retention-days: 7

      - name: Cleanup kind cluster and port-forwards
        if: always()
        run: |
          set +e
          for pid_file in \
            "${RUNNER_TEMP}/postgres-port-forward.pid" \
            "${RUNNER_TEMP}/redis-port-forward.pid" \
            "${RUNNER_TEMP}/minio-port-forward.pid" \
            "${RUNNER_TEMP}/api-port-forward.pid" \
            "${RUNNER_TEMP}/sandbox-log-watcher.pid"; do
            if [ -f "${pid_file}" ]; then
              kill "$(cat "${pid_file}")" 2>/dev/null || true
            fi
          done
          kind delete cluster --name onyx-craft-ci || true
          docker network disconnect kind "${KIND_REGISTRY_NAME}" 2>/dev/null || true
          docker rm -f "${KIND_REGISTRY_NAME}" 2>/dev/null || true

  craft-k8s-tests-required:
    # Single required status check for the craft-k8s suite. Always runs so
    # branch protection has a stable target, and passes cleanly when `changes`
    # reports no relevant paths changed (i.e. the test job was legitimately
    # skipped).
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: [changes, discover-test-files, build-images, craft-k8s-tests]
    if: ${{ always() }}
    steps:
      - name: Check job status
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          RUN_TESTS: ${{ needs.changes.outputs.craft_k8s }}
          DISCOVER_RESULT: ${{ needs.discover-test-files.result }}
          BUILD_RESULT: ${{ needs.build-images.result }}
          TEST_RESULT: ${{ needs.craft-k8s-tests.result }}
        run: |
          # Fail closed if `changes` didn't succeed. Otherwise an empty
          # RUN_TESTS (which is what we'd see when `changes` failed/cancelled)
          # would be indistinguishable from "no relevant paths changed" and we
          # would incorrectly pass the required check.
          if [ "${CHANGES_RESULT}" != "success" ]; then
            echo "changes job did not succeed (result: ${CHANGES_RESULT})"
            exit 1
          fi
          if [ "${RUN_TESTS}" != "true" ]; then
            echo "No relevant paths changed -- required check passes."
            exit 0
          fi
          if [ "${DISCOVER_RESULT}" != "success" ] || [ "${BUILD_RESULT}" != "success" ]; then
            echo "Setup results: discover-test-files=${DISCOVER_RESULT}, build-images=${BUILD_RESULT}"
            exit 1
          fi
          if [ "${TEST_RESULT}" != "success" ]; then
            echo "Test result: ${TEST_RESULT}"
            exit 1
          fi
          echo "All tests passed."