pr-craft-compose-integration.yml

# Stands up the full Craft docker-compose stack with the --include-craft overlay
# and exercises the Docker-backed Craft E2E suite: approval-gate flow, workspace
# setup, and docker-specific posture invariants the K8s lane can't catch by
# construction (image ENTRYPOINT concat, HOME after setpriv, curl httpoxy on the
# bridge).
#
# Images are built once and pushed to ECR; the lane shards per test file, each
# shard pulling the prebuilt images onto its own compose stack.
#
# Always triggers on PRs + merge_group so the `craft-compose-required` job below
# can serve as the single stable required status check in branch protection; the
# heavy test job is gated internally by the `changes` job and is a no-op
# when no relevant paths changed. Also runs nightly, on release tags, and on
# demand.

name: Craft Docker-Compose Integration

concurrency:
  group: Craft-Compose-Integration-${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

on:
  schedule:
    - cron: "0 7 * * *"   # 07:00 UTC nightly
  merge_group:
  pull_request:
    branches: [main]
    # NOTE: Intentionally no `paths:` filter. We always trigger and let the
    # `changes` job below decide whether the real test job runs.
  push:
    tags:
      - "v*.*.*"
  workflow_dispatch:

permissions:
  contents: read

env:
  # Pin S3_ENDPOINT_URL defensively. The runner env is usually clean, but we hit
  # a leak during local smoke (host shell exported localhost:9004, compose's
  # ${VAR:-default} picked it over --env-file).
  S3_ENDPOINT_URL: "http://minio:9000"
  SANDBOX_BACKEND: "docker"
  ENABLE_CRAFT: "true"
  POSTGRES_PASSWORD: "password"
  POSTGRES_USER: "postgres"
  IMAGE_TAG: "latest"
  # Pin to a CI-only tag so the locally-built sandbox image is what compose
  # passes to api_server, and what DockerSandboxManager.provision then loads
  # into each sandbox container.
  SANDBOX_CONTAINER_IMAGE: "onyxdotapp/sandbox:ci"
  # Read by api_server on boot; Sandbox containers call back via this URL on the
  # compose bridge.
  ONYX_SERVER_URL: "http://api_server:8080"

jobs:
  changes:
    # Decides whether the heavy integration job runs. On pull_request /
    # merge_group we use paths-filter; on schedule / push (tags) /
    # workflow_dispatch the filter is skipped and the output defaults to `true`
    # so everything runs.
    runs-on: ubuntu-latest
    timeout-minutes: 5
    # paths-filter needs pull-requests:read to list PR files on private repos.
    permissions:
      contents: read
      pull-requests: read
    outputs:
      craft_compose: ${{ steps.filter.outputs.craft_compose || 'true' }}
    steps:
      - name: Checkout code
        if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
        with:
          persist-credentials: false
      - uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
        id: filter
        if: github.event_name == 'pull_request' || github.event_name == 'merge_group'
        with:
          filters: |
            craft_compose:
              - 'backend/Dockerfile'
              - 'backend/onyx/sandbox_proxy/**'
              - 'backend/onyx/server/features/build/**'
              - 'backend/onyx/skills/**'
              - 'deployment/docker_compose/docker-compose.yml'
              - 'deployment/docker_compose/docker-compose.dev.yml'
              - 'deployment/docker_compose/docker-compose.craft.yml'
              - 'backend/tests/integration/conftest.py'
              - 'backend/tests/integration/common_utils/**'
              - 'backend/tests/integration/tests/craft/*.py'
              - 'backend/tests/integration/tests/craft/docker_e2e/**'
              - '.github/workflows/pr-craft-compose-integration.yml'
              - '.github/actions/setup-test-license/**'
              - '.github/actions/setup-python-and-install-dependencies/**'
              - '.github/actions/login-ecr-pullthrough-cache/**'

  discover-test-files:
    # One shard per craft compose test file (top-level craft/ + docker_e2e/,
    # excluding the k8s/ subdir).
    needs: changes
    if: needs.changes.outputs.craft_compose == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 5
    outputs:
      test-files: ${{ steps.set-matrix.outputs.test-files }}
    steps:
      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Discover craft compose test files
        id: set-matrix
        run: |
          set -eo pipefail
          # `path` is relative to backend/ (the test job's working-directory).
          matrix_file="${RUNNER_TEMP}/craft-compose-test-files.jsonl"
          : > "${matrix_file}"
          {
            find backend/tests/integration/tests/craft \
              -maxdepth 1 -name 'test_*.py' -type f -print0
            find backend/tests/integration/tests/craft/docker_e2e \
              -maxdepth 1 -name 'test_*.py' -type f -print0
          } | sort -z | while IFS= read -r -d '' f; do
            base=$(basename "${f}" .py)
            shard="${base#test_}"
            rel="${f#backend/}"
            jq -cn --arg path "${rel}" --arg name "${shard}" \
              '{path: $path, name: $name}' >> "${matrix_file}"
          done
          if [ ! -s "${matrix_file}" ]; then
            echo "::error::no craft compose test files discovered"
            exit 1
          fi
          echo "test-files=$(jq -cs . "${matrix_file}")" >> "$GITHUB_OUTPUT"

  build-images:
    # Build the 2 images in parallel (one matrix leg each) and push to the shared
    # ECR repo so each test shard pulls prebuilt images instead of cold-building.
    name: build-image (${{ matrix.image }})
    needs: changes
    if: needs.changes.outputs.craft_compose == 'true'
    strategy:
      fail-fast: false
      matrix:
        include:
          - image: sandbox
            context: ./backend/onyx/server/features/build/sandbox/image
            file: ./backend/onyx/server/features/build/sandbox/image/Dockerfile
            # CI tests don't exercise skill runtime deps (soffice/pdftoppm/pptxgenjs),
            # so skip LibreOffice et al. to keep the CI image lean. Browser stays
            # on (ENABLE_BROWSER defaults true) — chromium is lighter than the
            # skill bundle and keeps the CI image consistent with prod.
            extra_build_args: "ENABLE_SKILLS=false"
            target: ""
          - image: backend
            context: ./backend
            file: ./backend/Dockerfile
            extra_build_args: ""
            # The production image; the backend Dockerfile's default (last) stage is the
            # dev variant.
            target: runtime
    runs-on:
      - runs-on
      - runner=8cpu-linux-x64
      - spot=false
      - volume=100gb
      - ${{ format('run-id={0}-craft-compose-build-{1}', github.run_id, matrix.image) }}
      - extras=ecr-cache
    timeout-minutes: 30
    permissions:
      contents: read
    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout code
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
        with:
          persist-credentials: false

      - name: Log in to ECR pull-through cache
        uses: ./.github/actions/login-ecr-pullthrough-cache
        with:
          ecr-registry: ${{ vars.ECR_REGISTRY }}

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4

      - name: Build and push ${{ matrix.image }} image
        uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
        with:
          context: ${{ matrix.context }}
          file: ${{ matrix.file }}
          # Empty target means the Dockerfile's default (last) stage.
          target: ${{ matrix.target }}
          platforms: linux/amd64
          build-args: |
            BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}
            ${{ matrix.extra_build_args }}
          tags: ${{ env.RUNS_ON_ECR_CACHE }}:craft-compose-${{ matrix.image }}-${{ github.run_id }}
          push: true
          # Attestations attach as ECR referrers to the image digest, which is
          # stable across runs and caps out at 100 per subject.
          provenance: false
          sbom: false
          cache-from: type=gha,scope=craft-compose-${{ matrix.image }}
          cache-to: type=gha,scope=craft-compose-${{ matrix.image }},mode=max

  craft-compose-integration-test:
    name: craft-compose (${{ matrix.test-file.name }})
    needs: [changes, discover-test-files, build-images]
    if: needs.changes.outputs.craft_compose == 'true'
    permissions:
      contents: read
      id-token: write
    runs-on:
      - runs-on
      - runner=4cpu-linux-x64
      - spot=false
      - volume=100gb
      - ${{ format('run-id={0}-craft-compose-tests-{1}', github.run_id, matrix.test-file.name) }}
      - extras=ecr-cache
    timeout-minutes: 30

    strategy:
      # fail-fast off so one shard's failure doesn't cancel the others.
      fail-fast: false
      matrix:
        test-file: ${{ fromJson(needs.discover-test-files.outputs.test-files) }}

    env:
      PYTHONPATH: ./backend
      DISABLE_TELEMETRY: "true"

    steps:
      - uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

      - name: Checkout
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
        with:
          persist-credentials: false

      - name: Fetch dev license
        uses: ./.github/actions/setup-test-license
        with:
          aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}

      - name: Setup Python + deps
        uses: ./.github/actions/setup-python-and-install-dependencies
        with:
          requirements: |
            backend/requirements/default.txt
            backend/requirements/dev.txt
            backend/requirements/ee.txt

      - name: Log in to ECR pull-through cache
        uses: ./.github/actions/login-ecr-pullthrough-cache
        with:
          ecr-registry: ${{ vars.ECR_REGISTRY }}

      # Retag the prebuilt ECR images to the local names compose expects (compose
      # consumes local docker images directly, so pull+tag is enough).
      - name: Pull and tag prebuilt images
        env:
          ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
          RUN_ID: ${{ github.run_id }}
        run: |
          set -eo pipefail
          retag() {
            local src="$1" dst="$2"
            docker pull "$src"
            docker tag "$src" "$dst"
          }
          retag "${ECR_CACHE}:craft-compose-sandbox-${RUN_ID}" "${SANDBOX_CONTAINER_IMAGE}"
          retag "${ECR_CACHE}:craft-compose-backend-${RUN_ID}" "onyxdotapp/onyx-backend:latest"

      - name: Create compose-external resources
        # docker-compose.craft.yml declares both as external. In prod install.sh
        # --include-craft creates them, but CI doesn't run install.sh.
        run: |
          docker network create onyx_craft_sandbox
          docker volume create sandbox_proxy_ca

      # CI skips install.sh, so these have to be added explicitly.
      - name: Provide USER_AUTH_SECRET + upload caps (.env)
        working-directory: deployment/docker_compose
        run: |
          {
            echo 'USER_AUTH_SECRET=craft-compose-ci-only-dummy-secret'
            echo 'BUILD_MAX_UPLOAD_FILE_SIZE_MB=2'
            echo 'BUILD_MAX_UPLOAD_FILES_PER_SESSION=5'
            echo 'BUILD_MAX_TOTAL_UPLOAD_SIZE_MB=4'
            echo 'USER_LIBRARY_MAX_FILES_PER_UPLOAD=5'
            echo 'SANDBOX_IDLE_CLEANUP_INTERVAL_SECONDS=10'
          } > .env

      # Raw docker compose (ods compose has no craft overlay yet). Only
      # api_server/background/sandbox-proxy are named; depends_on pulls in db /
      # cache / opensearch / model servers / minio. web_server + nginx omitted.
      - name: Bring up the stack (yml + dev + craft)
        working-directory: deployment/docker_compose
        run: |
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            -f docker-compose.craft.yml \
            --env-file env.template \
            up -d --wait --wait-timeout 180 \
            api_server background sandbox-proxy

      - name: Sanity check stack health
        # --fail: without it, curl exits 0 on 4xx/5xx and the step would accept
        # an unhealthy api_server. -w still emits on --fail.
        run: |
          docker ps --format 'table {{.Names}}\t{{.Status}}'
          curl --fail -sS -o /dev/null -w "api_server /health -> %{http_code}\n" \
            http://localhost:8080/health

      - name: Seed dev license
        run: |
          docker exec \
            -e ONYX_DEV_LICENSE="${ONYX_DEV_LICENSE}" \
            onyx-api_server-1 \
            python -m scripts.seed_dev_license

      - name: Run integration tests
        timeout-minutes: 25
        shell: bash
        working-directory: backend
        env:
          API_SERVER_HOST: "127.0.0.1"
          API_SERVER_PORT: "8080"
          POSTGRES_DB: "postgres"
          POSTGRES_HOST: "127.0.0.1"
          POSTGRES_PORT: "5432"
          REDIS_HOST: "127.0.0.1"
          REDIS_PORT: "6379"
          S3_ENDPOINT_URL: "http://127.0.0.1:9004"
          S3_AWS_ACCESS_KEY_ID: "minioadmin"
          S3_AWS_SECRET_ACCESS_KEY: "minioadmin"
          # onyx.* reads these at import; DockerSandboxManager needs the
          # proxy/docker env. USER_AUTH_SECRET must match the stack's .env.
          ENABLE_CRAFT: "true"
          ENABLE_PAID_ENTERPRISE_EDITION_FEATURES: "true"
          USER_AUTH_SECRET: "craft-compose-ci-only-dummy-secret"
          SANDBOX_BACKEND: "docker"
          SANDBOX_PROXY_HOST: "sandbox-proxy"
          SANDBOX_PROXY_PORT: "8080"
          SANDBOX_DOCKER_NETWORK: "onyx_craft_sandbox"
          SANDBOX_DOCKER_SOCKET: "/var/run/docker.sock"
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          set -o pipefail
          mkdir -p ../compose-logs
          PYTHONUNBUFFERED=1 stdbuf -oL -eL py.test \
            -v \
            --tb=short \
            -rs \
            --durations=10 \
            "${{ matrix.test-file.path }}" \
            2>&1 | tee ../compose-logs/pytest.log

      - name: Stack state + log tails on failure
        if: ${{ !success() }}
        run: |
          echo "=== docker ps -a ==="; docker ps -a || true
          echo "=== api_server tail ==="; docker logs --tail 200 onyx-api_server-1 || true
          echo "=== sandbox-proxy tail ==="; docker logs --tail 200 onyx-sandbox-proxy-1 || true
          echo "=== background tail ==="; docker logs --tail 100 onyx-background-1 || true
          echo "=== sandbox-* tails ==="; \
            for c in $(docker ps -aq --filter "name=sandbox-" | grep -v sandbox-proxy || true); do \
              echo "--- $c ---"; docker logs --tail 200 "$c" || true; \
            done

      - name: Collect Docker logs on failure
        if: ${{ !success() }}
        run: |
          mkdir -p compose-logs
          # Compose-managed services (api_server, background, sandbox-proxy
          # and their depends_on graph).
          cd deployment/docker_compose
          for cid in $(docker compose \
              -f docker-compose.yml \
              -f docker-compose.dev.yml \
              -f docker-compose.craft.yml \
              ps -aq); do
            name=$(docker inspect --format='{{.Name}}' "$cid" | sed 's/^\///')
            docker logs "$cid" > "../../compose-logs/${name}.log" 2>&1 || true
          done
          # Test-provisioned sandbox containers (not in any compose file --
          # DockerSandboxManager creates them at runtime).
          cd ../..
          for cid in $(docker ps -aq --filter "name=sandbox-" \
              | grep -v sandbox-proxy || true); do
            name=$(docker inspect --format='{{.Name}}' "$cid" | sed 's/^\///')
            docker logs "$cid" > "compose-logs/${name}.log" 2>&1 || true
          done

      - name: Upload logs
        if: ${{ !success() }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
        with:
          name: craft-compose-logs-${{ matrix.test-file.name }}
          path: compose-logs/
          retention-days: 7

      # -v drops the project's named volumes; the external sandbox_proxy_ca /
      # onyx_craft_sandbox are removed explicitly below.
      - name: Teardown
        if: always()
        working-directory: deployment/docker_compose
        run: |
          docker compose \
            -f docker-compose.yml \
            -f docker-compose.dev.yml \
            -f docker-compose.craft.yml \
            down -v || true
          docker volume rm sandbox_proxy_ca || true
          docker network rm onyx_craft_sandbox || true

  craft-compose-integration-required:
    # Single required status check for the craft-compose integration suite.
    # Always runs so branch protection has a stable target, and passes cleanly
    # when `changes` reports no relevant paths changed (i.e. the test job was
    # legitimately skipped).
    runs-on: ubuntu-latest
    timeout-minutes: 5
    needs: [changes, discover-test-files, build-images, craft-compose-integration-test]
    if: ${{ always() }}
    steps:
      - name: Check job status
        env:
          CHANGES_RESULT: ${{ needs.changes.result }}
          RUN_TESTS: ${{ needs.changes.outputs.craft_compose }}
          DISCOVER_RESULT: ${{ needs.discover-test-files.result }}
          BUILD_RESULT: ${{ needs.build-images.result }}
          TEST_RESULT: ${{ needs.craft-compose-integration-test.result }}
        run: |
          # Fail closed if `changes` didn't succeed. Otherwise an empty
          # RUN_TESTS (which is what we'd see when `changes` failed/cancelled)
          # would be indistinguishable from "no relevant paths changed" and we
          # would incorrectly pass the required check.
          if [ "${CHANGES_RESULT}" != "success" ]; then
            echo "changes job did not succeed (result: ${CHANGES_RESULT})"
            exit 1
          fi
          if [ "${RUN_TESTS}" != "true" ]; then
            echo "No relevant paths changed -- required check passes."
            exit 0
          fi
          if [ "${DISCOVER_RESULT}" != "success" ] || [ "${BUILD_RESULT}" != "success" ]; then
            echo "Setup results: discover-test-files=${DISCOVER_RESULT}, build-images=${BUILD_RESULT}"
            exit 1
          fi
          if [ "${TEST_RESULT}" != "success" ]; then
            echo "Test result: ${TEST_RESULT}"
            exit 1
          fi
          echo "All tests passed."