ci.yaml

name: CI Build

on:
  push:
    branches: [main, "release/**"]
    paths-ignore:
      - "**/*.md"
  pull_request:
    branches: [main, "release/**"]
    paths-ignore:
      - "**/*.md"
  workflow_dispatch:

concurrency:
  group: ci-${{ github.workflow }}-${{ github.event_name }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Cache key components for better organization
  CACHE_KEY_PREFIX: kagent-v2
  BRANCH_CACHE_KEY: ${{ github.head_ref || github.ref_name }}
  # Consistent builder configuration
  BUILDX_BUILDER_NAME: kagent-builder-v0.23.0
  BUILDX_VERSION: v0.23.0
jobs:
  setup:
    runs-on: ubuntu-latest
    outputs:
      cache-key: ${{ steps.cache-key.outputs.key }}
    steps:
      - name: Generate cache keys
        id: cache-key
        run: |
          # Generate branch-based cache key
          BRANCH_KEY="${CACHE_KEY_PREFIX}-${BRANCH_CACHE_KEY}"
          echo "key=${BRANCH_KEY}" >> $GITHUB_OUTPUT
          echo "::notice title=Cache Key::Branch: ${BRANCH_KEY}"
  test-e2e:
    needs:
      - setup
    env:
      VERSION: v0.0.1-test
      SUBSTRATE_VERSION: 0.0.20
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
      - name: Initialize Environment
        uses: ./.github/actions/initialize-environment
      - name: Allow unprivileged user namespaces
        # Ubuntu 24.04 (ubuntu-latest) enables AppArmor-based restrictions on
        # unprivileged user namespaces by default, which causes bubblewrap
        # to fail with EPERM on unshare(CLONE_NEWUSER)
        # See https://github.com/openai/codex/issues/14919
        run: |
          sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 || true
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          name: ${{ env.BUILDX_BUILDER_NAME }}
          version: ${{ env.BUILDX_VERSION }}
          platforms: linux/amd64
          use: "true"
          driver-opts: network=host

      - name: Set up Helm
        uses: azure/setup-helm@v5.0.1
        with:
          version: v3.18.0

      - name: Install Kind
        id: kind
        uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc
        with:
          install_only: true

      - name: Create Kind cluster
        run: |
          make create-kind-cluster

      - name: Install Agent Substrate
        run: |
          curl -fsSL -o kubectl-ate "https://github.com/kagent-dev/substrate/releases/download/v${SUBSTRATE_VERSION}/kubectl-ate-linux-amd64"
          chmod +x kubectl-ate
          helm upgrade --install substrate-crds oci://ghcr.io/kagent-dev/substrate/helm/substrate-crds --version "${SUBSTRATE_VERSION}" --namespace ate-system --create-namespace
          helm upgrade --install substrate oci://ghcr.io/kagent-dev/substrate/helm/substrate --version "${SUBSTRATE_VERSION}" --namespace ate-system --set-string 'atelet.extraArgs[0]=--localhost-registry-replacement=kind-registry:5000'
          ./kubectl-ate --context kind-kagent admin make-ca-pool --ca-id=1 --name=service-dns-ca-pool --secret-namespace=podcertificate-controller-system
          ./kubectl-ate --context kind-kagent admin make-ca-pool --ca-id=1 --name=pod-identity-ca-pool --secret-namespace=podcertificate-controller-system
          ./kubectl-ate --context kind-kagent admin make-jwt-pool --key-id=1 --name=actor-id-jwt-pool --secret-namespace=ate-system
          ./kubectl-ate --context kind-kagent admin make-ca-pool --ca-id=1 --name=actor-id-ca-pool --secret-namespace=ate-system
          actor_id_ca_root="$(kubectl get secret actor-id-ca-pool -n ate-system -o jsonpath='{.data.pool}' | base64 --decode | jq -r '.CAs[0].RootCertificateDER' | base64 --decode | openssl x509 -inform der -outform pem)"
          kubectl create secret generic actor-id-ca-certs -n ate-system --from-literal=ca.crt="${actor_id_ca_root}"
          kubectl create configmap ate-api-authentication -n ate-system --from-literal=authentication.yaml=$'actorIdentityJWTProvider: kubernetes\njwtProviders:\n- name: kubernetes\n  issuer: https://kubernetes.default.svc\n  audiences: [api.ate-system.svc]\n  certificateAuthorityFile: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt\n  discoveryTokenFile: /var/run/secrets/kubernetes.io/serviceaccount/token\n'
          helm upgrade substrate oci://ghcr.io/kagent-dev/substrate/helm/substrate --version "${SUBSTRATE_VERSION}" --namespace ate-system --reuse-values --wait --timeout 5m

      - name: Install Kagent
        id: install-kagent
        env:
          OPENAI_API_KEY: fake
          BUILDX_BUILDER_NAME: ${{ env.BUILDX_BUILDER_NAME }}
          KMCP_ENABLED: "false"
          KAGENT_HELM_EXTRA_ARGS: >-
            --cleanup-on-fail=false
            --set ui.replicas=0
            --set kagent-tools.enabled=false
            --set grafana-mcp.enabled=false
            --set controller.substrate.enabled=true
            --set controller.substrate.ateApiEndpoint=dns:///api.ate-system.svc:443
            --set controller.substrate.atenetRouterURL=http://atenet-router.ate-system.svc:80
            --set controller.substrate.defaultWorkerPool.name=kagent-default
            --set substrateWorkerPool.create=true
            --set substrateWorkerPool.replicas=1
            --set-string substrateWorkerPool.ateomImage=ghcr.io/kagent-dev/substrate/ateom-gvisor:v${{ env.SUBSTRATE_VERSION }}
          DOCKER_BUILD_ARGS: >-
            --cache-from=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e
            --cache-from=type=gha,scope=${{ env.CACHE_KEY_PREFIX }}-main-e2e
            --cache-to=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e,mode=max
            --platform=linux/amd64
            --push
        run: |
          echo "Cache key: ${{ needs.setup.outputs.cache-key }}"
          make build-controller build-golang-adk
          make helm-install-provider
          kubectl rollout status deployment/kagent-controller -n kagent --timeout=120s
          kubectl wait --for=condition=Ready pod -l app.kubernetes.io/component=controller -n kagent --timeout=120s

      - name: Run e2e tests
        if: success()
        working-directory: go
        run: |
          RUNTIME_DIGEST=$(docker buildx imagetools inspect "localhost:5001/kagent-dev/kagent/golang-adk:${VERSION}" | awk '$1 == "Digest:" { print $2; exit }')
          test -n "$RUNTIME_DIGEST"
          export KAGENT_E2E_RUNTIME_IMAGE="localhost:5001/kagent-dev/kagent/golang-adk@${RUNTIME_DIGEST}"
          envsubst < core/test/e2e/manifests/lifecycle.yaml.tmpl | kubectl apply -f -
          for _ in $(seq 1 60); do
            READY=$(kubectl get agenttemplate smoke -n kagent -o jsonpath='{.status.harnesses[?(@.harness=="kagent")].conditions[?(@.type=="Ready")].status}')
            if [ "$READY" = True ]; then break; fi
            sleep 5
          done
          test "$READY" = True
          export KAGENT_GRPC_URL="$(kubectl get svc -n kagent kagent-controller -o jsonpath='{.status.loadBalancer.ingress[0].ip}'):8084"
          echo "KAGENT_GRPC_URL: $KAGENT_GRPC_URL"
          go test -v github.com/kagent-dev/kagent/go/core/test/e2e -failfast -shuffle=on

      - name: fail print info
        if: failure()
        run: |
          echo "::error::Failed to run e2e tests"
          echo "::error::Kubectl get pods -n kagent"
          kubectl describe pods -n kagent
          echo "::error::Kubectl get events -n kagent"
          kubectl get events -n kagent
          echo "::error::Kubectl get Harnesses and AgentTemplates -n kagent"
          kubectl get harnesses,agenttemplates -n kagent
          echo "::error::Kubectl logs -n kagent deployment/kagent-controller"
          kubectl logs -n kagent deployment/kagent-controller
          kubectl get pods,actortemplates,workerpools -A
          echo "::error::Substrate logs"
          kubectl logs -n ate-system -l app --all-containers --prefix --tail=200 || true
          kubectl logs -n ate-system -l app --all-containers --prefix --previous --tail=200 || true
          kubectl logs -n kagent -l ate.dev/worker-pool --all-containers --prefix --tail=200 || true
          kubectl logs -n kagent -l ate.dev/worker-pool --all-containers --prefix --previous --tail=200 || true

  query-contraction-check:
    name: Query Contraction Check
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          # Full history + tags so the previous release's queries can be read.
          fetch-depth: 0
          fetch-tags: true
      - name: Version resolution
        run: ./scripts/version-resolution_test.sh
      - name: Set up Go
        uses: actions/setup-go@v7
        with:
          go-version-file: go/go.mod
          cache: true
          cache-dependency-path: go/go.sum
      - name: Previous-release queries vs current schema
        run: make -C go check-query-contraction

  proto-check:
    name: Protobuf Contract Check
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          fetch-depth: 0
      - name: Set up Go
        uses: actions/setup-go@v7
        with:
          go-version-file: go/go.mod
          cache: true
          cache-dependency-path: go/go.sum
      - name: Lint and generate protobuf contracts
        run: make proto-check
      - name: Check protobuf compatibility
        run: make proto-breaking

  upgrade-tests:
    needs:
      - setup
    env:
      VERSION: v0.0.1-test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # adjacent: the same-line latest published version, including prereleases
        #   — only meaningful on a release branch and skipped on main.
        upgrade-from: [adjacent]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          # Full history + tags so the version resolvers can derive the
          # upgrade-from release, and so the local action below is on disk.
          fetch-depth: 0
          fetch-tags: true
      - name: Prepare upgrade test environment
        id: prep
        uses: ./.github/actions/upgrade-test-setup
        with:
          upgrade-from: ${{ matrix.upgrade-from }}
      - name: Run upgrade tests
        if: steps.prep.outputs.skip != 'true'
        env:
          OPENAI_API_KEY: fake
          BUILDX_BUILDER_NAME: ${{ env.BUILDX_BUILDER_NAME }}
          KAGENT_HELM_EXTRA_ARGS: --cleanup-on-fail=false
          DOCKER_BUILD_ARGS: >-
            --cache-from=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e
            --cache-from=type=gha,scope=${{ env.CACHE_KEY_PREFIX }}-main-e2e
            --platform=linux/amd64
            --push
        run: |
          make run-upgrade-tests UPGRADE_FROM_VERSION="${{ steps.prep.outputs.version }}"
      - name: fail print info
        if: failure() && steps.prep.outputs.skip != 'true'
        run: |
          echo "::error::Failed to run upgrade tests"
          kubectl describe pods -n kagent
          kubectl get events -n kagent
          kubectl logs -n kagent deployment/kagent-controller || true

  rolling-upgrade-tests:
    needs:
      - setup
    env:
      VERSION: v0.0.1-test
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # adjacent: the same-line latest published version, including prereleases
        #   — only meaningful on a release branch and skipped on main.
        upgrade-from: [adjacent]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
        with:
          # Full history + tags so the version resolvers can derive the
          # upgrade-from release, and so the local action below is on disk.
          fetch-depth: 0
          fetch-tags: true
      - name: Prepare upgrade test environment
        id: prep
        uses: ./.github/actions/upgrade-test-setup
        with:
          upgrade-from: ${{ matrix.upgrade-from }}
      - name: Run rolling upgrade tests
        if: steps.prep.outputs.skip != 'true'
        env:
          OPENAI_API_KEY: fake
          BUILDX_BUILDER_NAME: ${{ env.BUILDX_BUILDER_NAME }}
          KAGENT_HELM_EXTRA_ARGS: --cleanup-on-fail=false
          DOCKER_BUILD_ARGS: >-
            --cache-from=type=gha,scope=${{ needs.setup.outputs.cache-key }}-e2e
            --cache-from=type=gha,scope=${{ env.CACHE_KEY_PREFIX }}-main-e2e
            --platform=linux/amd64
            --push
        run: |
          make run-rolling-upgrade-tests UPGRADE_FROM_VERSION="${{ steps.prep.outputs.version }}"
      - name: fail print info
        if: failure() && steps.prep.outputs.skip != 'true'
        run: |
          echo "::error::Failed to run rolling upgrade tests"
          kubectl describe pods -n kagent
          kubectl get events -n kagent
          kubectl logs -n kagent deployment/kagent-controller || true

  go-unit-tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v7
        with:
          go-version-file: go/go.mod
          cache: true
          cache-dependency-path: go/go.sum

      - name: Run Go unit tests
        working-directory: go
        env:
          # Runners are ephemeral, so Ryuk's leaked-container cleanup buys nothing,
          # and its single session-scoped reaper is contended by the parallel package
          # processes of `go test ./...`, intermittently timing out the Postgres tests.
          TESTCONTAINERS_RYUK_DISABLED: "true"
        run: |
          make setup-envtest
          export KUBEBUILDER_ASSETS="$(make -s envtest-path)"
          go test -race -v ./... -skip 'TestE2E.*'

  helm-unit-tests:
    env:
      VERSION: v0.0.1-test
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Helm
        uses: azure/setup-helm@v5.0.1
        with:
          version: v3.18.0
      # Install unittest plugin
      - name: Install unittest plugin
        run: |
          helm plugin install https://github.com/helm-unittest/helm-unittest

      - name: Chart init
        run: |
          make helm-version

      - name: Run helm unit tests
        run: |
          helm unittest helm/kagent
          helm unittest helm/tools/grafana-mcp

  ui-tests:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: write
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Setup Node.js
        uses: actions/setup-node@v7
        with:
          node-version-file: ui/.nvmrc
          cache: "npm"
          cache-dependency-path: ui/package-lock.json

      # Honor the pinned npm from ui/package.json "packageManager" so npm ci
      # resolves the lock file with the same npm version it was generated with.
      - name: Enable Corepack
        run: corepack enable

      - name: Install dependencies
        working-directory: ./ui
        run: npm ci

      - name: Run lint
        working-directory: ./ui
        run: npm run lint

      - name: Run unit tests (Jest)
        working-directory: ./ui
        run: npm run test

      - name: Install Playwright browser (Chromium)
        working-directory: ./ui
        run: npx playwright install --with-deps chromium

      - name: Run Storybook tests (Vitest + Playwright)
        working-directory: ./ui
        run: npm run test:vitest

  # This job builds the Docker images for the controller, UI, ADKs, and CLI on arm64.
  build:
    needs: setup
    env:
      VERSION: v0.0.1-test
    strategy:
      matrix:
        image:
          - controller
          - ui
          - kagent-adk
          - cli
          - golang-adk
    runs-on: ubuntu-latest
    services:
      registry:
        image: registry:2
        ports:
          - 5001:5000
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v4
        with:
          platforms: linux/amd64,linux/arm64
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4
        with:
          name: ${{ env.BUILDX_BUILDER_NAME }}
          platforms: linux/amd64,linux/arm64
          version: ${{ env.BUILDX_VERSION }}
          use: "true"
          driver-opts: network=host
      - name: Run make build
        env:
          BUILDX_BUILDER_NAME: ${{ env.BUILDX_BUILDER_NAME }}
          DOCKER_BUILD_ARGS: >-
            --cache-from=type=gha,scope=${{ needs.setup.outputs.cache-key }}-${{ matrix.image }}
            --cache-from=type=gha,scope=${{ env.CACHE_KEY_PREFIX }}-main-${{ matrix.image }}
            --cache-to=type=gha,scope=${{ needs.setup.outputs.cache-key }}-${{ matrix.image }},mode=max
            --platform=linux/amd64,linux/arm64
            --push
          DOCKER_REPO: "${{ github.repository_owner }}/kagent"
          DOCKER_BUILDER: "docker buildx build"
        run: make build-${{ matrix.image }}
        working-directory: ./

  go-lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v7
        with:
          go-version-file: go/go.mod
          cache: true
          cache-dependency-path: go/go.sum
      - name: golangci-lint
        working-directory: go
        run: make lint

  python-test:
    env:
      OPENAI_API_KEY: fake
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11", "3.12", "3.13"]
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install uv
        uses: astral-sh/setup-uv@v7

      - name: Install python
        run: uv python install ${{ matrix.python-version }}

      - name: Sync dependencies
        working-directory: ./python
        run: uv sync --all-extras

      - name: Generate test certificates
        working-directory: ./python
        run: make generate-test-certs

      - name: Run tests
        working-directory: ./python
        run: make test

  python-lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Install uv
        uses: astral-sh/setup-uv@v7

      - name: Install python
        run: uv python install 3.11

      - name: Verify uv.lock matches pyproject.toml
        working-directory: ./python
        run: uv lock --check

      - name: Sync dependencies
        working-directory: ./python
        run: uv sync --all-extras

      - name: Run ruff linter
        working-directory: ./python
        run: uv run ruff check

      - name: Run ruff format check
        working-directory: ./python
        run: |
          if ! uv run ruff format --diff .; then
            echo "Ruff formatting issues detected. Please run 'uv run ruff format' locally to fix formatting issues."
            exit 1
          fi

  manifests-check:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout repository
        uses: actions/checkout@v6

      - name: Set up Go
        uses: actions/setup-go@v7
        with:
          go-version-file: go/go.mod
          cache: true
          cache-dependency-path: go/go.sum

      - name: Generate controller manifests
        run: make controller-manifests

      - name: Check for uncommitted changes
        run: |
          if ! git diff --exit-code; then
            echo "::error::Controller manifests are out of date!"
            echo "::error::Please run 'make controller-manifests' and commit the changes."
            echo ""
            echo "The following files have differences:"
            git diff --name-only
            echo ""
            echo "Diff details:"
            git diff
            exit 1
          fi
          echo "✓ Controller manifests are up to date"