Craft Kubernetes Integration Tests #7567

Workflow file for this run

.github/workflows/pr-craft-k8s-tests.yml at b77e759

	# Runs the Craft Kubernetes integration tests against a Helm-installed kind
	# cluster with the real API, Celery workers, sandbox proxy, and sandbox pods.
	#
	# Each test file owns a module-scoped sandbox pod and the single-node kind
	# cluster fits one sandbox pod at a time, so the lane shards per test file, each
	# shard on its own kind cluster (dynamic `fail-fast: false` matrix).
	#
	# Images are built once and pushed to the shared ECR repo, then each shard
	# mirrors them into a local `localhost:5001` registry that kind pulls from
	# unauthenticated — sandbox pods have no `imagePullSecrets`, so the image must
	# come from an unauthenticated registry. (kind pulls from the local registry
	# rather than `kind load`, which writes a multi-GB tar to tmpfs /tmp.)
	# https://kind.sigs.k8s.io/docs/user/local-registry/
	name: Craft Kubernetes Integration Tests
	concurrency:
	group: Craft-K8s-Tests-${{ github.workflow }}-${{ github.head_ref \|\| github.ref }}
	cancel-in-progress: true

	on:
	schedule:
	- cron: "0 7 * * *" # 07:00 UTC nightly
	merge_group:
	pull_request:
	branches: [main]
	# NOTE: Intentionally no `paths:` filter. Trigger-level `paths:` is ignored
	# for `merge_group`, so we always trigger and let the `changes` job below
	# decide whether the real test job runs (matches the docker-compose lane
	# cadence; this lane is expensive and must not run on every PR).
	push:
	tags:
	- "v..*"
	workflow_dispatch:

	permissions:
	contents: read

	env:
	S3_ENDPOINT_URL: "http://127.0.0.1:9004"
	S3_FILE_STORE_BUCKET_NAME: "onyx-file-store-bucket"
	S3_AWS_ACCESS_KEY_ID: "minioadmin"
	S3_AWS_SECRET_ACCESS_KEY: "minioadmin"

	SANDBOX_BACKEND: "kubernetes"
	SANDBOX_NAMESPACE: "onyx-sandboxes"
	SANDBOX_SERVICE_ACCOUNT_NAME: "sandbox"
	SANDBOX_CONTAINER_IMAGE: "localhost:5001/onyx-sandbox:ci"
	# Sandbox pod resource requests live in values-ci.yaml (configMap.SANDBOX_POD_*).
	ONYX_SERVER_URL: "http://onyx-api-service.onyx.svc.cluster.local:8080"

	# Proxy lives in release namespace `onyx`. SANDBOX_PROXY_HOST is set
	# later to the proxy Service ClusterIP — the runner has no cluster DNS,
	# so a Service FQDN here would fail _resolve_proxy_ip in the manager.
	HELM_RELEASE_NAMESPACE: "onyx"
	SANDBOX_PROXY_PORT: "8080"
	SANDBOX_PROXY_CA_SECRET: "sandbox-proxy-ca"
	SANDBOX_PROXY_CA_CONFIGMAP: "sandbox-proxy-ca-bundle"
	BACKEND_IMAGE: "localhost:5001/onyx-backend:ci"
	# Fail fast on opencode hangs (default is 900s) so a stuck turn surfaces logs
	# well within the job timeout instead of being killed with zero signal.
	SANDBOX_TURN_TIMEOUT_SECONDS: "120"
	KIND_REGISTRY_NAME: "kind-registry"
	KIND_REGISTRY_PORT: "5001"
	KIND_VERSION: "v0.31.0"
	KUBECTL_VERSION: "v1.35.0"
	CRAFT_ASSETS_ARTIFACT: "craft-k8s-assets"

	# The pytest process runs on the runner, so it reaches chart-managed
	# Postgres/Redis through kubectl port-forwards.
	POSTGRES_HOST: "127.0.0.1"
	POSTGRES_PORT: "5432"
	POSTGRES_USER: "postgres"
	POSTGRES_PASSWORD: "password"
	POSTGRES_DB: "postgres"
	REDIS_HOST: "127.0.0.1"
	REDIS_PORT: "6379"
	REDIS_PASSWORD: "password"
	API_SERVER_HOST: "127.0.0.1"
	API_SERVER_PORT: "8080"

	OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}

	jobs:
	changes:
	# Gates the expensive test job. On pull_request / merge_group we use
	# paths-filter; on schedule / push (tags) / workflow_dispatch the filter is
	# skipped and the output defaults to `true` so the lane runs.
	runs-on: ubuntu-latest
	timeout-minutes: 5
	permissions:
	contents: read
	pull-requests: read
	outputs:
	craft_k8s: ${{ steps.filter.outputs.craft_k8s \|\| 'true' }}
	steps:
	- name: Checkout code
	if: github.event_name == 'pull_request' \|\| github.event_name == 'merge_group'
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0
	with:
	persist-credentials: false
	- uses: dorny/paths-filter@7b450fff21473bca461d4b92ce414b9d0420d706
	id: filter
	if: github.event_name == 'pull_request' \|\| github.event_name == 'merge_group'
	with:
	filters: \|
	craft_k8s:
	- 'backend/Dockerfile'
	- 'backend/onyx/sandbox_proxy/**'
	- 'backend/onyx/server/features/build/**'
	- 'backend/onyx/server/features/skill/**'
	- 'backend/onyx/skills/**'
	- 'backend/shared_configs/**'
	# Only the sandbox/craft-specific chart bits gate this lane. Generic
	# chart changes (workers, deps, version bumps) are install-tested by
	# the `Helm - Lint and Test Charts` lane (`ct install` into kind), so
	# they don't need the full sandbox suite here.
	- 'deployment/helm/charts/onyx/templates/sandbox-*'
	- 'deployment/helm/charts/onyx/templates/sandbox-proxy/**'
	- 'deployment/helm/charts/onyx/templates/network-policy-sandbox-*'
	- 'deployment/helm/charts/onyx/templates/craft-*'
	- 'deployment/helm/charts/onyx/templates/celery-worker-scheduled-tasks*'
	- 'deployment/helm/charts/onyx/charts/code-interpreter-*'
	- 'deployment/helm/charts/onyx/values-ci.yaml'
	- 'backend/tests/integration/tests/craft/*.py'
	- 'backend/tests/integration/tests/craft/k8s/**'
	- 'backend/tests/common/craft/**'
	- 'backend/tests/integration/conftest.py'
	- 'backend/tests/integration/common_utils/**'
	- '.github/workflows/pr-craft-k8s-tests.yml'
	- '.github/actions/setup-python-and-install-dependencies/**'
	- '.github/actions/setup-test-license/**'
	- '.github/actions/login-ecr-pullthrough-cache/**'

	discover-test-files:
	# One shard per craft k8s test file (module-scoped pool fixtures keep a file on one shard).
	needs: changes
	if: needs.changes.outputs.craft_k8s == 'true'
	runs-on: ubuntu-latest
	timeout-minutes: 5
	outputs:
	test-files: ${{ steps.set-matrix.outputs.test-files }}
	steps:
	- name: Checkout code
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Discover craft k8s test files
	id: set-matrix
	run: \|
	set -eo pipefail
	# `name` = log-friendly shard id; `path` = repo-root-relative pytest target.
	entries=""
	for f in $(find backend/tests/integration/tests/craft/k8s \
	-maxdepth 1 -name 'test_*.py' -type f \| sort); do
	base=$(basename "$f" .py)
	shard="${base#test_}"
	entries="${entries}{\"path\":\"${f}\",\"name\":\"${shard}\"},"
	done
	if [ -z "${entries}" ]; then
	echo "::error::no craft k8s test files discovered"
	exit 1
	fi
	echo "test-files=[${entries%,}]" >> "$GITHUB_OUTPUT"

	prepare-craft-assets:
	name: Prepare Craft test assets
	needs: changes
	if: needs.changes.outputs.craft_k8s == 'true'
	runs-on:
	- runs-on
	- runner=2cpu-linux-x64
	- spot=false
	- ${{ format('run-id={0}-craft-k8s-tools', github.run_id) }}
	timeout-minutes: 10
	permissions:
	contents: read
	steps:
	- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

	- name: Checkout code
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Download and verify kind tools
	run: \|
	set -euo pipefail

	download() {
	curl --fail --location --silent --show-error \
	--retry 5 --retry-delay 2 --retry-all-errors \
	--output "$2" "$1"
	}

	cache_dir="${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64"
	kind_dir="${cache_dir}/kind/bin"
	kubectl_dir="${cache_dir}/kubectl/bin"
	mkdir -p "${kind_dir}" "${kubectl_dir}"

	kind_filename="kind-linux-amd64"
	kind_url="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}"
	download "${kind_url}/${kind_filename}" "${kind_dir}/${kind_filename}"
	download "${kind_url}/${kind_filename}.sha256sum" "${kind_dir}/${kind_filename}.sha256sum"
	(
	cd "${kind_dir}"
	grep "${kind_filename}" "${kind_filename}.sha256sum" \| sha256sum --check -
	mv "${kind_filename}" kind
	rm "${kind_filename}.sha256sum"
	chmod +x kind
	)

	kubectl_url="https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64"
	download "${kubectl_url}/kubectl" "${kubectl_dir}/kubectl"
	download "${kubectl_url}/kubectl.sha256" "${kubectl_dir}/kubectl.sha256"
	(
	cd "${kubectl_dir}"
	echo "$(cat kubectl.sha256) kubectl" \| sha256sum --check -
	rm kubectl.sha256
	chmod +x kubectl
	)

	- name: Download Helm chart dependencies
	run: \|
	set -euo pipefail

	retry() {
	local attempt
	for attempt in 1 2 3 4 5; do
	if "$@"; then
	return 0
	fi
	if [ "${attempt}" -eq 5 ]; then
	return 1
	fi
	echo "Command failed (attempt ${attempt}/5); retrying ..."
	sleep $((attempt * 5))
	done
	}

	retry helm repo add --force-update ingress-nginx https://kubernetes.github.io/ingress-nginx
	retry helm repo add --force-update opensearch https://opensearch-project.github.io/helm-charts
	retry helm repo add --force-update cloudnative-pg https://cloudnative-pg.github.io/charts
	retry helm repo add --force-update ot-container-kit https://ot-container-kit.github.io/helm-charts
	retry helm repo add --force-update minio https://charts.min.io/
	retry helm repo add --force-update code-interpreter https://onyx-dot-app.github.io/python-sandbox/
	retry helm repo update
	if ! retry helm dependency build --skip-refresh deployment/helm/charts/onyx; then
	echo "helm dependency build failed; pulling disabled code-interpreter dependency directly"
	code_interpreter_version=$(awk '
	$1 == "-" && $2 == "name:" && $3 == "code-interpreter" { found = 1 }
	found && $1 == "version:" { print $2; exit }
	' deployment/helm/charts/onyx/Chart.yaml)
	test -n "${code_interpreter_version}"
	retry helm pull code-interpreter/code-interpreter \
	--version "${code_interpreter_version}" \
	--destination deployment/helm/charts/onyx/charts
	fi
	helm dependency list deployment/helm/charts/onyx
	helm dependency list deployment/helm/charts/onyx \
	\| awk 'NR > 1 && NF && $4 != "ok" { bad = 1 } END { exit bad }'

	# Tar preserves executable permissions for kind and kubectl.
	- name: Package Craft test assets
	run: \|
	set -euo pipefail
	assets_dir="${RUNNER_TEMP}/craft-assets"
	mkdir -p "${assets_dir}"
	tar -C "${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64" \
	-czf "${assets_dir}/kind-tools.tar.gz" .
	tar -C deployment/helm/charts/onyx/charts \
	-czf "${assets_dir}/helm-dependencies.tar.gz" .

	- name: Upload Craft test assets
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
	with:
	name: ${{ env.CRAFT_ASSETS_ARTIFACT }}
	path: ${{ runner.temp }}/craft-assets
	if-no-files-found: error
	retention-days: 1
	compression-level: 0
	overwrite: true

	build-images:
	# Build both images in parallel (one matrix leg each) and push to the shared
	# ECR repo so each test shard pulls prebuilt images instead of cold-building.
	name: build-image (${{ matrix.image }})
	needs: changes
	if: needs.changes.outputs.craft_k8s == 'true'
	strategy:
	fail-fast: false
	matrix:
	include:
	- image: sandbox
	context: ./backend/onyx/server/features/build/sandbox/image
	file: ./backend/onyx/server/features/build/sandbox/image/Dockerfile
	# CI tests don't exercise skill runtime deps (soffice/pdftoppm/pptxgenjs),
	# so skip LibreOffice et al. to keep the CI image lean. Browser stays
	# on (ENABLE_BROWSER defaults true) — chromium is lighter and the
	# browser integration test needs it.
	extra_build_args: "ENABLE_SKILLS=false"
	target: ""
	- image: backend
	context: ./backend
	file: ./backend/Dockerfile
	extra_build_args: ""
	# The production image; the backend Dockerfile's default (last) stage is the
	# dev variant.
	target: runtime
	runs-on:
	- runs-on
	- runner=8cpu-linux-x64
	- spot=false
	- volume=100gb
	- ${{ format('run-id={0}-craft-k8s-build-{1}', github.run_id, matrix.image) }}
	- extras=ecr-cache
	timeout-minutes: 30
	permissions:
	contents: read
	steps:
	- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

	- name: Checkout code
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Log in to ECR pull-through cache
	uses: ./.github/actions/login-ecr-pullthrough-cache
	with:
	ecr-registry: ${{ vars.ECR_REGISTRY }}

	- name: Set up Docker Buildx
	uses: docker/setup-buildx-action@bb05f3f5519dd87d3ba754cc423b652a5edd6d2c # ratchet:docker/setup-buildx-action@v4

	- name: Build and push ${{ matrix.image }} image
	uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a
	with:
	context: ${{ matrix.context }}
	file: ${{ matrix.file }}
	# Empty target means the Dockerfile's default (last) stage.
	target: ${{ matrix.target }}
	platforms: linux/amd64
	build-args: \|
	BASE_IMAGE_REGISTRY=${{ env.BASE_IMAGE_REGISTRY }}
	${{ matrix.extra_build_args }}
	tags: ${{ env.RUNS_ON_ECR_CACHE }}:craft-k8s-${{ matrix.image }}-${{ github.run_id }}
	push: true
	# Attestations attach as ECR referrers to the image digest, which is
	# stable across runs and caps out at 100 per subject.
	provenance: false
	sbom: false
	cache-from: type=gha,scope=craft-${{ matrix.image }}
	cache-to: type=gha,scope=craft-${{ matrix.image }},mode=max

	craft-k8s-tests:
	name: craft-k8s (${{ matrix.test-file.name }})
	needs: [changes, discover-test-files, prepare-craft-assets, build-images]
	if: needs.changes.outputs.craft_k8s == 'true'
	# spot=false: this is a long lane (full kind cluster per shard); on-demand
	# avoids mid-run spot reclamation. Matches the compose lane.
	runs-on:
	- runs-on
	- runner=8cpu-linux-x64
	- spot=false
	- volume=100gb
	- ${{ format('run-id={0}-craft-k8s-tests-{1}', github.run_id, matrix.test-file.name) }}
	- extras=ecr-cache
	timeout-minutes: 40

	strategy:
	# fail-fast off so one shard's failure doesn't cancel the others.
	fail-fast: false
	matrix:
	test-file: ${{ fromJson(needs.discover-test-files.outputs.test-files) }}

	# id-token: OIDC for the setup-test-license step.
	permissions:
	contents: read
	id-token: write

	env:
	PYTHONPATH: ./backend
	MODEL_SERVER_HOST: "disabled"
	DISABLE_TELEMETRY: "true"
	DISABLE_VECTOR_DB: "false"

	steps:
	- uses: runs-on/action@4e5f72399b6b17f2e79c511c1b38a315a64d22dc

	- name: Checkout code
	uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # ratchet:actions/checkout@v6
	with:
	persist-credentials: false

	- name: Setup Python and Install Dependencies
	uses: ./.github/actions/setup-python-and-install-dependencies
	with:
	requirements: \|
	backend/requirements/default.txt
	backend/requirements/dev.txt
	backend/requirements/ee.txt

	- name: Download Craft test assets
	uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c
	with:
	name: ${{ env.CRAFT_ASSETS_ARTIFACT }}
	path: ${{ runner.temp }}/craft-assets

	- name: Install Craft test assets
	run: \|
	set -euo pipefail
	kind_cache_dir="${RUNNER_TOOL_CACHE}/kind/${KIND_VERSION}/amd64"
	charts_dir="deployment/helm/charts/onyx/charts"
	mkdir -p "${kind_cache_dir}" "${charts_dir}"
	tar -C "${kind_cache_dir}" \
	-xzf "${RUNNER_TEMP}/craft-assets/kind-tools.tar.gz"
	tar -C "${charts_dir}" \
	-xzf "${RUNNER_TEMP}/craft-assets/helm-dependencies.tar.gz"

	- name: Log in to ECR pull-through cache
	uses: ./.github/actions/login-ecr-pullthrough-cache
	with:
	ecr-registry: ${{ vars.ECR_REGISTRY }}

	- name: Start local image registry
	run: \|
	docker run -d --restart=always \
	-p "127.0.0.1:${KIND_REGISTRY_PORT}:5000" \
	--name "${KIND_REGISTRY_NAME}" \
	"${BASE_IMAGE_REGISTRY:-docker.io}/library/registry:2"

	# Re-push the prebuilt ECR images into this shard's local registry under the
	# tags kind pulls (unauthenticated); the blob copy is cheap vs a cold build.
	- name: Mirror prebuilt images into local registry
	env:
	ECR_CACHE: ${{ env.RUNS_ON_ECR_CACHE }}
	RUN_ID: ${{ github.run_id }}
	run: \|
	set -eo pipefail
	mirror() {
	local src="$1" dst="$2"
	docker pull "$src"
	docker tag "$src" "$dst"
	docker push "$dst"
	}
	mirror "${ECR_CACHE}:craft-k8s-sandbox-${RUN_ID}" "${SANDBOX_CONTAINER_IMAGE}"
	mirror "${ECR_CACHE}:craft-k8s-backend-${RUN_ID}" "${BACKEND_IMAGE}"

	- name: Write kind config
	run: \|
	cat > "${RUNNER_TEMP}/kind-config.yaml" <<'EOF'
	kind: Cluster
	apiVersion: kind.x-k8s.io/v1alpha4
	containerdConfigPatches:
	- \|-
	[plugins."io.containerd.grpc.v1.cri".registry]
	config_path = "/etc/containerd/certs.d"
	EOF

	- name: Create kind cluster
	uses: helm/kind-action@ef37e7f390d99f746eb8b610417061a60e82a6cc # ratchet:helm/kind-action@v1.14.0
	with:
	version: ${{ env.KIND_VERSION }}
	kubectl_version: ${{ env.KUBECTL_VERSION }}
	cluster_name: onyx-craft-ci
	node_image: kindest/node:v1.33.1
	config: ${{ runner.temp }}/kind-config.yaml

	- name: Connect local registry to kind network
	run: \|
	docker network connect "kind" "${KIND_REGISTRY_NAME}" \|\| true
	REGISTRY_DIR="/etc/containerd/certs.d/localhost:${KIND_REGISTRY_PORT}"
	docker exec onyx-craft-ci-control-plane mkdir -p "${REGISTRY_DIR}"
	docker exec onyx-craft-ci-control-plane bash -c "cat > ${REGISTRY_DIR}/hosts.toml <<EOF
	[host.\"http://${KIND_REGISTRY_NAME}:5000\"]
	EOF"

	# Pod spec sets nodeSelector onyx.app/workload=sandbox
	# (kubernetes_sandbox_manager.py). In prod this matches a dedicated
	# node pool; in kind we just label the single control-plane node.
	- name: Label kind node for sandbox workload
	run: kubectl label node onyx-craft-ci-control-plane onyx.app/workload=sandbox

	- name: Validate Helm chart dependencies
	run: \|
	helm dependency list deployment/helm/charts/onyx
	helm dependency list deployment/helm/charts/onyx \
	\| awk 'NR > 1 && NF && $4 != "ok" { bad = 1 } END { exit bad }'

	- name: Generate sandbox push key
	run: \|
	python - <<'PY' >> "$GITHUB_ENV"
	import base64
	from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey
	from cryptography.hazmat.primitives.serialization import Encoding
	from cryptography.hazmat.primitives.serialization import NoEncryption
	from cryptography.hazmat.primitives.serialization import PrivateFormat

	key = Ed25519PrivateKey.generate()
	raw = key.private_bytes(
	encoding=Encoding.Raw,
	format=PrivateFormat.Raw,
	encryption_algorithm=NoEncryption(),
	)
	print(f"ONYX_SANDBOX_PUSH_PRIVATE_KEY={base64.b64encode(raw).decode('ascii')}")
	PY

	- name: Install Onyx chart into kind
	run: \|
	set -eo pipefail
	kubectl create namespace "${HELM_RELEASE_NAMESPACE}" \
	--dry-run=client -o yaml \| kubectl apply -f -
	kubectl config set-context --current --namespace="${HELM_RELEASE_NAMESPACE}"

	for attempt in 1 2 3; do
	if helm upgrade --install onyx deployment/helm/charts/onyx \
	-n "${HELM_RELEASE_NAMESPACE}" \
	-f deployment/helm/charts/onyx/values-ci.yaml \
	--set "api.image.repository=localhost:5001/onyx-backend" \
	--set "api.image.tag=ci" \
	--set "celery_shared.image.repository=localhost:5001/onyx-backend" \
	--set "celery_shared.image.tag=ci" \
	--set-string "auth.sandboxPushSecret.values.private_key=${ONYX_SANDBOX_PUSH_PRIVATE_KEY}" \
	--timeout 10m; then
	exit 0
	fi
	if [ "$attempt" -lt 3 ]; then
	echo "helm install failed (attempt ${attempt}/3); waiting before retry ..."
	kubectl -n "${HELM_RELEASE_NAMESPACE}" get pods \|\| true
	sleep 20
	fi
	done

	helm status onyx -n "${HELM_RELEASE_NAMESPACE}" \|\| true
	kubectl get pods -A \|\| true
	exit 1

	- name: Wait for chart Postgres ready
	run: \|
	set -eo pipefail
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	deployment/onyx-cloudnative-pg --timeout=180s
	# Don't add a `pod -l cnpg.io/cluster` wait — it also matches the
	# transient initdb pod (deleted mid-wait → NotFound).
	kubectl -n "${HELM_RELEASE_NAMESPACE}" wait \
	--for=condition=Ready cluster/onyx-pg \
	--timeout=300s

	- name: Forward chart Postgres to runner
	run: \|
	set -eo pipefail
	# setsid survives the step-boundary process-group cleanup; the loop
	# survives kubectl port-forward's mid-run drops.
	setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-pg-rw \"$POSTGRES_PORT\":5432; sleep 1; done" \
	> "${RUNNER_TEMP}/postgres-port-forward.log" 2>&1 &
	echo "$!" > "${RUNNER_TEMP}/postgres-port-forward.pid"
	for _ in $(seq 1 60); do
	if python -c 'import os, psycopg2; conn = psycopg2.connect(dbname=os.environ["POSTGRES_DB"], user=os.environ["POSTGRES_USER"], password=os.environ["POSTGRES_PASSWORD"], host=os.environ["POSTGRES_HOST"], port=os.environ["POSTGRES_PORT"], connect_timeout=2); conn.close()'; then
	echo "chart Postgres is reachable at ${POSTGRES_HOST}:${POSTGRES_PORT}"
	exit 0
	fi
	sleep 2
	done
	cat "${RUNNER_TEMP}/postgres-port-forward.log" \|\| true
	exit 1

	- name: Wait for API startup migrations
	run: \|
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	deployment/onyx-api-server --timeout=360s
	kubectl -n "${HELM_RELEASE_NAMESPACE}" exec deployment/onyx-api-server -- \
	alembic current --check-heads

	- name: Wait for chart OpenSearch ready
	run: \|
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	statefulset/onyx-opensearch-master --timeout=420s

	- name: Wait for chart Redis ready
	run: \|
	set -eo pipefail
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	deployment/redis-operator --timeout=180s
	for _ in $(seq 1 120); do
	endpoints=$(kubectl -n "${HELM_RELEASE_NAMESPACE}" get endpoints onyx \
	-o jsonpath='{.subsets[0].addresses[0].ip}' 2>/dev/null \|\| true)
	if [ -n "${endpoints}" ]; then
	exit 0
	fi
	sleep 2
	done
	kubectl -n "${HELM_RELEASE_NAMESPACE}" get redis,pods,svc \|\| true
	exit 1

	- name: Forward chart Redis to runner
	run: \|
	set -eo pipefail
	setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx \"$REDIS_PORT\":6379; sleep 1; done" \
	> "${RUNNER_TEMP}/redis-port-forward.log" 2>&1 &
	echo "$!" > "${RUNNER_TEMP}/redis-port-forward.pid"
	for _ in $(seq 1 60); do
	if python -c 'import os; from redis import Redis; Redis(host=os.environ["REDIS_HOST"], port=int(os.environ["REDIS_PORT"]), password=os.environ["REDIS_PASSWORD"], socket_connect_timeout=2).ping()'; then
	echo "chart Redis is reachable at ${REDIS_HOST}:${REDIS_PORT}"
	exit 0
	fi
	sleep 2
	done
	cat "${RUNNER_TEMP}/redis-port-forward.log" \|\| true
	exit 1

	- name: Wait for chart MinIO ready
	run: \|
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	deployment/onyx-minio --timeout=180s
	if kubectl -n "${HELM_RELEASE_NAMESPACE}" get job/onyx-minio-post-job >/dev/null 2>&1; then
	kubectl -n "${HELM_RELEASE_NAMESPACE}" wait \
	--for=condition=complete job/onyx-minio-post-job \
	--timeout=180s
	else
	echo "MinIO post-install hook job already completed and was deleted by Helm"
	fi

	- name: Forward chart MinIO to runner
	run: \|
	set -eo pipefail
	setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-minio 9004:9000; sleep 1; done" \
	> "${RUNNER_TEMP}/minio-port-forward.log" 2>&1 &
	echo "$!" > "${RUNNER_TEMP}/minio-port-forward.pid"
	for _ in $(seq 1 30); do
	if curl -fsS "${S3_ENDPOINT_URL}/minio/health/ready"; then
	echo "chart MinIO is reachable at ${S3_ENDPOINT_URL}"
	exit 0
	fi
	sleep 1
	done
	cat "${RUNNER_TEMP}/minio-port-forward.log" \|\| true
	exit 1

	- name: Wait for chart app runtime ready
	run: \|
	set -eo pipefail
	for deployment in \
	onyx-sandbox-proxy \
	onyx-api-server \
	onyx-celery-beat \
	onyx-celery-worker-primary \
	onyx-celery-worker-light \
	onyx-celery-worker-heavy \
	onyx-celery-worker-scheduled-tasks; do
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	"deployment/${deployment}" --timeout=360s
	done

	# Tier-gated EE routes (test_skill_push user-groups) need a license.
	- name: Fetch dev license
	uses: ./.github/actions/setup-test-license
	with:
	aws-oidc-role-arn: ${{ secrets.AWS_OIDC_ROLE_ARN }}

	- name: Seed dev license
	# The api-server can briefly restart if Postgres/cnpg flaps right after
	# rollout, so re-assert readiness and retry the exec.
	run: \|
	set -eo pipefail
	for attempt in $(seq 1 5); do
	kubectl -n "${HELM_RELEASE_NAMESPACE}" rollout status \
	deployment/onyx-api-server --timeout=120s \|\| true
	if kubectl -n "${HELM_RELEASE_NAMESPACE}" exec deployment/onyx-api-server -- \
	env ONYX_DEV_LICENSE="${ONYX_DEV_LICENSE}" python -m scripts.seed_dev_license; then
	echo "seeded dev license (attempt ${attempt})"
	exit 0
	fi
	echo "seed dev license failed (attempt ${attempt}); retrying ..."
	sleep 10
	done
	exit 1

	- name: Forward chart API to runner
	run: \|
	set -eo pipefail
	setsid bash -c "while true; do kubectl -n \"$HELM_RELEASE_NAMESPACE\" port-forward svc/onyx-api-service \"$API_SERVER_PORT\":8080; sleep 1; done" \
	> "${RUNNER_TEMP}/api-port-forward.log" 2>&1 &
	echo "$!" > "${RUNNER_TEMP}/api-port-forward.pid"
	for _ in $(seq 1 60); do
	if curl -fsS "http://${API_SERVER_HOST}:${API_SERVER_PORT}/health"; then
	echo "chart API reachable at ${API_SERVER_HOST}:${API_SERVER_PORT}"
	exit 0
	fi
	sleep 2
	done
	cat "${RUNNER_TEMP}/api-port-forward.log" \|\| true
	exit 1

	# The test process runs on the CI runner, which has no cluster DNS.
	# `socket.gethostbyname(IP)` returns the IP unchanged, so resolving
	# the Service ClusterIP here lets _resolve_proxy_ip in the manager
	# succeed without any DNS resolution. host_aliases on sandbox pods
	# then pins `sandbox-proxy` to this same IP.
	- name: Resolve proxy Service IP and export to env
	run: \|
	PROXY_IP=$(kubectl -n "${HELM_RELEASE_NAMESPACE}" get svc \
	onyx-sandbox-proxy -o jsonpath='{.spec.clusterIP}')
	echo "SANDBOX_PROXY_HOST=${PROXY_IP}" >> "$GITHUB_ENV"
	echo "resolved proxy ClusterIP: ${PROXY_IP}"

	# Sandbox pods are torn down on failure before `kind export logs` runs,
	# so kubelet GCs their opencode-serve logs. Stream them to disk live (one
	# follower per pod, retrying the attach) so failures leave a trail.
	- name: Start sandbox pod log capture
	run: \|
	mkdir -p sandbox-logs
	# The inner `bash -c` body is single-quoted on purpose: its $vars
	# expand at runtime inside that shell, not here.
	# shellcheck disable=SC2016
	nohup bash -c '
	while true; do
	for name in $(kubectl -n onyx-sandboxes get pods \
	-o jsonpath="{.items[*].metadata.name}" 2>/dev/null); do
	if mkdir "sandbox-logs/.lock-${name}" 2>/dev/null; then
	(
	for _ in $(seq 1 120); do
	kubectl -n onyx-sandboxes logs --all-containers=true \
	--prefix=true -f "$name" \
	>> "sandbox-logs/${name}.log" 2>&1 && break
	kubectl -n onyx-sandboxes get pod "$name" \
	>/dev/null 2>&1 \|\| break
	sleep 1
	done
	) &
	fi
	done
	sleep 1
	done
	' > sandbox-logs/_watcher.log 2>&1 &
	echo "$!" > "${RUNNER_TEMP}/sandbox-log-watcher.pid"
	disown \|\| true
	echo "sandbox pod log capture started (pid $!)"

	- name: Run Craft K8s tests (${{ matrix.test-file.name }})
	shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
	run: \|
	py.test \
	--durations=8 \
	-o junit_family=xunit2 \
	-v \
	--ff \
	-m "not nightly" \
	"${{ matrix.test-file.path }}"

	- name: Collect diagnostics on failure
	if: failure()
	run: \|
	mkdir -p kind-logs

	echo "::group::sandbox pod logs"
	if compgen -G "sandbox-logs/*.log" > /dev/null; then
	for f in sandbox-logs/*.log; do echo "--- ${f} ---"; cat "${f}" \|\| true; done
	else
	echo "no sandbox pod logs were captured"
	fi
	echo "::endgroup::"

	echo "::group::disk and docker status"
	df -h \|\| true; docker system df \|\| true; docker images \|\| true; ls -lah /tmp \|\| true
	echo "::endgroup::"

	echo "::group::MinIO diagnostics"
	kubectl -n "${HELM_RELEASE_NAMESPACE}" get pods,svc,job -l release=onyx \|\| true
	kubectl -n "${HELM_RELEASE_NAMESPACE}" logs deployment/onyx-minio --tail=500 \|\| true
	kubectl -n "${HELM_RELEASE_NAMESPACE}" logs job/onyx-minio-post-job --all-containers --tail=500 \|\| true
	python - <<'PY' \|\| true
	import os, boto3
	from botocore.config import Config
	s3 = boto3.client(
	"s3",
	endpoint_url=os.environ.get("S3_ENDPOINT_URL"),
	aws_access_key_id=os.environ.get("S3_AWS_ACCESS_KEY_ID"),
	aws_secret_access_key=os.environ.get("S3_AWS_SECRET_ACCESS_KEY"),
	region_name=os.environ.get("AWS_REGION_NAME") or "us-east-1",
	config=Config(s3={"addressing_style": "path"}),
	)
	print("buckets:", [b["Name"] for b in s3.list_buckets().get("Buckets", [])])
	for b in s3.list_buckets().get("Buckets", []):
	objs = s3.list_objects_v2(Bucket=b["Name"]).get("Contents", [])
	print(b["Name"], "->", [o["Key"] for o in objs][:20])
	PY
	echo "::endgroup::"

	echo "::group::chart resources and app logs"
	helm status onyx -n "${HELM_RELEASE_NAMESPACE}" \|\| true
	for dep in onyx-api-server onyx-celery-worker-primary \
	onyx-celery-worker-heavy onyx-celery-worker-scheduled-tasks \
	onyx-cloudnative-pg redis-operator; do
	kubectl -n "${HELM_RELEASE_NAMESPACE}" logs "deployment/${dep}" --all-containers --tail=500 \|\| true
	done
	kubectl -n "${HELM_RELEASE_NAMESPACE}" logs statefulset/onyx-opensearch-master --all-containers --tail=500 \|\| true
	kubectl -n "${HELM_RELEASE_NAMESPACE}" get cluster,redis,statefulset,pods,svc,pvc \|\| true
	echo "::endgroup::"

	echo "::group::port-forward logs"
	for f in postgres redis minio api; do
	echo "--- ${f} ---"; cat "${RUNNER_TEMP}/${f}-port-forward.log" 2>/dev/null \|\| true
	done
	echo "::endgroup::"

	kind export logs ./kind-logs --name onyx-craft-ci \|\| true
	kubectl get pods -A -o wide > kind-logs/pods.txt 2>&1 \|\| true
	kubectl get svc -A -o wide > kind-logs/services.txt 2>&1 \|\| true
	kubectl describe pods -n onyx-sandboxes > kind-logs/sandbox-pods-describe.txt 2>&1 \|\| true
	kubectl describe pods -n "${HELM_RELEASE_NAMESPACE}" > kind-logs/release-ns-pods-describe.txt 2>&1 \|\| true
	kubectl -n "${HELM_RELEASE_NAMESPACE}" logs -l app.kubernetes.io/component=sandbox-proxy --tail=500 > kind-logs/sandbox-proxy.log 2>&1 \|\| true
	cp "${RUNNER_TEMP}"/*-port-forward.log kind-logs/ 2>/dev/null \|\| true

	- name: Upload logs
	if: failure()
	uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a
	with:
	name: craft-k8s-logs-${{ matrix.test-file.name }}
	path: \|
	kind-logs/
	sandbox-logs/
	retention-days: 7

	- name: Cleanup kind cluster and port-forwards
	if: always()
	run: \|
	set +e
	for pid_file in \
	"${RUNNER_TEMP}/postgres-port-forward.pid" \
	"${RUNNER_TEMP}/redis-port-forward.pid" \
	"${RUNNER_TEMP}/minio-port-forward.pid" \
	"${RUNNER_TEMP}/api-port-forward.pid" \
	"${RUNNER_TEMP}/sandbox-log-watcher.pid"; do
	if [ -f "${pid_file}" ]; then
	kill "$(cat "${pid_file}")" 2>/dev/null \|\| true
	fi
	done
	kind delete cluster --name onyx-craft-ci \|\| true
	docker network disconnect kind "${KIND_REGISTRY_NAME}" 2>/dev/null \|\| true
	docker rm -f "${KIND_REGISTRY_NAME}" 2>/dev/null \|\| true

	craft-k8s-tests-required:
	# Single required status check for the craft-k8s suite. Always runs so
	# branch protection has a stable target, and passes cleanly when `changes`
	# reports no relevant paths changed (i.e. the test job was legitimately
	# skipped).
	runs-on: ubuntu-latest
	timeout-minutes: 5
	needs: [changes, discover-test-files, build-images, craft-k8s-tests]
	if: ${{ always() }}
	steps:
	- name: Check job status
	env:
	CHANGES_RESULT: ${{ needs.changes.result }}
	RUN_TESTS: ${{ needs.changes.outputs.craft_k8s }}
	DISCOVER_RESULT: ${{ needs.discover-test-files.result }}
	BUILD_RESULT: ${{ needs.build-images.result }}
	TEST_RESULT: ${{ needs.craft-k8s-tests.result }}
	run: \|
	# Fail closed if `changes` didn't succeed. Otherwise an empty
	# RUN_TESTS (which is what we'd see when `changes` failed/cancelled)
	# would be indistinguishable from "no relevant paths changed" and we
	# would incorrectly pass the required check.
	if [ "${CHANGES_RESULT}" != "success" ]; then
	echo "changes job did not succeed (result: ${CHANGES_RESULT})"
	exit 1
	fi
	if [ "${RUN_TESTS}" != "true" ]; then
	echo "No relevant paths changed -- required check passes."
	exit 0
	fi
	if [ "${DISCOVER_RESULT}" != "success" ] \|\| [ "${BUILD_RESULT}" != "success" ]; then
	echo "Setup results: discover-test-files=${DISCOVER_RESULT}, build-images=${BUILD_RESULT}"
	exit 1
	fi
	if [ "${TEST_RESULT}" != "success" ]; then
	echo "Test result: ${TEST_RESULT}"
	exit 1
	fi
	echo "All tests passed."

Navigation Menu

Uh oh!

Craft Kubernetes Integration Tests #7567

Workflow file

Craft Kubernetes Integration Tests #7567

Uh oh!

Workflow file for this run