Skip to content
Local KIND Deployment

ci: run Azure deployment gate against HEAD #573

Sign in to view logs

ci: run Azure deployment gate against HEAD

ci: run Azure deployment gate against HEAD #573

Workflow file for this run

.github/workflows/oetf-kind.yaml at 2d1355c
name: Local KIND Deployment
on:
pull_request:
branches: [main]
paths:
- 'deployments/charts/**'
- 'deployments/scripts/**'
- 'deployments/values/**'
- 'test/oetf/**'
- 'test/smoke/**'
- 'test/scenarios/**'
- 'test/workflow/**'
- 'bzl/**'
- 'MODULE.bazel'
- 'src/**'
- '!src/scripts/testbot/**'
- '.github/workflows/oetf-kind.yaml'
workflow_dispatch: {}
permissions:
contents: read
concurrency:
group: oetf-kind-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
jobs:
oetf-kind:
name: oetf:deploy_and_run --env kind --tags kind
runs-on: ubuntu-latest
timeout-minutes: 60
steps:
- name: Checkout
uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
with:
# No git push from this job — drop the auto-injected token from
# the workspace so artifact uploads (kind-smoke-result,
# bazel-testlogs) can't accidentally include or leak it.
persist-credentials: false
- name: Install kind
run: |
set -euo pipefail
KIND_VERSION=v0.24.0
KIND_SHA256=b89aada5a39d620da3fcd16435b7f28d858927dd53f92cbac77686b0588b600d
curl -fsSLo /usr/local/bin/kind \
"https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64"
echo "${KIND_SHA256} /usr/local/bin/kind" | sha256sum -c -
sudo chmod +x /usr/local/bin/kind
- name: Install kubectl
run: |
set -euo pipefail
KUBECTL_VERSION=v1.31.0
curl -fsSLo /usr/local/bin/kubectl \
"https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl"
curl -fsSL "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl.sha256" \
| awk '{print $1" /usr/local/bin/kubectl"}' | sha256sum -c -
sudo chmod +x /usr/local/bin/kubectl
- name: Install helm
run: |
set -euo pipefail
HELM_VERSION=v3.16.2
HELM_SHA256=9318379b847e333460d33d291d4c088156299a26cd93d570a7f5d0c36e50b5bb
curl -fsSLo /tmp/helm.tgz \
"https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz"
echo "${HELM_SHA256} /tmp/helm.tgz" | sha256sum -c -
tar -xzf /tmp/helm.tgz -C /tmp linux-amd64/helm
sudo mv /tmp/linux-amd64/helm /usr/local/bin/helm
rm -rf /tmp/helm.tgz /tmp/linux-amd64
- name: Tool versions
run: |
docker --version
kind version
kubectl version --client --output=yaml | head -2 | tail -1
helm version --short
- name: Setup Bazel
uses: bazel-contrib/setup-bazel@4fd964a13a440a8aeb0be47350db2fc640f19ca8
with:
bazelisk-cache: true
bazelisk-version: 1.27.0
disk-cache: ${{ github.workflow }}
repository-cache: true
- name: Add quick-start.osmo to /etc/hosts
run: |
# The KIND adapter's check_kind_prereqs requires this name to
# resolve so tests can reach the chart's ingress at
# http://quick-start.osmo via the host port mapping published by
# `kind create cluster --config`.
echo "127.0.0.1 quick-start.osmo" | sudo tee -a /etc/hosts
- name: Preflight (OETF KIND adapter check_kind_prereqs)
run: |
set -euo pipefail
missing=()
for bin in docker kind kubectl helm; do
command -v "$bin" >/dev/null 2>&1 || missing+=("$bin")
done
if (( ${#missing[@]} > 0 )); then
echo "ERROR: missing on runner: ${missing[*]}"
exit 3
fi
- name: Free runner disk for --build-local
# Reclaim ~22-25 GB by removing pre-installed tooling the OETF
# gate doesn't use. Pairs with the intra-step host-docker cleanup
# in local_images.build_and_load (docker rmi + tarball delete per
# image after kind-load), needed because the chart's 6-node KIND
# profile duplicates every loaded image into 6 separate
# containerd content stores. Without both mitigations the runner
# crashes mid-job with "No space left on device" (measured #29,
# #31, #32).
run: |
set -euo pipefail
for d in /usr/share/dotnet /usr/local/lib/android /opt/ghc \
/opt/hostedtoolcache/PyPy /opt/hostedtoolcache/CodeQL \
/opt/google/chrome /opt/microsoft /opt/az /opt/pipx \
/opt/swift /usr/local/share/boost \
/usr/local/share/powershell; do
if [ -d "$d" ]; then
sudo rm -rf "$d" &
fi
done
wait
sudo docker image prune -af >/dev/null 2>&1 || true
- name: Disk space (pre-deploy)
run: df -h /
- name: Build osmo CLI
run: |
# CLI-mode scenarios (e.g. router-connectivity, smoke:cli-checks)
# need an `osmo` binary to invoke. Build from //src/cli and expose
# at /usr/local/bin/osmo (the default --local-osmo path) via
# symlink.
bazel build //src/cli:cli
sudo ln -sf "$(pwd)/bazel-bin/src/cli/cli" /usr/local/bin/osmo
/usr/local/bin/osmo --help >/dev/null
- name: Run OETF deploy_and_run on KIND
id: deploy_and_run
run: |
# --build-local + --use-local-registry: build 9 OSMO service images
# + web-ui from PR source, push to a host-side `registry:2`
# container, KIND nodes pull from it on-demand. Replaces `kind
# load docker-image` which duplicates each image into every node's
# containerd content store (6 nodes × 10 images = 60 image-copies
# — overruns hosted ubuntu-latest's 145 GB disk). With registry
# mode each KIND node only pulls images its pods actually
# schedule (1-2 nodes per image under the chart's node_group
# selectors), dropping the multiplier from 6x to 1-2x.
#
# web-ui IS built and pushed: ingress-nginx has a hard
# `wait-for-web-ui` init container that polls osmo-ui:80, so
# scaling web-ui to 0 deadlocks the chart (verified empirically
# in run #35). Registry-push UI keeps the chart whole at ~1 GB
# host disk vs the prior ~5 GB kind-load duplication.
#
# "Free runner disk" step above reclaims 22-25 GB pre-deploy.
#
# Scenario set narrowed for PR-gate cost: smoke (api-checks +
# websocket-checks, ~5s) covers service+router+logger HTTP/WS;
# templates + mount-validation (~6s) submit workflows so
# backend-listener / backend-worker / osmo_ctrl runtime get
# touched. workflow-labels exercises the label policy gate
# (off/warn/enforce), label-syntax validation, list filters, and a
# labeled round-trip through the API.
#
# --jobs=1 serializes: chart's single-replica ingress-nginx 504s
# under parallel scenario submits.
#
# `time` writes total wall clock to the step log so regressions
# past the ~15-min PR-gate budget show up in the run timeline.
# EXIT trap so the timing block lands on the failure path too.
#
# --keep + --keep-on-failure: leave the cluster intact on either
# deploy or test failure so the "Diagnose KIND state on failure"
# step below can capture pod state.
trap '
echo "::group::deploy_and_run wall clock"
cat /tmp/oetf.time 2>/dev/null || echo "(no timing file written)"
echo "::endgroup::"
echo "::group::Disk space (post-run)"
df -h /
echo "::endgroup::"
' EXIT
# Keep this below the job's 60-minute ceiling. Without an inner
# deadline, a stuck Helm/Kubernetes wait consumes the whole job and
# GitHub kills it before the diagnostic and artifact steps can run.
# `timeout` returns 124, which is an ordinary step failure, leaving
# roughly ten minutes for cluster state and logs to be collected.
/usr/bin/time -p -o /tmp/oetf.time \
timeout --signal=TERM --kill-after=30s 40m \
bazel run //test/oetf:deploy_and_run -- \
--env kind --tags kind \
--target-pattern //test/smoke/... \
--target-pattern //test/scenarios:templates,//test/scenarios:mount-validation,//test/scenarios:workflow-labels \
--jobs 1 \
--build-local --use-local-registry \
--keep --keep-on-failure \
--output-json "${GITHUB_WORKSPACE}/kind-smoke-result.json"
- name: Diagnose KIND state on failure
if: failure() && steps.deploy_and_run.outcome == 'failure'
# Triggered only on deploy_and_run failure — preserves the cluster's
# post-mortem state for inspection BEFORE the cleanup step deletes
# it. Each subsection is wrapped in ::group:: so reviewers can fold
# what they don't need. `|| true` everywhere — diagnostic must
# never mask the real failure.
run: |
echo "::group::All pods (kubectl get pods -A -o wide)"
kubectl get pods -A -o wide || true
echo "::endgroup::"
echo "::group::Recent events (last 100, sorted by lastTimestamp)"
kubectl get events -A --sort-by=.lastTimestamp 2>/dev/null | tail -100 || true
echo "::endgroup::"
echo "::group::Non-Ready pod descriptions (osmo + kai-scheduler + ingress-nginx + kube-system)"
for ns in osmo kai-scheduler ingress-nginx kube-system; do
# `kubectl get pods` field-selector can't filter on .status.conditions; do it in jsonpath.
not_ready=$(kubectl get pods -n "$ns" -o jsonpath='{range .items[?(@.status.phase!="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
also_unready=$(kubectl get pods -n "$ns" -o jsonpath='{range .items[?(@.status.containerStatuses[*].ready==false)]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
for pod in $(printf '%s\n%s\n' "$not_ready" "$also_unready" | sort -u); do
[ -z "$pod" ] && continue
echo "--- $ns/$pod ---"
kubectl describe pod -n "$ns" "$pod" 2>/dev/null || true
done
done
echo "::endgroup::"
echo "::group::Non-Ready pod logs (last 200 lines, with --previous on crash)"
for ns in osmo kai-scheduler ingress-nginx; do
not_ready=$(kubectl get pods -n "$ns" -o jsonpath='{range .items[?(@.status.phase!="Running")]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
also_unready=$(kubectl get pods -n "$ns" -o jsonpath='{range .items[?(@.status.containerStatuses[*].ready==false)]}{.metadata.name}{"\n"}{end}' 2>/dev/null || true)
for pod in $(printf '%s\n%s\n' "$not_ready" "$also_unready" | sort -u); do
[ -z "$pod" ] && continue
echo "--- $ns/$pod (current) ---"
kubectl logs -n "$ns" "$pod" --all-containers --tail=200 2>/dev/null || true
echo "--- $ns/$pod (previous, if crash-looped) ---"
kubectl logs -n "$ns" "$pod" --all-containers --previous --tail=200 2>/dev/null || true
done
done
echo "::endgroup::"
echo "::group::All OSMO pod logs (last 80 lines)"
for pod in $(kubectl get pods -n osmo -o name 2>/dev/null || true); do
echo "--- osmo/$pod ---"
kubectl logs -n osmo "$pod" --all-containers --tail=80 2>/dev/null || true
done
echo "::endgroup::"
echo "::group::Helm releases (helm list -A)"
helm list -A || true
echo "::endgroup::"
echo "::group::Disk space"
df -h /
echo "::endgroup::"
- name: Cleanup KIND cluster
if: always()
run: |
kind delete clusters --all || true
- name: Write run summary to GitHub step summary
if: always()
# Surfaces the [PASS]/[FAIL] table on the PR check UI so reviewers
# see results without scrolling the 10-min step log. Reads from
# kind-smoke-result.json (always written by oetf:run, even on test
# failure) and falls back to a deploy-failure note if missing.
run: |
set -euo pipefail
{
echo "## OETF KIND results"
echo
if [ -s kind-smoke-result.json ]; then
# jq is preinstalled on ubuntu-latest runners.
total=$(jq -r '.total' kind-smoke-result.json)
passed=$(jq -r '.passed' kind-smoke-result.json)
failed=$(jq -r '.failed' kind-smoke-result.json)
errored=$(jq -r '.errored' kind-smoke-result.json)
skipped=$(jq -r '.skipped' kind-smoke-result.json)
echo "**Total ${total} Passed ${passed} Failed ${failed} Errors ${errored} Skipped ${skipped}**"
echo
echo "| Result | Test | Duration |"
echo "| --- | --- | --- |"
# `.target` is the bazel label (e.g. //test/smoke:api-checks); fall back to `.name` for ad-hoc tests.
jq -r '.results[] | "| \(.status|ascii_upcase) | \(if .target != "" then .target else .name end) | \(.time | tonumber | (. * 100 | round / 100))s |"' kind-smoke-result.json
else
echo "_No kind-smoke-result.json — deploy likely failed before tests ran. See \"Diagnose KIND state\" step above._"
fi
echo
if [ -s /tmp/oetf.time ]; then
echo "### Wall clock"
echo '```'
cat /tmp/oetf.time
echo '```'
fi
} >> "$GITHUB_STEP_SUMMARY"
- name: Upload OETF result JSON
if: always()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: kind-smoke-result
path: kind-smoke-result.json
if-no-files-found: warn
- name: Upload bazel test logs
if: failure()
uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
with:
name: bazel-testlogs
path: bazel-testlogs/
if-no-files-found: warn
# Warm-run trigger (no behavior change).