Skip to content

Commit 9d8e057

Browse files
AnouarMohamedAnouarMohamed
committed
ci(operator): centralize Kubernetes test versions
Signed-off-by: AnouarMohamed <m.anouar@mundiapolis.ma>
1 parent 81ef6be commit 9d8e057

13 files changed

Lines changed: 291 additions & 51 deletions

File tree

‎.github/workflows/agent-ci.yaml‎

Lines changed: 17 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ on:
2424
- '!agent/go/**'
2525
- containers/agent.Dockerfile
2626
- .github/actions/**
27+
- operator/k8s-test-versions.mk
2728
- .github/workflows/agent-ci.yaml
2829
push:
2930
branches:
@@ -35,6 +36,7 @@ on:
3536
- '!agent/go/**'
3637
- containers/agent.Dockerfile
3738
- .github/actions/**
39+
- operator/k8s-test-versions.mk
3840
- .github/workflows/agent-ci.yaml
3941
env:
4042
REGISTRY: ghcr.io
@@ -322,20 +324,31 @@ jobs:
322324
registry: ${{ env.REGISTRY }}
323325
username: ${{ github.actor }}
324326
password: ${{ secrets.GITHUB_TOKEN }}
327+
328+
- name: Load Kubernetes test versions
329+
id: k8s-versions
330+
run: |
331+
cd operator
332+
make print-k8s-test-versions-github-output >> "$GITHUB_OUTPUT"
333+
334+
- name: Validate KinD node image
335+
run: |
336+
cd operator
337+
make validate-kind-node-image KIND_NODE_IMAGE_VERSION=${{ steps.k8s-versions.outputs.kind-node-image-version }}
325338
326339
- name: Create Kubernetes KinD Cluster
327340
uses: helm/kind-action@v1.14.0
328341
with:
329-
version: v0.31.0
330-
node_image: kindest/node:v1.35.0
342+
version: ${{ steps.k8s-versions.outputs.kind-binary-version }}
343+
node_image: kindest/node:v${{ steps.k8s-versions.outputs.kind-node-image-version }}
331344
config: operator/config/local-dev/kind-config.yaml
332345
cluster_name: kind
333346

334347
- name: Restore cached Binaries
335348
id: cached-binaries
336349
uses: actions/cache/restore@v5
337350
with:
338-
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
351+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk', 'operator/k8s-test-versions.mk') }}
339352
restore-keys: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-
340353
path: |
341354
${{ github.workspace }}/operator/bin
@@ -351,7 +364,7 @@ jobs:
351364
if: steps.cached-binaries.outputs.cache-hit != 'true'
352365
uses: actions/cache/save@v5
353366
with:
354-
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
367+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk', 'operator/k8s-test-versions.mk') }}
355368
path: |
356369
${{ github.workspace }}/operator/bin
357370
~/.cache/go-build

‎.github/workflows/operator-ci.yaml‎

Lines changed: 41 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ on:
2525
- operator/go.mod
2626
- operator/go.sum
2727
- operator/deps.mk
28+
- operator/k8s-test-versions.mk
2829
- operator/config/**
2930
- containers/operator.Dockerfile
3031
- .github/actions/**
@@ -41,6 +42,7 @@ on:
4142
- operator/go.mod
4243
- operator/go.sum
4344
- operator/deps.mk
45+
- operator/k8s-test-versions.mk
4446
- operator/config/**
4547
- containers/operator.Dockerfile
4648
- .github/actions/**
@@ -59,11 +61,30 @@ env:
5961
# waiting for each action (checkout/setup-go/cache/upload-artifact) to
6062
# publish a Node 24 release.
6163
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
62-
KIND_BINARY_VERSION: v0.31.0
6364
PLATFORMS: linux/amd64,linux/arm64
6465
PUSH_TO_REGISTRY: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository }}
6566

6667
jobs:
68+
k8s-test-versions:
69+
name: Load Kubernetes Test Versions
70+
runs-on: ubuntu-latest
71+
permissions:
72+
contents: read
73+
outputs:
74+
envtest-k8s-version: ${{ steps.versions.outputs.envtest-k8s-version }}
75+
kind-version: ${{ steps.versions.outputs.kind-version }}
76+
kind-node-image-version: ${{ steps.versions.outputs.kind-node-image-version }}
77+
kind-binary-version: ${{ steps.versions.outputs.kind-binary-version }}
78+
ci-kind-node-image-versions-json: ${{ steps.versions.outputs.ci-kind-node-image-versions-json }}
79+
ci-primary-kind-node-image-version: ${{ steps.versions.outputs.ci-primary-kind-node-image-version }}
80+
steps:
81+
- uses: actions/checkout@v6
82+
- name: Load versions
83+
id: versions
84+
run: |
85+
cd operator
86+
make print-k8s-test-versions-github-output >> "$GITHUB_OUTPUT"
87+
6788
fetch-distroless-versions:
6889
name: Fetch Latest Distroless Versions
6990
runs-on: ubuntu-latest
@@ -85,27 +106,28 @@ jobs:
85106
# Test operator across supported Kubernetes versions and test suites
86107
tests:
87108
runs-on: ubuntu-latest
109+
needs: [k8s-test-versions]
88110
strategy:
89111
matrix:
90112
# Standard E2E tests on all supported K8s versions
91-
k8s-version: ["1.32.11", "1.33.7", "1.34.3", "1.35.0"]
113+
k8s-version: ${{ fromJson(needs.k8s-test-versions.outputs.ci-kind-node-image-versions-json) }}
92114
test-suite: ["e2e"]
93115
pool: ["core", "interrupt", "uninstall", "lifecycle"]
94116
make-targets: ["setup-kind-cluster e2e-tests"]
95117
include:
96-
# Deployment policy tests on 15-node cluster (K8s 1.35 only)
97-
- k8s-version: "1.35.0"
118+
# Deployment policy tests on 15-node cluster use the primary Kind node image.
119+
- k8s-version: ${{ needs.k8s-test-versions.outputs.ci-primary-kind-node-image-version }}
98120
test-suite: deployment-policy
99121
kind-config: k8s-tests/chainsaw/deployment-policy/kind-config.yaml
100122
make-targets: "setup-kind-cluster deployment-policy-tests"
101-
# CLI e2e tests on K8s 1.34 only
102-
- k8s-version: "1.35.0"
123+
# Focus the heavier suites on the primary Kind node image.
124+
- k8s-version: ${{ needs.k8s-test-versions.outputs.ci-primary-kind-node-image-version }}
103125
test-suite: cli-e2e
104126
make-targets: "setup-kind-cluster cli-e2e-tests"
105-
- k8s-version: "1.35.0"
127+
- k8s-version: ${{ needs.k8s-test-versions.outputs.ci-primary-kind-node-image-version }}
106128
test-suite: unit-tests
107129
make-targets: "vet lint unit-tests"
108-
- k8s-version: "1.35.0"
130+
- k8s-version: ${{ needs.k8s-test-versions.outputs.ci-primary-kind-node-image-version }}
109131
test-suite: helm-tests
110132
make-targets: "helm-tests"
111133
fail-fast: false # Continue testing other versions if one fails
@@ -132,7 +154,7 @@ jobs:
132154
id: cached-binaries
133155
uses: actions/cache/restore@v5
134156
with:
135-
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
157+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk', 'operator/k8s-test-versions.mk') }}
136158
restore-keys: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-
137159
path: |
138160
${{ github.workspace }}/operator/bin
@@ -147,34 +169,38 @@ jobs:
147169
if: steps.cached-binaries.outputs.cache-hit != 'true'
148170
uses: actions/cache/save@v5
149171
with:
150-
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk') }}
172+
key: ${{ env.GO_VERSION }}-${{ runner.os }}-${{ runner.arch }}-bin-${{ hashFiles('operator/deps.mk', 'operator/k8s-test-versions.mk') }}
151173
path: |
152174
${{ github.workspace }}/operator/bin
153175
~/.cache/go-build
176+
- name: Validate KinD node image v${{ matrix.k8s-version }}
177+
if: matrix.test-suite != 'unit-tests' # unit tests don't need a cluster
178+
run: |
179+
cd operator
180+
make validate-kind-node-image KIND_NODE_IMAGE_VERSION=${{ matrix.k8s-version }}
154181
- name: Create Kubernetes KinD Cluster v${{ matrix.k8s-version }}
155182
if: matrix.test-suite != 'unit-tests' && matrix.test-suite != 'helm-tests' # unit tests don't need a cluster; helm-tests uses ctlptl
156183
id: kind
157184
uses: helm/kind-action@v1.14.0
158185
with:
159-
version: ${{ env.KIND_BINARY_VERSION }}
186+
version: ${{ needs.k8s-test-versions.outputs.kind-binary-version }}
160187
node_image: kindest/node:v${{ matrix.k8s-version }}
161188
config: ${{ matrix.kind-config || 'operator/config/local-dev/kind-config.yaml' }}
162189
cluster_name: kind
163190
- name: Install kind
164191
if: matrix.test-suite == 'helm-tests'
192+
env:
193+
KIND_BINARY_VERSION: ${{ needs.k8s-test-versions.outputs.kind-binary-version }}
165194
run: |
166195
curl -fsSL -o /tmp/kind-linux-amd64 https://kind.sigs.k8s.io/dl/${KIND_BINARY_VERSION}/kind-linux-amd64
167196
curl -fsSL -o /tmp/kind-linux-amd64.sha256sum https://kind.sigs.k8s.io/dl/${KIND_BINARY_VERSION}/kind-linux-amd64.sha256sum
168197
cd /tmp && sha256sum -c kind-linux-amd64.sha256sum
169198
sudo install /tmp/kind-linux-amd64 /usr/local/bin/kind
170199
- name: Create ctlptl KinD Cluster
171-
if: matrix.test-suite == 'helm-tests'
172-
run: ./operator/bin/ctlptl apply -f operator/config/local-dev/ctlptl-config.yaml
173-
- name: Set up ctlptl KinD Cluster
174200
if: matrix.test-suite == 'helm-tests'
175201
run: |
176202
cd operator
177-
make setup-kind-cluster
203+
make create-kind-cluster KIND_NODE_IMAGE_VERSION=${{ matrix.k8s-version }}
178204
# Run test suite
179205
- name: Run ${{ matrix.test-suite }} tests
180206
env:

‎docs/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,9 @@ This directory contains user and operator documentation for Skyhook. Here you'll
3333
- [Webhook Bootstrap Lease](designs/webhook-bootstrap-lease.md): Why the admission-webhook cert bootstrap runs on a dedicated leader-election lease, and the runbook for upgrading from v0.7.x (where the new lease does not yet exist).
3434

3535
- **Process**
36+
- [Local Development](development.md):
37+
Developer workflow notes, including local kind clusters and Kubernetes test version ownership.
38+
3639
- [Releases](releases.md):
3740
Release notes and upgrade information for Skyhook.
3841

‎docs/development.md‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,24 @@ brew install tilt-dev/tap/ctlptl
1818

1919
Keep manual installs in sync with `CTLPTL_VERSION` in `operator/deps.mk`.
2020

21+
## Kubernetes Test Versions
22+
23+
`operator/k8s-test-versions.mk` is the source of truth for Kubernetes versions used by envtest, local kind clusters, and GitHub Actions test matrices.
24+
25+
- `ENVTEST_K8S_VERSION` controls `setup-envtest` binary assets under `operator/bin/k8s/`.
26+
- `KIND_NODE_IMAGE_VERSION` controls the default `kindest/node` image for local kind clusters. `KIND_VERSION` remains as a backward-compatible alias for local overrides.
27+
- `KIND_BINARY_VERSION` controls the Kind CLI/action version and is intentionally separate from Kubernetes node image versions.
28+
- `CI_KIND_NODE_IMAGE_VERSIONS_JSON` and `CI_PRIMARY_KIND_NODE_IMAGE_VERSION` feed the GitHub Actions matrix.
29+
30+
These versions often match, but they are allowed to diverge. Kind does not publish every Kubernetes patch version as a `kindest/node` image, so validate a new node image before using it:
31+
32+
```bash
33+
cd operator
34+
make validate-kind-node-image KIND_NODE_IMAGE_VERSION=1.35.0
35+
```
36+
37+
When updating supported test versions, edit `operator/k8s-test-versions.mk` first, then run the validation target and the relevant tests.
38+
2139
## Local Cluster
2240

2341
Bring up the kind cluster and local registry:
@@ -27,7 +45,9 @@ cd operator
2745
make create-kind-cluster
2846
```
2947

30-
`make create-kind-cluster` applies `config/local-dev/ctlptl-config.yaml`, which creates the kind cluster and a local registry at `localhost:5005`. It also runs `make setup-kind-cluster`, which is idempotent and labels the test node and creates the `skyhook` namespace pull secret.
48+
`make create-kind-cluster` renders `config/local-dev/ctlptl-config.yaml` into `reporting/ctlptl-config.yaml`, deletes any existing cluster defined by that rendered config, and then creates the kind cluster plus a local registry at `localhost:5005`. It also runs `make setup-kind-cluster`, which is idempotent and labels the test node and creates the `skyhook` namespace pull secret.
49+
50+
Do not run `ctlptl apply -f config/local-dev/ctlptl-config.yaml` directly; the tracked file is a Make-rendered template.
3151

3252
## Webhook Iteration
3353

@@ -42,7 +62,7 @@ The local registry removes the need for `kind load docker-image` or registry-pin
4262

4363
## Teardown
4464

45-
Delete the cluster and registry with the same ctlptl config:
65+
Delete the cluster and registry with the rendered ctlptl config:
4666

4767
```bash
4868
cd operator

‎docs/kubernetes-support.md‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ This document outlines Skyhook's approach to supporting different Kubernetes ver
4141
**For new Kubernetes releases:**
4242

4343
1. Wait **4+ weeks** after K8s release for ecosystem stability
44-
2. Add to our CI testing matrix
44+
2. Add to the CI testing matrix in `operator/k8s-test-versions.mk`
4545
3. Include in next Skyhook release
4646

4747
**For EOL Kubernetes versions:**
@@ -120,13 +120,12 @@ Waiting 4+ weeks lets the ecosystem stabilize and gives us confidence in support
120120
### How do you test compatibility?
121121

122122
For each Skyhook release, we test against all supported Kubernetes versions using:
123-
124-
- GitHub Actions matrix builds with multiple K8s versions (currently 1.31, 1.32, 1.33, 1.34, 1.35)
123+
- GitHub Actions matrix builds with multiple K8s versions. The exact tested patch versions are owned by `CI_KIND_NODE_IMAGE_VERSIONS_JSON` in `operator/k8s-test-versions.mk`.
125124
- Local testing with [kind](https://kind.sigs.k8s.io/)
126125
- Basic functionality and integration tests
127126

128127
## Notes
129128

130129
This is a living document that will evolve as the project grows. Our current approach supports all actively maintained Kubernetes versions (typically 3 versions) while providing reasonable predictability for users.
131130

132-
For questions about Kubernetes support, please open an issue in our [GitHub repository](https://github.com/NVIDIA/skyhook).
131+
For questions about Kubernetes support, please open an issue in our [GitHub repository](https://github.com/NVIDIA/skyhook).

‎operator/Makefile‎

Lines changed: 45 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@ IMG ?= $(IMG_REPO)/operator
2727
LOCAL_REGISTRY ?= localhost:5005
2828
LOCAL_OPERATOR_IMG ?= $(LOCAL_REGISTRY)/skyhook-operator:testing
2929

30-
## default Kubernetes version for kind-based development clusters
31-
KIND_VERSION ?= 1.36.0
32-
3330
PLATFORM := $(shell uname -s 2>/dev/null || echo unknown)
3431
SKYHOOK_NAMESPACE ?= skyhook
3532

@@ -154,6 +151,44 @@ reporting: $(REPORTING)
154151
$(REPORTING):
155152
mkdir -p $@
156153

154+
CTLPTL_CONFIG_TEMPLATE ?= config/local-dev/ctlptl-config.yaml
155+
CTLPTL_CONFIG_RENDERED ?= $(REPORTING)/ctlptl-config.yaml
156+
157+
.PHONY: print-k8s-test-versions-github-output
158+
print-k8s-test-versions-github-output: ## Print Kubernetes test versions in GitHub Actions output format.
159+
@printf 'envtest-k8s-version=%s\n' '$(ENVTEST_K8S_VERSION)'
160+
@printf 'kind-version=%s\n' '$(KIND_VERSION)'
161+
@printf 'kind-node-image-version=%s\n' '$(KIND_NODE_IMAGE_VERSION)'
162+
@printf 'kind-binary-version=%s\n' '$(KIND_BINARY_VERSION)'
163+
@printf 'ci-kind-node-image-versions-json=%s\n' '$(CI_KIND_NODE_IMAGE_VERSIONS_JSON)'
164+
@printf 'ci-primary-kind-node-image-version=%s\n' '$(CI_PRIMARY_KIND_NODE_IMAGE_VERSION)'
165+
166+
.PHONY: validate-kind-node-image
167+
validate-kind-node-image: ## Check that the configured kindest/node image tag exists before creating a cluster.
168+
@echo "Validating kindest/node:v$(KIND_NODE_IMAGE_VERSION)"
169+
@if command -v $(DOCKER_CMD) >/dev/null 2>&1; then \
170+
if [ "$(DOCKER_CMD)" = "podman" ]; then \
171+
$(DOCKER_CMD) manifest inspect docker://kindest/node:v$(KIND_NODE_IMAGE_VERSION) >/dev/null; \
172+
else \
173+
$(DOCKER_CMD) manifest inspect kindest/node:v$(KIND_NODE_IMAGE_VERSION) >/dev/null; \
174+
fi; \
175+
elif command -v docker >/dev/null 2>&1; then \
176+
docker manifest inspect kindest/node:v$(KIND_NODE_IMAGE_VERSION) >/dev/null; \
177+
elif command -v podman >/dev/null 2>&1; then \
178+
podman manifest inspect docker://kindest/node:v$(KIND_NODE_IMAGE_VERSION) >/dev/null; \
179+
else \
180+
curl -fsSL https://hub.docker.com/v2/repositories/kindest/node/tags/v$(KIND_NODE_IMAGE_VERSION) >/dev/null; \
181+
fi || { \
182+
echo "kindest/node:v$(KIND_NODE_IMAGE_VERSION) is not published or cannot be inspected."; \
183+
echo "Update KIND_NODE_IMAGE_VERSION in k8s-test-versions.mk, or override it with a published kindest/node tag."; \
184+
exit 1; \
185+
}
186+
187+
.PHONY: render-ctlptl-config
188+
render-ctlptl-config: reporting ## Render local ctlptl config using KIND_NODE_IMAGE_VERSION.
189+
sed 's|@@KIND_NODE_IMAGE_VERSION@@|$(KIND_NODE_IMAGE_VERSION)|g' $(CTLPTL_CONFIG_TEMPLATE) > $(CTLPTL_CONFIG_RENDERED)
190+
@echo "$(CTLPTL_CONFIG_RENDERED)"
191+
157192
.PHONY: test
158193
test:: reporting manifests generate fmt vet lint build-cli unit-tests e2e-tests cli-e2e-tests helm-tests operator-agent-tests ## Run all tests.
159194

@@ -267,8 +302,8 @@ rollout-local: push-local-image ## push locally-built operator and restart the d
267302

268303

269304
.PHONY: delete-kind-cluster
270-
delete-kind-cluster: ctlptl ## delete the local ctlptl kind cluster and registry
271-
$(CTLPTL) delete -f config/local-dev/ctlptl-config.yaml
305+
delete-kind-cluster: ctlptl render-ctlptl-config ## delete the local ctlptl kind cluster and registry
306+
$(CTLPTL) delete -f $(CTLPTL_CONFIG_RENDERED)
272307

273308
operator-agent-tests: chainsaw install ## Run operator agent tests.
274309
@if [ -z "$(AGENT_IMAGE)" ]; then \
@@ -292,7 +327,7 @@ deployment-policy-tests: chainsaw install run ensure-test-symlinks ## Run all de
292327
go tool covdata textfmt -i=$(REPORTING)/int -o reporting/int.coverprofile
293328

294329
.PHONY: create-deployment-policy-cluster
295-
create-deployment-policy-cluster: ## Create a 15-node Kind cluster for deployment policy tests.
330+
create-deployment-policy-cluster: validate-kind-node-image ## Create a 15-node Kind cluster for deployment policy tests.
296331
@echo "🔧 Creating 15-node Kind cluster for deployment policy tests..."
297332
@# Fix for Podman: increase kernel keyring quota
298333
@if command -v podman >/dev/null 2>&1; then \
@@ -306,7 +341,7 @@ create-deployment-policy-cluster: ## Create a 15-node Kind cluster for deploymen
306341
fi; \
307342
fi
308343
kind delete cluster --name skyhook-dp-test 2>/dev/null || true
309-
kind create cluster --name skyhook-dp-test --config ../k8s-tests/chainsaw/deployment-policy/kind-config.yaml --image=kindest/node:v$(KIND_VERSION)
344+
kind create cluster --name skyhook-dp-test --config ../k8s-tests/chainsaw/deployment-policy/kind-config.yaml --image=kindest/node:v$(KIND_NODE_IMAGE_VERSION)
310345

311346
.PHONY: delete-deployment-policy-cluster
312347
delete-deployment-policy-cluster: ## Delete the 15-node Kind cluster for deployment policy tests.
@@ -331,9 +366,9 @@ setup-kind-cluster: ## setup kind cluster with local docker creds and skyhook na
331366
fi
332367

333368

334-
create-kind-cluster: ctlptl ## deletes any existing kind cluster from ctlptl-config.yaml and creates a fresh one.
335-
$(CTLPTL) delete -f config/local-dev/ctlptl-config.yaml || true
336-
$(CTLPTL) apply -f config/local-dev/ctlptl-config.yaml
369+
create-kind-cluster: ctlptl validate-kind-node-image render-ctlptl-config ## creates or updates a kind cluster.
370+
$(CTLPTL) delete -f $(CTLPTL_CONFIG_RENDERED) || true
371+
$(CTLPTL) apply -f $(CTLPTL_CONFIG_RENDERED)
337372
$(MAKE) setup-kind-cluster
338373

339374
podman-create-machine: ## creates a podman machine

‎operator/README.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -163,7 +163,7 @@ Development
163163
watch-tests watch unit tests and auto run on changes.
164164
unit-tests Run unit tests.
165165
e2e-tests Run end to end tests.
166-
create-kind-cluster deletes and creates a new kind cluster. versions is set via KIND_VERSION
166+
create-kind-cluster deletes and creates a new kind cluster. versions are set via KIND_NODE_IMAGE_VERSION
167167
podman-create-machine creates a podman machine
168168
lint Run golangci-lint linter & yamllint
169169
lint-fix Run golangci-lint linter and perform fixes

0 commit comments

Comments
 (0)