diff --git a/.ci/generate-e2e-matrix.sh b/.ci/generate-e2e-matrix.sh new file mode 100755 index 000000000..f684f56da --- /dev/null +++ b/.ci/generate-e2e-matrix.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -Eeuo pipefail + +# generate-e2e-matrix.sh - converts test/e2e/pict/generated-matrix.tsv into a +# GitHub Actions `strategy.matrix.include` JSON array. +# +# Each row is enriched here with the concrete invocation details its +# Scenario implies (which suite, which GINKGO_FOCUS, which env vars) so the +# consuming workflow step stays a generic dispatcher instead of duplicating +# scenario-specific logic once per matrix case. +# +# Usage: generate-e2e-matrix.sh +# Writes "matrix=" to $GITHUB_OUTPUT if set, otherwise prints the JSON +# to stdout (for local debugging via `make generate-e2e-matrix-json`). + +# Ginkgo v2's -ginkgo.focus reports "SUCCESS!" and exits 0 when it matches +# zero specs (verified directly: a typo'd focus runs 0 of N specs and still +# passes) -- a Describe() tag renamed in a spec file without updating the +# case statement above would silently turn a matrix case into a no-op that +# reports green forever. Catch that here, once, before the expensive matrix +# job starts, by checking every focus value actually appears in the source +# it's supposed to select. +verify_focus_values_are_real() { + local json=$1 + local suite focus dir stale=0 + + while IFS= read -r obj; do + suite=$(jq -r '.suite' <<<"${obj}") + focus=$(jq -r '.ginkgo_focus' <<<"${obj}") + [[ -z "${focus}" ]] && continue + dir="test/e2e" + [[ "${suite}" == "packaging" ]] && dir="test/e2e/packaging" + + # @tsv would double-escape the backslashes in a \[Tag\] focus regex, so + # fields are pulled directly off each object above instead. Unescape + # \[ / \] back to literal [ / ] to grep for the plain Describe() text. + local literal=${focus//\\[/[} + literal=${literal//\\]/]} + + if ! grep -rFq -- "${literal}" "${dir}"/*.go; then + echo "generate-e2e-matrix.sh: GINKGO_FOCUS '${focus}' (literal: '${literal}') matches no Describe() under ${dir}/*.go -- stale mapping in this script" >&2 + stale=1 + fi + done < <(jq -c '.[]' <<<"${json}") + + if ((stale)); then + echo "generate-e2e-matrix.sh: refusing to generate a matrix with stale focus values -- a case above would silently run 0 specs and report success" >&2 + exit 1 + fi +} + +main() { + local tsv=$1 + local rows="[]" + + while IFS=$'\t' read -r scenario k8s_version; do + local suite="e2e" ginkgo_focus="" kubernetes_version="" \ + e2e_k8s_version_from="" e2e_k8s_version_to="" + + case "${scenario}" in + Join) ginkgo_focus='\[PR-Blocking\]' ;; + Installer) ginkgo_focus='\[Installer\]' ;; + ByoHCtl) ginkgo_focus='\[Byohctl\]' ;; + Reuse) ginkgo_focus='\[Reuse\]' ;; + ClusterClass) ginkgo_focus='\[Cluster-Class\]' ;; + MDScale) ginkgo_focus='\[MD-Scale\]' ;; + UpgradeCluster) ginkgo_focus='\[K8s-Upgrade-Cluster\]' ;; + UpgradeClusterClass) ginkgo_focus='\[K8s-Upgrade-ClusterClass\]' ;; + PackagingDeb) suite="packaging"; ginkgo_focus="pf9-byohost deb" ;; + PackagingRpm) suite="packaging"; ginkgo_focus="pf9-byohost RPM" ;; + *) + echo "generate-e2e-matrix.sh: unknown Scenario '${scenario}' in ${tsv}" >&2 + exit 1 + ;; + esac + + # e2e_suite_test.go's SynchronizedBeforeSuite builds a local k8s bundle + # for KUBERNETES_VERSION unconditionally, before any spec runs, + # regardless of GINKGO_FOCUS -- confirmed by running this: leaving it + # unset for ByoHCtl/UpgradeCluster/UpgradeClusterClass failed the whole + # suite's setup with 'unexpected Kubernetes version format ""', even + # though none of those three scenarios read KUBERNETES_VERSION in their + # own spec body. clusterctl's GetVariableOrEmpty (unlike this repo's own + # getEnvOrDefault, used for E2E_K8S_VERSION_FROM/_TO) treats an + # explicitly-empty env var as set, so it can't be left blank the way + # those two safely can. Every suite=="e2e" row needs a real value here; + # only suite=="packaging" rows (a separate Go test binary, no + # SynchronizedBeforeSuite) are exempt. + if [[ "${suite}" == "e2e" ]]; then + if [[ "${k8s_version}" == "NA" ]]; then + kubernetes_version="v1.31.0" + else + kubernetes_version="${k8s_version}" + fi + fi + + case "${scenario}" in + UpgradeCluster | UpgradeClusterClass) + e2e_k8s_version_from="${k8s_version}" + # model.pict has no upgrade-target column -- it has no independent + # freedom to cross (see the model's own comment) -- so this is the + # one place the target is decided, matching this repo's current + # real default (cluster_upgrade_test.go/clusterclass_upgrade_test.go's + # E2E_K8S_VERSION_TO default). + e2e_k8s_version_to="v1.31.2" + ;; + esac + + # GitHub Actions auto-names a matrix job by concatenating every field in + # its object -- without this, the job title leaks the raw GINKGO_FOCUS + # regex (e.g. Join's is literally the pre-existing, unrelated + # "[PR-Blocking]" spec tag from e2e_test.go), which reads as if this + # gated, optional matrix were blocking something. label is what the + # workflow's job `name:` displays instead. + job_label="${scenario}" + [[ "${suite}" == "e2e" ]] && job_label="${scenario} (${kubernetes_version})" + + # jq's `label $out | ...`/`break $out` control-flow keyword makes + # $label itself unparseable as a --arg/variable name (confirmed: even + # `jq -n --arg label 1 '$label'` alone fails on jq 1.6, the version + # this repo's CI runners have -- unrelated to whether the resulting + # object *key* is named "label", which works fine either way). Named + # job_label here to avoid that, independent of the "label" JSON field + # name below. + row=$(jq -nc \ + --arg scenario "${scenario}" \ + --arg suite "${suite}" \ + --arg ginkgo_focus "${ginkgo_focus}" \ + --arg kubernetes_version "${kubernetes_version}" \ + --arg e2e_k8s_version_from "${e2e_k8s_version_from}" \ + --arg e2e_k8s_version_to "${e2e_k8s_version_to}" \ + --arg job_label "${job_label}" \ + '{scenario: $scenario, suite: $suite, ginkgo_focus: $ginkgo_focus, + kubernetes_version: $kubernetes_version, + e2e_k8s_version_from: $e2e_k8s_version_from, + e2e_k8s_version_to: $e2e_k8s_version_to, + label: $job_label}') + + rows=$(jq -c --argjson row "${row}" '. + [$row]' <<<"${rows}") + done < <(tail -n +2 "${tsv}") + + local json + json=$(jq -c '.' <<<"${rows}") + echo "generated $(jq 'length' <<<"${json}") matrix cases from ${tsv}" >&2 + + verify_focus_values_are_real "${json}" + + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "matrix=${json}" >>"${GITHUB_OUTPUT}" + else + echo "${json}" + fi +} + +main "$@" diff --git a/.github/workflows/e2e-matrix.yml b/.github/workflows/e2e-matrix.yml new file mode 100644 index 000000000..bda7b3683 --- /dev/null +++ b/.github/workflows/e2e-matrix.yml @@ -0,0 +1,169 @@ +# Copyright 2026 Platform9, Inc. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Dynamic, PICT-driven e2e matrix. Deliberately workflow_dispatch-only for +# now, not a default PR/push gate -- this exists to measure real job count +# and wall-clock against test/e2e/pict/generated-matrix.tsv's 10 cases +# before deciding whether/how it replaces the single-case e2e/packaging +# jobs in e2e.yml. + +name: e2e-matrix + +on: + # TEMPORARY, for this PR's own review only: workflow_dispatch can't + # dispatch a workflow that doesn't exist on the default branch yet, so + # there's no way to get a real execution of a brand-new workflow_dispatch + # workflow before it merges except by triggering it some other way once. + # Remove this pull_request trigger again before merging -- see the plan + # this PR is part of for why workflow_dispatch-only is the intended, + # permanent state. + pull_request: {} + workflow_dispatch: {} + +permissions: + contents: read + +jobs: + generate-matrix: + runs-on: ubuntu-22.04 + outputs: + matrix: ${{ steps.generate.outputs.matrix }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Generate matrix from PICT output + id: generate + run: bash .ci/generate-e2e-matrix.sh test/e2e/pict/generated-matrix.tsv + + # Builds the controller manager image and the agent .deb bundle once, + # shared by every matrix case below via artifacts -- the same pattern + # e2e.yml already uses across build-controller-manager.yml/ + # build-agent-bundle.yml, just within this one workflow since those two + # don't trigger on workflow_dispatch. + build: + runs-on: ubuntu-22.04 + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Build controller manager image + run: bash .ci/build-controller-manager.sh + + - name: Save controller manager image artifact + run: | + TAG=$(make tag) + docker save "quay.io/platform9/cluster-api-provider-bringyourownhost/controller-manager:${TAG}" -o controller-manager-image.tar + + - name: Upload controller manager image artifact + uses: actions/upload-artifact@v4 + with: + name: controller-manager-image + path: controller-manager-image.tar + retention-days: 1 + + - name: Install fpm build dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends ruby ruby-dev rubygems build-essential + sudo gem install --no-document fpm + + - name: Build agent .deb bundle + run: make build-host-agent-deb + + - name: Upload agent bundle artifact + uses: actions/upload-artifact@v4 + with: + name: agent-bundle-deb + path: build/pf9-byohost/debsrc/ + retention-days: 1 + + test: + name: test (${{ matrix.label }}) + needs: [ generate-matrix, build ] + runs-on: ubuntu-22.04 + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.generate-matrix.outputs.matrix) }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Set up Go + uses: actions/setup-go@v7 + with: + go-version-file: go.mod + cache: true + + - name: Download controller manager image artifact + if: ${{ matrix.suite == 'e2e' }} + uses: actions/download-artifact@v4 + with: + name: controller-manager-image + path: . + + - name: Load controller manager image + if: ${{ matrix.suite == 'e2e' }} + run: | + docker load -i controller-manager-image.tar + TAG=$(make tag) + IMAGE=quay.io/platform9/cluster-api-provider-bringyourownhost/controller-manager + docker tag "${IMAGE}:${TAG}" "${IMAGE}:dev" + + - name: Download agent bundle artifact + if: ${{ matrix.suite == 'e2e' }} + uses: actions/download-artifact@v4 + with: + name: agent-bundle-deb + path: build/pf9-byohost/debsrc + + - name: turn off swap + run: sudo swapoff -a + + - name: Set netfilter conntrack max + run: sudo sysctl -w net.netfilter.nf_conntrack_max=131072 + + - name: Install rpmbuild + if: ${{ matrix.suite == 'packaging' }} + run: sudo apt-get update && sudo apt-get install -y rpm + + - name: Install fpm build dependencies + if: ${{ matrix.suite == 'packaging' }} + run: | + sudo apt-get install -y --no-install-recommends ruby ruby-dev rubygems build-essential + sudo gem install --no-document fpm + + - name: Run e2e scenario + if: ${{ matrix.suite == 'e2e' }} + env: + SKIP_BUILD: "1" + GINKGO_FOCUS: ${{ matrix.ginkgo_focus }} + KUBERNETES_VERSION: ${{ matrix.kubernetes_version }} + E2E_K8S_VERSION_FROM: ${{ matrix.e2e_k8s_version_from }} + E2E_K8S_VERSION_TO: ${{ matrix.e2e_k8s_version_to }} + run: yes | GINKGO_NODES=1 make test-e2e + + - name: Run packaging scenario + if: ${{ matrix.suite == 'packaging' }} + env: + PACKAGING_GINKGO_FOCUS: ${{ matrix.ginkgo_focus }} + run: make test-packaging + + - name: Upload e2e artifacts + if: ${{ failure() && matrix.suite == 'e2e' }} + uses: actions/upload-artifact@v4 + with: + name: e2e-artifacts-${{ matrix.scenario }} + path: _artifacts/ + retention-days: 5 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9786643df..b7ade023f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,25 +1,12 @@ name: e2e-suite +# push/pull_request triggers disabled while the PICT-driven e2e-matrix.yml +# (workflow_dispatch-gated) is being evaluated as this suite's replacement, +# to stop paying for both on every PR. Still runnable manually via +# workflow_dispatch. Re-add the triggers below if e2e-matrix.yml doesn't +# pan out, or remove this workflow once it's promoted (see the PICT e2e +# refactor plan's PR 9). on: - push: - branches: [ main ] - tags: [ 'ci-*' ] - paths-ignore: - - '*.md' - - 'docs/**' - - 'LICENSE' - - 'NOTICE' - - 'PROJECT' - - 'SECURITY_CONTACTS' - pull_request: - types: [opened, synchronize, reopened, ready_for_review] - paths-ignore: - - '*.md' - - 'docs/**' - - 'LICENSE' - - 'NOTICE' - - 'PROJECT' - - 'SECURITY_CONTACTS' workflow_dispatch: {} permissions: diff --git a/.github/workflows/generated-drift.yml b/.github/workflows/generated-drift.yml index 98c0d8162..c5e0f1e86 100644 --- a/.github/workflows/generated-drift.yml +++ b/.github/workflows/generated-drift.yml @@ -30,11 +30,12 @@ jobs: make manifests make cluster-templates-v1beta1 make cluster-templates-e2e + make generate-pict - name: Check for uncommitted generated changes run: | if [[ -n "$(git status --porcelain)" ]]; then - echo "::error::Generated files are out of date. Run 'make generate manifests cluster-templates-v1beta1 cluster-templates-e2e' and commit the result." + echo "::error::Generated files are out of date. Run 'make generate manifests cluster-templates-v1beta1 cluster-templates-e2e generate-pict' and commit the result." git status --porcelain git diff exit 1 diff --git a/Makefile b/Makefile index c49d7a631..7f63db0bb 100644 --- a/Makefile +++ b/Makefile @@ -20,6 +20,8 @@ BYOH_BASE_IMG = byoh/node:e2e BYOH_BASE_IMG_DEV = byoh/node:dev LINUX_VM_IMG = byoh/linux-test-runner:dev PACKAGING_TEST_RPM_IMG = byoh/packaging-test-rocky:dev +PICT_IMG = byoh/pict:dev +PICT_DIR = $(REPO_ROOT)/test/e2e/pict # Path to the podman machine's own socket, from inside the VM (not the macOS-side # forwarding socket at /var/run/docker.sock). See the *-linux-vm targets below. LINUX_VM_PODMAN_SOCK ?= /run/podman/podman.sock @@ -225,8 +227,9 @@ prepare-byoh-docker-host-image: build-packaging-test-image: ## Build the Rocky Linux image used by test-packaging docker build test/e2e/packaging -f test/e2e/packaging/RockyDockerFile -t $(PACKAGING_TEST_RPM_IMG) +PACKAGING_GINKGO_FOCUS ?= test-packaging: build-packaging-test-image prepare-byoh-docker-host-image ## Run the pf9-byohost RPM/deb install/uninstall tests - go test ./test/e2e/packaging/... -v -timeout 5m + go test ./test/e2e/packaging/... -v -timeout 5m -args -ginkgo.focus="$(PACKAGING_GINKGO_FOCUS)" prepare-byoh-docker-host-image-dev: docker build test/e2e -f docs/BYOHDockerFileDev -t ${BYOH_BASE_IMG_DEV} @@ -235,6 +238,22 @@ cluster-templates-v1beta1: kustomize ## Generate cluster templates for v1beta1 $(KUSTOMIZE) build $(BYOH_TEMPLATES)/v1beta1/templates/vm --load-restrictor LoadRestrictionsNone > $(BYOH_TEMPLATES)/v1beta1/templates/vm/cluster-template.yaml $(KUSTOMIZE) build $(BYOH_TEMPLATES)/v1beta1/templates/docker --load-restrictor LoadRestrictionsNone > $(BYOH_TEMPLATES)/v1beta1/templates/docker/cluster-template.yaml +##@ PICT + +pict-image: ## Build the PICT (pairwise combinatorial test generator) image from its Homebrew bottle + docker build -f hack/docker/pict.Dockerfile -t $(PICT_IMG) hack/docker + +# /o:1 (each value covered at least once) instead of PICT's default /o:2 +# (every pair covered): each row is a full e2e run, and with only one +# real parameter besides Scenario today, pairwise coverage of exactly two +# parameters is the same thing as the full cross product -- no +# combinatorial saving over a naive nested loop, just more rows. Revisit +# once model.pict grows a third free parameter (e.g. OS, once a second +# real image exists), where pairwise actually starts saving cases. +PICT_OPTS = /o:1 +generate-pict: pict-image ## Regenerate test/e2e/pict/generated-matrix.tsv from model.pict + docker run --rm -v $(PICT_DIR):/var/pict:Z $(PICT_IMG) model.pict $(PICT_OPTS) > $(PICT_DIR)/generated-matrix.tsv + ##@ Test # Run tests diff --git a/hack/docker/pict.Dockerfile b/hack/docker/pict.Dockerfile new file mode 100644 index 000000000..ec206ec2a --- /dev/null +++ b/hack/docker/pict.Dockerfile @@ -0,0 +1,12 @@ +# Copyright 2026 Platform9, Inc. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# microsoft/pict (https://github.com/microsoft/pict) publishes no Linux +# binary or container image of its own -- only source and a Windows .exe. +# homebrew-core ships a prebuilt Linux bottle for it, so this installs that +# instead of compiling PICT's C++ sources ourselves. +FROM homebrew/brew:4.6.20 +ENV HOMEBREW_NO_AUTO_UPDATE=1 HOMEBREW_NO_INSTALL_CLEANUP=1 HOMEBREW_NO_ENV_HINTS=1 +RUN brew install pict +WORKDIR /var/pict +ENTRYPOINT ["pict"] diff --git a/test/e2e/pict/generated-matrix.tsv b/test/e2e/pict/generated-matrix.tsv new file mode 100644 index 000000000..66039c9f8 --- /dev/null +++ b/test/e2e/pict/generated-matrix.tsv @@ -0,0 +1,11 @@ +Scenario K8sVersion +Reuse v1.33.2 +PackagingDeb NA +ByoHCtl NA +Installer v1.31.0 +UpgradeCluster v1.31.0 +MDScale v1.35.6 +Join v1.32.3 +PackagingRpm NA +UpgradeClusterClass v1.31.0 +ClusterClass v1.34.9 diff --git a/test/e2e/pict/model.pict b/test/e2e/pict/model.pict new file mode 100644 index 000000000..640671516 --- /dev/null +++ b/test/e2e/pict/model.pict @@ -0,0 +1,46 @@ +# Copyright 2026 Platform9, Inc. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# PICT (https://github.com/microsoft/pict) model. Regenerate +# generated-matrix.tsv with `make generate-pict` -- never hand-edit it, +# generated-drift.yml enforces that in CI. +# +# One unified model instead of one per scenario: install-method and +# template-flavor differences already collapse into which Scenario value a +# row picks, not a separately-crossed variable. +# +# Deliberately narrow: only parameters that actually vary independently of +# Scenario today are modeled -- +# - OS: every scenario runs a single, fixed image today (Ubuntu 22.04 for +# everything via spinUpByoHosts/ByoHostRunner/kindImage and byohctl, +# Rocky 9 only for PackagingRpm's separate provisioning path). 100% +# determined by Scenario, not a free-crossing dimension -- modeling it +# would just mirror the Scenario column back at itself. Reintroduce +# once a second real image exists for the spinUpByoHosts path. +# - IPFamily: only IPv4 is supported end-to-end today; IPv6/Dual are a +# future PR's networking work, not just this matrix's. +# - Upgrade target version: for UpgradeCluster/UpgradeClusterClass, +# K8sVersion is fixed to the one real starting version below, so the +# target has no independent freedom (always the same value) and isn't +# a second column. + +Scenario: Join, Installer, ByoHCtl, Reuse, ClusterClass, UpgradeCluster, UpgradeClusterClass, MDScale, PackagingDeb, PackagingRpm +# The non-NA values are one real, confirmed-pullable tag per minor line +# this repo's default bundle registry +# (quay.io/platform9/byoh-bundle-ubuntu_22.04_x86-64_k8s) publishes today +# -- latest patch for each minor, except v1.31.0 (this repo's own literal +# default, also the fixed upgrade-from version below, kept instead of the +# newer v1.31.2 patch so 1.31 isn't represented twice). A tag existing +# doesn't guarantee this repo's pinned CAPI/kubeadm providers can +# bootstrap that version yet; that's for whoever runs this matrix to find +# out, same as any other combo here. +K8sVersion: v1.31.0, v1.32.3, v1.33.2, v1.34.9, v1.35.6, NA + +IF [Scenario] IN {"ByoHCtl", "PackagingDeb", "PackagingRpm"} + THEN [K8sVersion] = "NA"; + +IF [Scenario] IN {"UpgradeCluster", "UpgradeClusterClass"} + THEN [K8sVersion] = "v1.31.0"; + +IF [Scenario] IN {"Join", "Installer", "Reuse", "ClusterClass", "MDScale"} + THEN [K8sVersion] <> "NA";