test(e2e): run the e2e host agent under systemd, not bare docker exec

test/e2e/agent_upgrade_fixture_helper_test.go

@@ -0,0 +1,165 @@
+// Copyright 2026 Platform9, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// nolint: testpackage
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+	goruntime "runtime"
+	"strings"
+
+	"github.com/docker/docker/client"
+	"github.com/onsi/gomega/gexec"
+)
+
+const (
+	fixtureAgentDebPackageName = "pf9-byohost-agent-fixture"
+	fixtureAgentDebFileName    = "pf9-byohost-agent.deb"
+)
+
+// buildFixtureAgentBinary builds a real BYOH host agent binary with version.GitVersion baked in
+// as gitVersion -- for use as a self-upgrade source/target in the agent-upgrade e2e specs. Reuses
+// the same build flags e2e_suite_test.go's SynchronizedBeforeSuite uses for the suite's own main
+// agent binary, just with a different, test-chosen version string, so the installed fixture is a
+// fully working agent, not a stand-in.
+func buildFixtureAgentBinary(gitVersion string) (string, error) {
+	return gexec.BuildWithEnvironment(
+		"github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent",
+		[]string{"CGO_ENABLED=0", "GOOS=linux", "GOARCH=" + goruntime.GOARCH},
+		"-ldflags", "-X github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/agent/version.GitVersion="+gitVersion,
+	)
+}
+
+// buildFixtureAgentDeb packages binaryPath as a minimal .deb that installs it at
+// systemdAgentBinaryPath -- the same path the real production package and this suite's systemd
+// harness both use. Returns the directory containing the built .deb (imgpkg push -f wants a
+// directory, not a single file).
+//
+// Deliberately hand-built with dpkg-deb rather than fpm: fpm/ruby are only installed in this
+// repo's CI for the real build-host-agent-deb pipeline, gated on workflow_dispatch (see
+// .github/workflows/e2e.yml and the CLAUDE.md gotcha about that target's skip/fail split) -- not
+// present for every e2e run. dpkg-deb ships with any Debian-family base (confirmed present in
+// golang:1.26.4, the base this suite's own linux-test-runner image builds from), so this needs no
+// new dependency at all.
+//
+func buildFixtureAgentDeb(ctx context.Context, binaryPath, gitVersion string) (debDir string, err error) {
+	stageDir, err := os.MkdirTemp("", "agent-fixture-deb-*")
+	if err != nil {
+		return "", err
+	}
+	defer os.RemoveAll(stageDir) //nolint:errcheck // best-effort temp dir cleanup
+
+	debianDir := filepath.Join(stageDir, "DEBIAN")
+	if mkdirErr := os.MkdirAll(debianDir, 0755); mkdirErr != nil {
+		return "", mkdirErr
+	}
+	// systemdAgentBinaryPath is "/binary/pf9-byoh-hostagent" -- dpkg-deb packs stageDir's tree
+	// onto the target filesystem's root, so the payload lives at stageDir+that same path.
+	payloadDir := filepath.Join(stageDir, filepath.Dir(systemdAgentBinaryPath))
+	if mkdirErr := os.MkdirAll(payloadDir, 0755); mkdirErr != nil {
+		return "", mkdirErr
+	}
+
+	binaryData, err := os.ReadFile(binaryPath) //nolint:gosec // binaryPath is a suite-built local binary path, not user input
+	if err != nil {
+		return "", err
+	}
+	if writeErr := os.WriteFile(filepath.Join(payloadDir, filepath.Base(systemdAgentBinaryPath)), binaryData, 0755); writeErr != nil { //nolint:gosec // the payload must be executable
+		return "", writeErr
+	}
+
+	// GOARCH happens to already match Debian's architecture naming for both values this repo's
+	// Makefile packages for (amd64/arm64) -- see Makefile's PACKAGE_GOARCH.
+	control := fmt.Sprintf("Package: %s\nVersion: %s\nArchitecture: %s\nMaintainer: byoh-e2e\nDescription: agent-upgrade e2e fixture package\n",
+		fixtureAgentDebPackageName, strings.TrimPrefix(gitVersion, "v"), goruntime.GOARCH)
+	if writeErr := os.WriteFile(filepath.Join(debianDir, "control"), []byte(control), 0644); writeErr != nil {
+		return "", writeErr
+	}
+
+	postinst := "#!/bin/sh\nset -e\nchmod +x " + systemdAgentBinaryPath + "\n"
+	if writeErr := os.WriteFile(filepath.Join(debianDir, "postinst"), []byte(postinst), 0755); writeErr != nil { //nolint:gosec // dpkg requires postinst to be executable
+		return "", writeErr
+	}
+
+	outDir, err := os.MkdirTemp("", "agent-fixture-bundle-*")
+	if err != nil {
+		return "", err
+	}
+	outPath := filepath.Join(outDir, fixtureAgentDebFileName)
+	buildCmd := exec.CommandContext(ctx, "dpkg-deb", "--build", "--root-owner-group", stageDir, outPath) // #nosec G204 -- fixed args, stageDir/outPath are our own temp dirs
+	if output, buildErr := buildCmd.CombinedOutput(); buildErr != nil {
+		return "", fmt.Errorf("dpkg-deb --build failed: %w\n%s", buildErr, output)
+	}
+
+	return outDir, nil
+}
+
+// pushFixtureAgentBundle imgpkg-pushes the .deb in debDir to the shared local bundle registry
+// (see e2e_agent_bundle_registry.go) under tag, returning the address containers on dockerNetwork
+// can pull it from. Starts the registry itself if it isn't already running -- a typical local
+// `make test-e2e-linux-vm` run never builds the real agent bundle
+// (build/pf9-byohost/debsrc/pf9-byohost-agent.deb), so ensureLocalAgentBundleRegistry may never
+// have started it.
+//
+// Returns the registry's container IP on dockerNetwork, not its container-name alias: imgpkg
+// (via go-containerregistry's name.Registry.Scheme) only skips TLS automatically for RFC1918/
+// loopback addresses, not arbitrary hostnames -- pointing agent/cloudinit/cmd_runner.go's
+// unconfigurable `imgpkg pull` at the real private IP gets that for free, with no wrapper script
+// or registry-insecure flag needed anywhere.
+func pushFixtureAgentBundle(ctx context.Context, dockerClient *client.Client, dockerNetwork, debDir, tag string) (string, error) {
+	repoRoot, err := resolveRepoRoot(ctx)
+	if err != nil {
+		return "", err
+	}
+	if startErr := startLocalBundleRegistry(ctx, dockerClient, dockerNetwork); startErr != nil {
+		return "", startErr
+	}
+	imgpkgPath, err := downloadImgpkg(ctx, repoRoot)
+	if err != nil {
+		return "", err
+	}
+
+	hostAddr := "localhost:" + localBundleRegistryHostPort
+	pushCmd := exec.CommandContext(ctx, imgpkgPath, "push", "-f", debDir, "-i", hostAddr+"/"+tag) // #nosec G204 -- fixed args, no user input
+	if output, pushErr := pushCmd.CombinedOutput(); pushErr != nil {
+		return "", fmt.Errorf("failed to push fixture agent bundle: %w\n%s", pushErr, output)
+	}
+
+	registryIP, err := localBundleRegistryIP(ctx, dockerClient, dockerNetwork)
+	if err != nil {
+		return "", err
+	}
+	return registryIP + ":5000/" + tag, nil
+}
+
+// localBundleRegistryIP returns the shared local bundle registry's own IP on dockerNetwork.
+func localBundleRegistryIP(ctx context.Context, dockerClient *client.Client, dockerNetwork string) (string, error) {
+	inspect, err := dockerClient.ContainerInspect(ctx, localBundleRegistryContainerName)
+	if err != nil {
+		return "", err
+	}
+	endpoint, ok := inspect.NetworkSettings.Networks[dockerNetwork]
+	if !ok || endpoint.IPAddress == "" {
+		return "", fmt.Errorf("local bundle registry has no IP on network %q", dockerNetwork)
+	}
+	return endpoint.IPAddress, nil
+}
+
+// installImgpkgOnHost puts imgpkg on containerID's PATH -- production hosts get it via the
+// k8s-installer's self-install fallback, which this suite's hosts skip (never join a cluster).
+func installImgpkgOnHost(ctx context.Context, dockerClient *client.Client, containerID, repoRoot string) error {
+	imgpkgPath, err := downloadImgpkg(ctx, repoRoot)
+	if err != nil {
+		return err
+	}
+	return copyToContainer(ctx, dockerClient, cpConfig{
+		sourcePath: imgpkgPath,
+		destPath:   "/usr/local/bin/imgpkg",
+		container:  containerID,
+	})
+}

test/e2e/agent_upgrade_rollout_test.go

@@ -0,0 +1,167 @@
+// Copyright 2026 Platform9, Inc. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+// nolint: testpackage
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/docker/docker/client"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	infrastructurev1beta1 "github.com/vmware-tanzu/cluster-api-provider-bringyourownhost/apis/infrastructure/v1beta1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	k8stypes "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// agentUpgradeRolloutFleetSize is the ADR's own §5.3 example size for this scenario.
+const agentUpgradeRolloutFleetSize = 4
+
+// agentUpgradeRolloutTimeout/Poll bound the full rollout: MaxUnavailable=1 mostly serializes the
+// 4 hosts, and each one's cycle is imgpkg pull (local registry, fast) + dpkg -i + os.Exit(0) +
+// systemd relaunch (RestartSec=5s) + the rollout controller's own 15s reconcile tick -- a few
+// minutes of headroom for 4 hosts.
+const (
+	agentUpgradeRolloutTimeout = 5 * time.Minute
+	agentUpgradeRolloutPoll    = 5 * time.Second
+)
+
+var _ = Describe("When an agent upgrade rollout stages across a fleet [AgentUpgrade]", func() {
+
+	var (
+		ctx              context.Context
+		specName         = "agent-upgrade-rollout"
+		namespace        *corev1.Namespace
+		cancelWatches    context.CancelFunc
+		clusterResources *clusterctl.ApplyClusterTemplateAndWaitResult
+		hosts            []byoHostHandle
+	)
+
+	BeforeEach(func() {
+		ctx, namespace, cancelWatches, clusterResources = commonSpecSetup(specName)
+	})
+
+	It("Should stage an agent upgrade across the fleet respecting maxUnavailable", func() {
+		const (
+			oldVersion = "v9.9.8"
+			newVersion = "v9.9.9"
+		)
+
+		dc, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
+		Expect(err).NotTo(HaveOccurred())
+		setDockerClient(dc)
+
+		By("Building the old and new agent binaries")
+		oldBinary, err := buildFixtureAgentBinary(oldVersion)
+		Expect(err).NotTo(HaveOccurred())
+		newBinary, err := buildFixtureAgentBinary(newVersion)
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Packaging the new version as a .deb fixture and pushing it to the local registry")
+		newDebDir, err := buildFixtureAgentDeb(ctx, newBinary, newVersion)
+		Expect(err).NotTo(HaveOccurred())
+		packageURL, err := pushFixtureAgentBundle(ctx, dockerClient, dockerNetworkInterfaceKind, newDebDir,
+			fmt.Sprintf("agent-fixture-%s:e2e", util.RandomString(6)))
+		Expect(err).NotTo(HaveOccurred())
+
+		By("Spinning up a 4-host fleet running the old version under systemd")
+		hosts, err = spinUpByoHostsWithSystemdAgent(ctx, dockerClient, namespace.Name, agentUpgradeRolloutFleetSize, oldBinary)
+		Expect(err).NotTo(HaveOccurred())
+		for _, host := range hosts {
+			defer host.StopLog()
+		}
+
+		By("Installing imgpkg on each host (stands in for the k8s-installer's self-install fallback)")
+		repoRoot, err := resolveRepoRoot(ctx)
+		Expect(err).NotTo(HaveOccurred())
+		for _, host := range hosts {
+			Expect(installImgpkgOnHost(ctx, dockerClient, host.ContainerID, repoRoot)).To(Succeed())
+		}
+
+		By("Waiting for all 4 hosts to connect")
+		for _, host := range hosts {
+			AssertByoHostConditionsTrue(ctx, bootstrapClusterProxy, namespace.Name, host.Name, specName, infrastructurev1beta1.AgentConnected)
+		}
+
+		// Captured before the rollout starts -- proof, once the rollout completes, that each
+		// host's agent process actually restarted (a new PID) inside the same container (an
+		// unchanged container ID), not that the container itself was recreated.
+		originalPIDs := make(map[string]int, len(hosts))
+		for _, host := range hosts {
+			originalPIDs[host.Name] = mainPID(ctx, dockerClient, host.ContainerID)
+			Expect(originalPIDs[host.Name]).To(BeNumerically(">", 0))
+		}
+
+		By("Creating a ByoHostAgentUpgrade targeting all 4 hosts with MaxUnavailable=1")
+		maxUnavailable := intstr.FromInt(1)
+		upgrade := &infrastructurev1beta1.ByoHostAgentUpgrade{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      fmt.Sprintf("%s-%s", specName, util.RandomString(6)),
+				Namespace: namespace.Name,
+			},
+			Spec: infrastructurev1beta1.ByoHostAgentUpgradeSpec{
+				Selector:       metav1.LabelSelector{},
+				TargetVersion:  newVersion,
+				PackageURL:     packageURL,
+				MaxUnavailable: &maxUnavailable,
+			},
+		}
+		Expect(bootstrapClusterProxy.GetClient().Create(ctx, upgrade)).To(Succeed())
+		upgradeKey := k8stypes.NamespacedName{Name: upgrade.Name, Namespace: upgrade.Namespace}
+
+		By("Polling the rollout to completion, asserting it never exceeds MaxUnavailable or fails")
+		Eventually(func(g Gomega) infrastructurev1beta1.ByoHostAgentUpgradePhase {
+			got := &infrastructurev1beta1.ByoHostAgentUpgrade{}
+			g.Expect(bootstrapClusterProxy.GetClient().Get(ctx, upgradeKey, got)).To(Succeed())
+			g.Expect(got.Status.UnavailableCount).To(BeNumerically("<=", 1),
+				"UnavailableCount exceeded MaxUnavailable mid-rollout")
+			g.Expect(got.Status.Phase).NotTo(Equal(infrastructurev1beta1.ByoHostAgentUpgradePhaseFailed),
+				"rollout failed: FailedHosts=%v", got.Status.FailedHosts)
+			return got.Status.Phase
+		}, agentUpgradeRolloutTimeout, agentUpgradeRolloutPoll).Should(Equal(infrastructurev1beta1.ByoHostAgentUpgradePhaseCompleted))
+
+		By("Asserting full convergence")
+		converged := &infrastructurev1beta1.ByoHostAgentUpgrade{}
+		Expect(bootstrapClusterProxy.GetClient().Get(ctx, upgradeKey, converged)).To(Succeed())
+		Expect(converged.Status.Upgraded).To(Equal(int32(agentUpgradeRolloutFleetSize)))
+		Expect(converged.Status.FailedHosts).To(BeEmpty())
+
+		By("Asserting each host's agent process restarted inside its unchanged container")
+		for _, host := range hosts {
+			h := &infrastructurev1beta1.ByoHost{}
+			Expect(bootstrapClusterProxy.GetClient().Get(ctx, k8stypes.NamespacedName{Name: host.Name, Namespace: namespace.Name}, h)).To(Succeed())
+			Expect(h.Status.AgentVersion).To(Equal(newVersion), "host %s never reported the new version", host.Name)
+
+			newPID := mainPID(ctx, dockerClient, host.ContainerID)
+			Expect(newPID).To(SatisfyAll(BeNumerically(">", 0), Not(BeNumerically("==", originalPIDs[host.Name]))),
+				"expected host %s's agent process to have restarted with a new PID", host.Name)
+
+			inspect, err := dockerClient.ContainerInspect(ctx, host.ContainerID)
+			Expect(err).NotTo(HaveOccurred(), "expected host %s's original container to still exist", host.Name)
+			Expect(inspect.State.Running).To(BeTrue(), "expected host %s's container to still be running", host.Name)
+		}
+	})
+
+	JustAfterEach(func() {
+		if CurrentGinkgoTestDescription().Failed {
+			logFiles := make([]string, 0, len(hosts))
+			for _, host := range hosts {
+				logFiles = append(logFiles, host.LogFilePath)
+			}
+			ShowInfo(logFiles)
+		}
+	})
+
+	AfterEach(func() {
+		dumpSpecResourcesAndCleanup(ctx, specName, bootstrapClusterProxy, artifactFolder, namespace, cancelWatches, clusterResources.Cluster, e2eConfig.GetIntervals, skipCleanup)
+
+		teardownByoHosts(ctx, getDockerClient(), hosts)
+	})
+})