Enable OpenShift cluster deployment via Pulumi on GCP and local environments

components/kubernetes/openshift.go

@@ -0,0 +1,133 @@
+package kubernetes
+
+import (
+	"github.com/DataDog/test-infra-definitions/common/config"
+	"github.com/DataDog/test-infra-definitions/common/utils"
+	"github.com/DataDog/test-infra-definitions/components"
+	"github.com/DataDog/test-infra-definitions/components/command"
+	oscomp "github.com/DataDog/test-infra-definitions/components/os"
+	"github.com/DataDog/test-infra-definitions/components/remote"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+)
+
+func NewLocalOpenShiftCluster(env config.Env, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) {
+	return components.NewComponent(env, name, func(clusterComp *Cluster) error {
+		openShiftClusterName := env.CommonNamer().DisplayName(49)
+		opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
+		commonEnvironment := env
+		runner := command.NewLocalRunner(env, command.LocalRunnerArgs{
+			OSCommand: command.NewUnixOSCommand(),
+		})
+
+		crcSetup, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{
+			Create: pulumi.String("crc setup"),
+		}, opts...)
+		if err != nil {
+			return err
+		}
+		startCluster, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{
+			Create: pulumi.Sprintf("crc start -p %s", pullSecretPath),
+			Delete: pulumi.String("crc stop"),
+			Triggers: pulumi.Array{
+				pulumi.String(pullSecretPath),
+			},
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(crcSetup))...)
+		if err != nil {
+			return err
+		}
+
+		kubeConfigCmd, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{
+			Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"),
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(startCluster))...)
+		if err != nil {
+			return err
+		}
+
+		clusterComp.KubeConfig = kubeConfigCmd.StdoutOutput()
+		clusterComp.ClusterName = openShiftClusterName.ToStringOutput()
+		return nil
+	}, opts...)
+}
+
+func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) {
+	return components.NewComponent(env, name, func(clusterComp *Cluster) error {
+		openShiftClusterName := env.CommonNamer().DisplayName(49)
+		opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
+		runner := vm.OS.Runner()
+		commonEnvironment := env
+
+		openShiftInstallBinary, err := InstallOpenShiftBinary(env, vm, opts...)
+		if err != nil {
+			return err
+		}
+
+		pullSecretContent, err := utils.ReadSecretFile(pullSecretPath)
+		if err != nil {
+			return err
+		}
+		pullSecretFile, err := vm.OS.FileManager().CopyInlineFile(
+			pullSecretContent,
+			"/tmp/pull-secret.txt",
+		)
+		if err != nil {
+			return err
+		}
+
+		installLibvirt, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-libvirt"), &command.Args{
+			Create: pulumi.String(`
+		sudo dnf install -y libvirt NetworkManager`),
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(openShiftInstallBinary))...)
+		if err != nil {
+			return err
+		}
+		// To avoid the crc-daemon.service being stopped when the user session ends, we enable linger for the user
+		enableLinger, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("enable-linger"), &command.Args{
+			Create: pulumi.String("loginctl enable-linger"),
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(installLibvirt))...)
+		if err != nil {
+			return err
+		}
+
+		setupCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{
+			Create: pulumi.String("crc setup"),
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(pullSecretFile, enableLinger))...)
+		if err != nil {
+			return err
+		}
+
+		startCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{
+			Create: pulumi.String(`crc start -p /tmp/pull-secret.txt`),
+			Delete: pulumi.String("crc stop && crc delete && crc cleanup"),
+			Triggers: pulumi.Array{
+				pulumi.String(pullSecretPath),
+			},
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(setupCRC))...)
+		if err != nil {
+			return err
+		}
+
+		kubeConfig, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{
+			Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"),
+		}, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
+		if err != nil {
+			return err
+		}
+
+		clusterComp.KubeConfig = kubeConfig.StdoutOutput()
+		clusterComp.ClusterName = openShiftClusterName.ToStringOutput()
+		return nil
+	}, opts...)
+}
+
+func InstallOpenShiftBinary(env config.Env, vm *remote.Host, opts ...pulumi.ResourceOption) (pulumi.Resource, error) {
+	openShiftArch := vm.OS.Descriptor().Architecture
+	if openShiftArch == oscomp.AMD64Arch {
+		openShiftArch = "amd64"
+	}
+	return vm.OS.Runner().Command(
+		env.CommonNamer().ResourceName("crc-install"),
+		&command.Args{
+			Create: pulumi.Sprintf(`curl -fsSL https://developers.redhat.com/content-gateway/file/pub/openshift-v4/clients/crc/2.52.0/crc-linux-%s.tar.xz | \
+	sudo tar -xJ -C /usr/local/bin --strip-components=1 crc-linux-2.52.0-%s/crc`, openShiftArch, openShiftArch),
+		}, opts...)
+}

registry/scenarios.go

@@ -4,6 +4,7 @@ import (
 	"strings"
 
 	"github.com/DataDog/test-infra-definitions/scenarios/gcp/gke"
+	"github.com/DataDog/test-infra-definitions/scenarios/gcp/openshiftvm"
 
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/ec2"
 	"github.com/DataDog/test-infra-definitions/scenarios/aws/ecs"
@@ -23,18 +24,19 @@ type ScenarioRegistry map[string]pulumi.RunFunc
 
 func Scenarios() ScenarioRegistry {
 	return ScenarioRegistry{
-		"aws/vm":         ec2.VMRun,
-		"aws/dockervm":   ec2.VMRunWithDocker,
-		"aws/ecs":        ecs.Run,
-		"aws/eks":        eks.Run,
-		"aws/installer":  installer.Run,
-		"aws/microvms":   microvms.Run,
-		"aws/kind":       kindvm.Run,
-		"az/vm":          computerun.VMRun,
-		"az/aks":         aks.Run,
-		"gcp/vm":         gcpcompute.VMRun,
-		"gcp/gke":        gke.Run,
-		"localpodman/vm": localpodmanrun.VMRun,
+		"aws/vm":          ec2.VMRun,
+		"aws/dockervm":    ec2.VMRunWithDocker,
+		"aws/ecs":         ecs.Run,
+		"aws/eks":         eks.Run,
+		"aws/installer":   installer.Run,
+		"aws/microvms":    microvms.Run,
+		"aws/kind":        kindvm.Run,
+		"az/vm":           computerun.VMRun,
+		"az/aks":          aks.Run,
+		"gcp/vm":          gcpcompute.VMRun,
+		"gcp/gke":         gke.Run,
+		"gcp/openshiftvm": openshiftvm.Run,
+		"localpodman/vm":  localpodmanrun.VMRun,
 	}
 }
 

resources/gcp/compute/vm.go

@@ -26,6 +26,14 @@ func NewLinuxInstance(e gcp.Environment, name string, imageName string, instance
 		},
 		Name:        e.Namer.DisplayName(63, pulumi.String(name)),
 		MachineType: pulumi.String(instanceType),
+		AdvancedMachineFeatures: func() *compute.InstanceAdvancedMachineFeaturesArgs {
+			if e.EnableNestedVirtualization() {
+				return &compute.InstanceAdvancedMachineFeaturesArgs{
+					EnableNestedVirtualization: pulumi.Bool(true),
+				}
+			}
+			return nil
+		}(),
 		Tags: pulumi.StringArray{
 			pulumi.String("appgate-gateway"),
 			pulumi.String("nat-us-central1"),
@@ -36,6 +44,7 @@ func NewLinuxInstance(e gcp.Environment, name string, imageName string, instance
 				Labels: pulumi.StringMap{
 					"my_label": pulumi.String("value"),
 				},
+				Size: pulumi.Int(100),
 			},
 		},
 		Metadata: pulumi.StringMap{

resources/gcp/environment.go

@@ -26,6 +26,8 @@ const (
 	DDInfraDefaultZoneNameParamName        = "gcp/defaultZone"
 	DDInfraDefautVMServiceAccountParamName = "gcp/defaultVMServiceAccount"
 	DDInfraGKEEnableAutopilot              = "gcp/gke/enableAutopilot"
+	DDInfraOpenShiftPullSecretPath         = "gcp/openshift/pullSecretPath"
+	DDInfraEnableNestedVirtualization      = "gcp/enableNestedVirtualization"
 )
 
 type Environment struct {
@@ -170,3 +172,13 @@ func (e *Environment) Region() string {
 func (e *Environment) Zone() string {
 	return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultZoneNameParamName, e.envDefault.gcp.zone)
 }
+
+// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file
+func (e *Environment) OpenShiftPullSecretPath() string {
+	return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath)
+}
+
+// EnableNestedVirtualization returns whether to enable nested virtualization
+func (e *Environment) EnableNestedVirtualization() bool {
+	return e.GetBoolWithDefault(e.InfraConfig, DDInfraEnableNestedVirtualization, e.envDefault.ddInfra.openshift.nestedVirtualization)
+}

resources/gcp/environmentDefaults.go

@@ -22,12 +22,17 @@ type ddInfra struct {
 	defaultSubnetName       string
 	defaultVMServiceAccount string
 	gke                     ddInfraGKE
+	openshift               ddInfraOpenShift
 }
 
 type ddInfraGKE struct {
 	autopilot bool
 }
 
+type ddInfraOpenShift struct {
+	nestedVirtualization bool
+}
+
 func getEnvironmentDefault(envName string) environmentDefault {
 	switch envName {
 	case agentSandboxEnv:
@@ -52,6 +57,7 @@ func agentSandboxDefault() environmentDefault {
 			defaultSubnetName:       "datadog-agent-sandbox-us-central1-private",
 			defaultVMServiceAccount: "vmserviceaccount@datadog-agent-sandbox.iam.gserviceaccount.com",
 			gke:                     ddInfraGKE{autopilot: false},
+			openshift:               ddInfraOpenShift{nestedVirtualization: false},
 		},
 	}
 }
@@ -69,6 +75,7 @@ func agentQaDefault() environmentDefault {
 			defaultSubnetName:       "datadog-agent-qa-us-central1-private",
 			defaultVMServiceAccount: "vmserviceaccount@datadog-agent-qa.iam.gserviceaccount.com",
 			gke:                     ddInfraGKE{autopilot: false},
+			openshift:               ddInfraOpenShift{nestedVirtualization: false},
 		},
 	}
 }

resources/local/environment.go

@@ -10,7 +10,8 @@ import (
 const (
 	localNamerNamespace = "local"
 	// local Infra (local)
-	DDInfraDefaultPublicKeyPath = "local/defaultPublicKeyPath"
+	DDInfraDefaultPublicKeyPath    = "local/defaultPublicKeyPath"
+	DDInfraOpenShiftPullSecretPath = "local/openshift/pullSecretPath"
 )
 
 type Environment struct {
@@ -62,3 +63,8 @@ func (e *Environment) InternalRegistryFullImagePathExists(_ string) (bool, error
 func (e *Environment) DefaultPublicKeyPath() string {
 	return e.InfraConfig.Get(DDInfraDefaultPublicKeyPath)
 }
+
+// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file
+func (e *Environment) OpenShiftPullSecretPath() string {
+	return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath)
+}

scenarios/gcp/compute/os_resolver.go

@@ -9,6 +9,7 @@ type imageResolveFunc func(e gcp.Environment, osInfo os.Descriptor) (string, err
 
 var imageResolvers = map[os.Flavor]imageResolveFunc{
 	os.Ubuntu: resolveUbuntuImage,
+	os.RedHat: resolveRhelImage,
 }
 
 func resolveUbuntuImage(_ gcp.Environment, osInfo os.Descriptor) (string, error) {
@@ -23,3 +24,14 @@ func resolveUbuntuImage(_ gcp.Environment, osInfo os.Descriptor) (string, error)
 		return "", nil
 	}
 }
+func resolveRhelImage(_ gcp.Environment, osInfo os.Descriptor) (string, error) {
+	if osInfo.Version == "" {
+		osInfo.Version = os.RedHatDefault.Version
+	}
+	switch osInfo.Version {
+	case os.RedHat9.Version:
+		return "rhel-9-v20250611", nil
+	}
+
+	return "", nil
+}

scenarios/gcp/compute/vmargs.go

@@ -2,6 +2,7 @@ package compute
 
 import (
 	"github.com/DataDog/test-infra-definitions/common"
+	"github.com/DataDog/test-infra-definitions/common/utils"
 	"github.com/DataDog/test-infra-definitions/components/os"
 )
 
@@ -18,3 +19,18 @@ func newParams(options ...VMOption) (*vmArgs, error) {
 
 	return common.ApplyOption(vmArgs, options)
 }
+
+// WithOS sets the OS
+// Version defaults to latest
+func WithOS(osDesc os.Descriptor) VMOption {
+	return WithOSArch(osDesc, osDesc.Architecture)
+}
+
+// WithArch set the architecture and the operating system.
+// Version defaults to latest
+func WithOSArch(osDesc os.Descriptor, arch os.Architecture) VMOption {
+	return func(p *vmArgs) error {
+		p.osInfo = utils.Pointer(osDesc.WithArch(arch))
+		return nil
+	}
+}

scenarios/gcp/openshiftvm/run.go

@@ -0,0 +1,35 @@
+package openshiftvm
+
+import (
+	"github.com/DataDog/test-infra-definitions/components/kubernetes"
+	"github.com/DataDog/test-infra-definitions/components/os"
+	resGcp "github.com/DataDog/test-infra-definitions/resources/gcp"
+	"github.com/DataDog/test-infra-definitions/scenarios/gcp/compute"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+)
+
+func Run(ctx *pulumi.Context) error {
+	gcpEnv, err := resGcp.NewEnvironment(ctx)
+	if err != nil {
+		return err
+	}
+
+	osDesc := os.DescriptorFromString("redhat:9", os.RedHat9)
+	vm, err := compute.NewVM(gcpEnv, "openshift",
+		compute.WithOS(osDesc))
+	if err != nil {
+		return err
+	}
+	if err := vm.Export(ctx, nil); err != nil {
+		return err
+	}
+
+	openshiftCluster, err := kubernetes.NewOpenShiftCluster(&gcpEnv, vm, "openshift", gcpEnv.OpenShiftPullSecretPath())
+	if err != nil {
+		return err
+	}
+	if err := openshiftCluster.Export(ctx, nil); err != nil {
+		return err
+	}
+	return openshiftCluster.Export(ctx, nil)
+}

tasks/config.py

@@ -48,6 +48,7 @@ class Azure(BaseModel, extra=Extra.forbid):
         class GCP(BaseModel, extra=Extra.forbid):
             _DEFAULT_ACCOUNT = "agent-sandbox"
             publicKeyPath: Optional[str] = None
+            pullSecretPath: Optional[str] = None
             account: Optional[str] = _DEFAULT_ACCOUNT
 
         gcp: Optional[GCP] = None

tasks/doc.py

@@ -37,4 +37,5 @@
 agent_env: str = "Extra environment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`"
 helm_config: str = "Path to a custom helm config file that will be merged with the default one"
 local_package: str = "Path to a local package to install, it can be either a folder or a file. If a folder is targetted we automatically take the first file with the correct extension depending on the targetted OS"
+pull_secret_path: str = "Path to the OpenShift pull secret file (required for OpenShift cluster creation)."
 local_chart_path: str = "Path to a local helm chart to install the Datadog Agent."

tasks/gcp/__init__.py

@@ -3,10 +3,13 @@
 from invoke.collection import Collection
 
 from tasks.gcp.gke import create_gke, destroy_gke
+from tasks.gcp.openshift import create_openshift, destroy_openshift
 from tasks.gcp.vm import create_vm, destroy_vm
 
 collection = Collection()
 collection.add_task(destroy_vm)
 collection.add_task(create_vm)
 collection.add_task(create_gke)
 collection.add_task(destroy_gke)
+collection.add_task(create_openshift)
+collection.add_task(destroy_openshift)