diff --git a/components/kubernetes/openshift.go b/components/kubernetes/openshift.go new file mode 100644 index 000000000..3183aeb6c --- /dev/null +++ b/components/kubernetes/openshift.go @@ -0,0 +1,133 @@ +package kubernetes + +import ( + "github.com/DataDog/test-infra-definitions/common/config" + "github.com/DataDog/test-infra-definitions/common/utils" + "github.com/DataDog/test-infra-definitions/components" + "github.com/DataDog/test-infra-definitions/components/command" + oscomp "github.com/DataDog/test-infra-definitions/components/os" + "github.com/DataDog/test-infra-definitions/components/remote" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +func NewLocalOpenShiftCluster(env config.Env, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) { + return components.NewComponent(env, name, func(clusterComp *Cluster) error { + openShiftClusterName := env.CommonNamer().DisplayName(49) + opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) + commonEnvironment := env + runner := command.NewLocalRunner(env, command.LocalRunnerArgs{ + OSCommand: command.NewUnixOSCommand(), + }) + + crcSetup, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{ + Create: pulumi.String("crc setup"), + }, opts...) + if err != nil { + return err + } + startCluster, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{ + Create: pulumi.Sprintf("crc start -p %s", pullSecretPath), + Delete: pulumi.String("crc stop"), + Triggers: pulumi.Array{ + pulumi.String(pullSecretPath), + }, + }, utils.MergeOptions(opts, utils.PulumiDependsOn(crcSetup))...) + if err != nil { + return err + } + + kubeConfigCmd, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{ + Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"), + }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCluster))...) + if err != nil { + return err + } + + clusterComp.KubeConfig = kubeConfigCmd.StdoutOutput() + clusterComp.ClusterName = openShiftClusterName.ToStringOutput() + return nil + }, opts...) +} + +func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) { + return components.NewComponent(env, name, func(clusterComp *Cluster) error { + openShiftClusterName := env.CommonNamer().DisplayName(49) + opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) + runner := vm.OS.Runner() + commonEnvironment := env + + openShiftInstallBinary, err := InstallOpenShiftBinary(env, vm, opts...) + if err != nil { + return err + } + + pullSecretContent, err := utils.ReadSecretFile(pullSecretPath) + if err != nil { + return err + } + pullSecretFile, err := vm.OS.FileManager().CopyInlineFile( + pullSecretContent, + "/tmp/pull-secret.txt", + ) + if err != nil { + return err + } + + installLibvirt, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-libvirt"), &command.Args{ + Create: pulumi.String(` + sudo dnf install -y libvirt NetworkManager`), + }, utils.MergeOptions(opts, utils.PulumiDependsOn(openShiftInstallBinary))...) + if err != nil { + return err + } + // To avoid the crc-daemon.service being stopped when the user session ends, we enable linger for the user + enableLinger, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("enable-linger"), &command.Args{ + Create: pulumi.String("loginctl enable-linger"), + }, utils.MergeOptions(opts, utils.PulumiDependsOn(installLibvirt))...) + if err != nil { + return err + } + + setupCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{ + Create: pulumi.String("crc setup"), + }, utils.MergeOptions(opts, utils.PulumiDependsOn(pullSecretFile, enableLinger))...) + if err != nil { + return err + } + + startCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{ + Create: pulumi.String(`crc start -p /tmp/pull-secret.txt`), + Delete: pulumi.String("crc stop && crc delete && crc cleanup"), + Triggers: pulumi.Array{ + pulumi.String(pullSecretPath), + }, + }, utils.MergeOptions(opts, utils.PulumiDependsOn(setupCRC))...) + if err != nil { + return err + } + + kubeConfig, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{ + Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"), + }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...) + if err != nil { + return err + } + + clusterComp.KubeConfig = kubeConfig.StdoutOutput() + clusterComp.ClusterName = openShiftClusterName.ToStringOutput() + return nil + }, opts...) +} + +func InstallOpenShiftBinary(env config.Env, vm *remote.Host, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { + openShiftArch := vm.OS.Descriptor().Architecture + if openShiftArch == oscomp.AMD64Arch { + openShiftArch = "amd64" + } + return vm.OS.Runner().Command( + env.CommonNamer().ResourceName("crc-install"), + &command.Args{ + Create: pulumi.Sprintf(`curl -fsSL https://developers.redhat.com/content-gateway/file/pub/openshift-v4/clients/crc/2.52.0/crc-linux-%s.tar.xz | \ + sudo tar -xJ -C /usr/local/bin --strip-components=1 crc-linux-2.52.0-%s/crc`, openShiftArch, openShiftArch), + }, opts...) +} diff --git a/registry/scenarios.go b/registry/scenarios.go index 0086cb52e..a5897ab1c 100644 --- a/registry/scenarios.go +++ b/registry/scenarios.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/DataDog/test-infra-definitions/scenarios/gcp/gke" + "github.com/DataDog/test-infra-definitions/scenarios/gcp/openshiftvm" "github.com/DataDog/test-infra-definitions/scenarios/aws/ec2" "github.com/DataDog/test-infra-definitions/scenarios/aws/ecs" @@ -23,18 +24,19 @@ type ScenarioRegistry map[string]pulumi.RunFunc func Scenarios() ScenarioRegistry { return ScenarioRegistry{ - "aws/vm": ec2.VMRun, - "aws/dockervm": ec2.VMRunWithDocker, - "aws/ecs": ecs.Run, - "aws/eks": eks.Run, - "aws/installer": installer.Run, - "aws/microvms": microvms.Run, - "aws/kind": kindvm.Run, - "az/vm": computerun.VMRun, - "az/aks": aks.Run, - "gcp/vm": gcpcompute.VMRun, - "gcp/gke": gke.Run, - "localpodman/vm": localpodmanrun.VMRun, + "aws/vm": ec2.VMRun, + "aws/dockervm": ec2.VMRunWithDocker, + "aws/ecs": ecs.Run, + "aws/eks": eks.Run, + "aws/installer": installer.Run, + "aws/microvms": microvms.Run, + "aws/kind": kindvm.Run, + "az/vm": computerun.VMRun, + "az/aks": aks.Run, + "gcp/vm": gcpcompute.VMRun, + "gcp/gke": gke.Run, + "gcp/openshiftvm": openshiftvm.Run, + "localpodman/vm": localpodmanrun.VMRun, } } diff --git a/resources/gcp/compute/vm.go b/resources/gcp/compute/vm.go index dac147086..81b4cdc07 100644 --- a/resources/gcp/compute/vm.go +++ b/resources/gcp/compute/vm.go @@ -26,6 +26,14 @@ func NewLinuxInstance(e gcp.Environment, name string, imageName string, instance }, Name: e.Namer.DisplayName(63, pulumi.String(name)), MachineType: pulumi.String(instanceType), + AdvancedMachineFeatures: func() *compute.InstanceAdvancedMachineFeaturesArgs { + if e.EnableNestedVirtualization() { + return &compute.InstanceAdvancedMachineFeaturesArgs{ + EnableNestedVirtualization: pulumi.Bool(true), + } + } + return nil + }(), Tags: pulumi.StringArray{ pulumi.String("appgate-gateway"), pulumi.String("nat-us-central1"), @@ -36,6 +44,7 @@ func NewLinuxInstance(e gcp.Environment, name string, imageName string, instance Labels: pulumi.StringMap{ "my_label": pulumi.String("value"), }, + Size: pulumi.Int(100), }, }, Metadata: pulumi.StringMap{ diff --git a/resources/gcp/environment.go b/resources/gcp/environment.go index 0662ca317..a23c5f63a 100644 --- a/resources/gcp/environment.go +++ b/resources/gcp/environment.go @@ -26,6 +26,8 @@ const ( DDInfraDefaultZoneNameParamName = "gcp/defaultZone" DDInfraDefautVMServiceAccountParamName = "gcp/defaultVMServiceAccount" DDInfraGKEEnableAutopilot = "gcp/gke/enableAutopilot" + DDInfraOpenShiftPullSecretPath = "gcp/openshift/pullSecretPath" + DDInfraEnableNestedVirtualization = "gcp/enableNestedVirtualization" ) type Environment struct { @@ -170,3 +172,13 @@ func (e *Environment) Region() string { func (e *Environment) Zone() string { return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultZoneNameParamName, e.envDefault.gcp.zone) } + +// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file +func (e *Environment) OpenShiftPullSecretPath() string { + return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath) +} + +// EnableNestedVirtualization returns whether to enable nested virtualization +func (e *Environment) EnableNestedVirtualization() bool { + return e.GetBoolWithDefault(e.InfraConfig, DDInfraEnableNestedVirtualization, e.envDefault.ddInfra.openshift.nestedVirtualization) +} diff --git a/resources/gcp/environmentDefaults.go b/resources/gcp/environmentDefaults.go index ed90d5fc4..cbb93def0 100644 --- a/resources/gcp/environmentDefaults.go +++ b/resources/gcp/environmentDefaults.go @@ -22,12 +22,17 @@ type ddInfra struct { defaultSubnetName string defaultVMServiceAccount string gke ddInfraGKE + openshift ddInfraOpenShift } type ddInfraGKE struct { autopilot bool } +type ddInfraOpenShift struct { + nestedVirtualization bool +} + func getEnvironmentDefault(envName string) environmentDefault { switch envName { case agentSandboxEnv: @@ -52,6 +57,7 @@ func agentSandboxDefault() environmentDefault { defaultSubnetName: "datadog-agent-sandbox-us-central1-private", defaultVMServiceAccount: "vmserviceaccount@datadog-agent-sandbox.iam.gserviceaccount.com", gke: ddInfraGKE{autopilot: false}, + openshift: ddInfraOpenShift{nestedVirtualization: false}, }, } } @@ -69,6 +75,7 @@ func agentQaDefault() environmentDefault { defaultSubnetName: "datadog-agent-qa-us-central1-private", defaultVMServiceAccount: "vmserviceaccount@datadog-agent-qa.iam.gserviceaccount.com", gke: ddInfraGKE{autopilot: false}, + openshift: ddInfraOpenShift{nestedVirtualization: false}, }, } } diff --git a/resources/local/environment.go b/resources/local/environment.go index 8150185bb..e257057d2 100644 --- a/resources/local/environment.go +++ b/resources/local/environment.go @@ -10,7 +10,8 @@ import ( const ( localNamerNamespace = "local" // local Infra (local) - DDInfraDefaultPublicKeyPath = "local/defaultPublicKeyPath" + DDInfraDefaultPublicKeyPath = "local/defaultPublicKeyPath" + DDInfraOpenShiftPullSecretPath = "local/openshift/pullSecretPath" ) type Environment struct { @@ -62,3 +63,8 @@ func (e *Environment) InternalRegistryFullImagePathExists(_ string) (bool, error func (e *Environment) DefaultPublicKeyPath() string { return e.InfraConfig.Get(DDInfraDefaultPublicKeyPath) } + +// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file +func (e *Environment) OpenShiftPullSecretPath() string { + return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath) +} diff --git a/scenarios/gcp/compute/os_resolver.go b/scenarios/gcp/compute/os_resolver.go index fbd5ae5ad..40d082ea4 100644 --- a/scenarios/gcp/compute/os_resolver.go +++ b/scenarios/gcp/compute/os_resolver.go @@ -9,6 +9,7 @@ type imageResolveFunc func(e gcp.Environment, osInfo os.Descriptor) (string, err var imageResolvers = map[os.Flavor]imageResolveFunc{ os.Ubuntu: resolveUbuntuImage, + os.RedHat: resolveRhelImage, } func resolveUbuntuImage(_ gcp.Environment, osInfo os.Descriptor) (string, error) { @@ -23,3 +24,14 @@ func resolveUbuntuImage(_ gcp.Environment, osInfo os.Descriptor) (string, error) return "", nil } } +func resolveRhelImage(_ gcp.Environment, osInfo os.Descriptor) (string, error) { + if osInfo.Version == "" { + osInfo.Version = os.RedHatDefault.Version + } + switch osInfo.Version { + case os.RedHat9.Version: + return "rhel-9-v20250611", nil + } + + return "", nil +} diff --git a/scenarios/gcp/compute/vmargs.go b/scenarios/gcp/compute/vmargs.go index 18f854c21..1930bfef2 100644 --- a/scenarios/gcp/compute/vmargs.go +++ b/scenarios/gcp/compute/vmargs.go @@ -2,6 +2,7 @@ package compute import ( "github.com/DataDog/test-infra-definitions/common" + "github.com/DataDog/test-infra-definitions/common/utils" "github.com/DataDog/test-infra-definitions/components/os" ) @@ -18,3 +19,18 @@ func newParams(options ...VMOption) (*vmArgs, error) { return common.ApplyOption(vmArgs, options) } + +// WithOS sets the OS +// Version defaults to latest +func WithOS(osDesc os.Descriptor) VMOption { + return WithOSArch(osDesc, osDesc.Architecture) +} + +// WithArch set the architecture and the operating system. +// Version defaults to latest +func WithOSArch(osDesc os.Descriptor, arch os.Architecture) VMOption { + return func(p *vmArgs) error { + p.osInfo = utils.Pointer(osDesc.WithArch(arch)) + return nil + } +} diff --git a/scenarios/gcp/openshiftvm/run.go b/scenarios/gcp/openshiftvm/run.go new file mode 100644 index 000000000..6a1298e6d --- /dev/null +++ b/scenarios/gcp/openshiftvm/run.go @@ -0,0 +1,35 @@ +package openshiftvm + +import ( + "github.com/DataDog/test-infra-definitions/components/kubernetes" + "github.com/DataDog/test-infra-definitions/components/os" + resGcp "github.com/DataDog/test-infra-definitions/resources/gcp" + "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" + "github.com/pulumi/pulumi/sdk/v3/go/pulumi" +) + +func Run(ctx *pulumi.Context) error { + gcpEnv, err := resGcp.NewEnvironment(ctx) + if err != nil { + return err + } + + osDesc := os.DescriptorFromString("redhat:9", os.RedHat9) + vm, err := compute.NewVM(gcpEnv, "openshift", + compute.WithOS(osDesc)) + if err != nil { + return err + } + if err := vm.Export(ctx, nil); err != nil { + return err + } + + openshiftCluster, err := kubernetes.NewOpenShiftCluster(&gcpEnv, vm, "openshift", gcpEnv.OpenShiftPullSecretPath()) + if err != nil { + return err + } + if err := openshiftCluster.Export(ctx, nil); err != nil { + return err + } + return openshiftCluster.Export(ctx, nil) +} diff --git a/tasks/config.py b/tasks/config.py index 2e2b48c6d..071394ea0 100644 --- a/tasks/config.py +++ b/tasks/config.py @@ -48,6 +48,7 @@ class Azure(BaseModel, extra=Extra.forbid): class GCP(BaseModel, extra=Extra.forbid): _DEFAULT_ACCOUNT = "agent-sandbox" publicKeyPath: Optional[str] = None + pullSecretPath: Optional[str] = None account: Optional[str] = _DEFAULT_ACCOUNT gcp: Optional[GCP] = None diff --git a/tasks/doc.py b/tasks/doc.py index 80c3b366e..1d8487731 100644 --- a/tasks/doc.py +++ b/tasks/doc.py @@ -37,4 +37,5 @@ agent_env: str = "Extra environment variables to run the agent with, the format is `VAR1=val1,VAR2=val2`" helm_config: str = "Path to a custom helm config file that will be merged with the default one" local_package: str = "Path to a local package to install, it can be either a folder or a file. If a folder is targetted we automatically take the first file with the correct extension depending on the targetted OS" +pull_secret_path: str = "Path to the OpenShift pull secret file (required for OpenShift cluster creation)." local_chart_path: str = "Path to a local helm chart to install the Datadog Agent." diff --git a/tasks/gcp/__init__.py b/tasks/gcp/__init__.py index c64cee35c..7328b51a5 100644 --- a/tasks/gcp/__init__.py +++ b/tasks/gcp/__init__.py @@ -3,6 +3,7 @@ from invoke.collection import Collection from tasks.gcp.gke import create_gke, destroy_gke +from tasks.gcp.openshift import create_openshift, destroy_openshift from tasks.gcp.vm import create_vm, destroy_vm collection = Collection() @@ -10,3 +11,5 @@ collection.add_task(create_vm) collection.add_task(create_gke) collection.add_task(destroy_gke) +collection.add_task(create_openshift) +collection.add_task(destroy_openshift) diff --git a/tasks/gcp/openshift.py b/tasks/gcp/openshift.py new file mode 100644 index 000000000..b0b9dff72 --- /dev/null +++ b/tasks/gcp/openshift.py @@ -0,0 +1,85 @@ +from typing import Optional + +from invoke.context import Context +from invoke.exceptions import Exit +from invoke.tasks import task +from pydantic_core._pydantic_core import ValidationError + +from tasks import config, doc +from tasks.config import get_full_profile_path +from tasks.deploy import deploy +from tasks.destroy import destroy + +scenario_name = "gcp/openshiftvm" + + +@task( + help={ + "config_path": doc.config_path, + "stack_name": doc.stack_name, + "pull_secret_path": doc.pull_secret_path, + } +) +def create_openshift( + ctx: Context, + config_path: Optional[str] = None, + stack_name: Optional[str] = None, + pull_secret_path: Optional[str] = None, + use_nested_virtualization: Optional[bool] = True, +): + """ + Create an OpenShift environment. + """ + + try: + cfg = config.get_local_config(config_path) + except ValidationError as e: + raise Exit(f"Error in config {get_full_profile_path(config_path)}") from e + + # Use parameter if provided during invoke setup, otherwise use config + if not pull_secret_path: + pull_secret_path = cfg.get_gcp().pullSecretPath + if not pull_secret_path: + raise Exit( + "pull_secret_path is required. Either use invoke.gcp.create-openshift -p or configure it with 'invoke setup'" + ) + + extra_flags = { + "scenario": scenario_name, + "ddinfra:env": f"gcp/{cfg.get_gcp().account}", + "ddinfra:gcp/defaultPublicKeyPath": cfg.get_gcp().publicKeyPath, + "ddinfra:gcp/openshift/pullSecretPath": pull_secret_path, + "ddinfra:gcp/enableNestedVirtualization": use_nested_virtualization, + "ddinfra:gcp/defaultInstanceType": "n2-standard-8", + } + + deploy( + ctx, + scenario_name, + config_path, + stack_name=stack_name, + install_agent=False, + extra_flags=extra_flags, + ) + + +@task( + help={ + "config_path": doc.config_path, + "stack_name": doc.stack_name, + } +) +def destroy_openshift( + ctx: Context, + config_path: Optional[str] = None, + stack_name: Optional[str] = None, +): + """ + Destroy an environment created by invoke gcp.create-openshift. + """ + destroy( + ctx, + scenario_name=scenario_name, + config_path=config_path, + stack=stack_name, + ) diff --git a/tasks/setup.py b/tasks/setup.py index 03bfac70f..98b6d0665 100644 --- a/tasks/setup.py +++ b/tasks/setup.py @@ -328,6 +328,20 @@ def setup_gcp_config(config: Config): if default_account: config.configParams.gcp.account = default_account + # openShift pull secret path + if config.configParams.gcp.pullSecretPath is None: + config.configParams.gcp.pullSecretPath = "" + default_pull_secret_path = config.configParams.gcp.pullSecretPath + while True: + config.configParams.gcp.pullSecretPath = default_pull_secret_path + pull_secret_path = ask("🔑 Path to your OpenShift pull secret file (optional, can be set later): ") + if pull_secret_path: + config.configParams.gcp.pullSecretPath = pull_secret_path + + if os.path.isfile(config.configParams.gcp.pullSecretPath): + break + warn(f"{config.configParams.gcp.pullSecretPath} is not a valid file") + def setupAgentConfig(config): if config.configParams.agent is None: