Enable OpenShift cluster deployment via Pulumi on GCP and local environments #1591
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,144 @@ | ||
| package kubernetes | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
|
|
||
| "github.com/DataDog/test-infra-definitions/common/config" | ||
| "github.com/DataDog/test-infra-definitions/common/utils" | ||
| "github.com/DataDog/test-infra-definitions/components" | ||
| "github.com/DataDog/test-infra-definitions/components/command" | ||
| oscomp "github.com/DataDog/test-infra-definitions/components/os" | ||
| "github.com/DataDog/test-infra-definitions/components/remote" | ||
| "github.com/pulumi/pulumi/sdk/v3/go/pulumi" | ||
| sdkconfig "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config" | ||
| ) | ||
|
|
||
| func NewLocalOpenShiftCluster(env config.Env, name string, opts ...pulumi.ResourceOption) (*Cluster, error) { | ||
| return components.NewComponent(env, name, func(clusterComp *Cluster) error { | ||
| opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) | ||
| commonEnvironment := env | ||
| runner := command.NewLocalRunner(env, command.LocalRunnerArgs{ | ||
| OSCommand: command.NewUnixOSCommand(), | ||
| }) | ||
|
|
||
| pullSecretPath := os.Getenv("PULL_SECRET_PATH") | ||
|
Member
There was a problem hiding this comment. Why do we need it to be passed as an environment variable? Can we make it a params of the function instead?
Contributor
Author
There was a problem hiding this comment. Updated the setup too use function params instead of env variables by reading the pull secret path from Pulumi config in openshift.go, adding support for |
||
| if pullSecretPath == "" { | ||
| return fmt.Errorf("PULL_SECRET_PATH environment variable is not set") | ||
| } | ||
|
|
||
| crcSetup, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{ | ||
| Create: pulumi.String("crc setup"), | ||
| }, opts...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| startCluster, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{ | ||
| Create: pulumi.Sprintf("crc start -p %s", pullSecretPath), | ||
| Delete: pulumi.String("crc stop"), | ||
| Triggers: pulumi.Array{ | ||
| pulumi.String(pullSecretPath), | ||
| }, | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(crcSetup))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| kubeConfigCmd, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{ | ||
| Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"), | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCluster))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| clusterComp.KubeConfig = kubeConfigCmd.StdoutOutput() | ||
| clusterComp.ClusterName = pulumi.String("openshift").ToStringOutput() | ||
| return nil | ||
| }, opts...) | ||
| } | ||
|
|
||
| func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, opts ...pulumi.ResourceOption) (*Cluster, error) { | ||
| return components.NewComponent(env, name, func(clusterComp *Cluster) error { | ||
| openShiftClusterName := env.CommonNamer().DisplayName(49) | ||
| opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp)) | ||
| runner := vm.OS.Runner() | ||
| commonEnvironment := env | ||
|
|
||
| infraConfig := sdkconfig.New(env.Ctx(), "ddinfra") | ||
| pullSecretPath := infraConfig.Require("openShiftPullSecretPath") | ||
|
|
||
| openShiftInstallBinary, err := InstallOpenShiftBinary(env, vm, opts...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| pullSecretContent, err := utils.ReadSecretFile(pullSecretPath) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| pullSecretFile, err := vm.OS.FileManager().CopyInlineFile( | ||
| pullSecretContent, | ||
| "/tmp/pull-secret.txt", | ||
| ) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| installLibvirt, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-libvirt"), &command.Args{ | ||
| Create: pulumi.String(` | ||
| sudo dnf install -y libvirt NetworkManager`), | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(openShiftInstallBinary))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| // To avoid the crc-daemon.service being stopped when the user session ends, we enable linger for the user | ||
| enableLinger, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("enable-linger"), &command.Args{ | ||
| Create: pulumi.String("loginctl enable-linger"), | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(installLibvirt))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| setupCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{ | ||
| Create: pulumi.String("crc setup"), | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(pullSecretFile, enableLinger))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| startCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-start"), &command.Args{ | ||
| Create: pulumi.String(`crc start -p /tmp/pull-secret.txt`), | ||
| Delete: pulumi.String("crc stop && crc delete && crc cleanup"), | ||
| Triggers: pulumi.Array{ | ||
| pulumi.String(pullSecretPath), | ||
| }, | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(setupCRC))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| kubeConfig, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{ | ||
| Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"), | ||
| }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| clusterComp.KubeConfig = kubeConfig.StdoutOutput() | ||
| clusterComp.ClusterName = openShiftClusterName.ToStringOutput() | ||
| return nil | ||
| }, opts...) | ||
| } | ||
|
|
||
| func InstallOpenShiftBinary(env config.Env, vm *remote.Host, opts ...pulumi.ResourceOption) (pulumi.Resource, error) { | ||
| openShiftArch := vm.OS.Descriptor().Architecture | ||
| if openShiftArch == oscomp.AMD64Arch { | ||
| openShiftArch = "amd64" | ||
| } | ||
| return vm.OS.Runner().Command( | ||
| env.CommonNamer().ResourceName("crc-install"), | ||
| &command.Args{ | ||
| Create: pulumi.Sprintf(`curl -fsSL https://developers.redhat.com/content-gateway/file/pub/openshift-v4/clients/crc/2.52.0/crc-linux-%s.tar.xz | \ | ||
| sudo tar -xJ -C /usr/local/bin --strip-components=1 crc-linux-2.52.0-%s/crc`, openShiftArch, openShiftArch), | ||
| }, opts...) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| package openshiftvm | ||
|
|
||
| import ( | ||
| localKubernetes "github.com/DataDog/test-infra-definitions/components/kubernetes" | ||
|
Member
There was a problem hiding this comment. Let's not alias it localKubernetes here, because it actually create the openshift cluster in a remote VM on GCP |
||
| "github.com/DataDog/test-infra-definitions/components/os" | ||
| resGcp "github.com/DataDog/test-infra-definitions/resources/gcp" | ||
| "github.com/DataDog/test-infra-definitions/scenarios/gcp/compute" | ||
| "github.com/pulumi/pulumi/sdk/v3/go/pulumi" | ||
| ) | ||
|
|
||
| func Run(ctx *pulumi.Context) error { | ||
| gcpEnv, err := resGcp.NewEnvironment(ctx) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| osDesc := os.DescriptorFromString("redhat:9", os.RedHat9) | ||
| vm, err := compute.NewVM(gcpEnv, "openshift", | ||
| compute.WithOS(osDesc), | ||
| compute.WithInstanceType("n2-standard-8")) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if err := vm.Export(ctx, nil); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| openshiftCluster, err := localKubernetes.NewOpenShiftCluster(&gcpEnv, vm, "openshift") | ||
| if err != nil { | ||
| return err | ||
| } | ||
| if err := openshiftCluster.Export(ctx, nil); err != nil { | ||
| return err | ||
| } | ||
| return openshiftCluster.Export(ctx, nil) | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What are the condition to be able to run that locally? Is it working on MacOS laptops?
Uh oh!
There was an error while loading. Please reload this page.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This was a commit where I ran the function and confirmed it worked on macOS. I modeled this after the unused NewLocalKindCluster setup. In both situations the local setups aren't referenced anywhere, so I'm fine with removing it if you think that's appropriate.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Actually the NewLocalKindCluster setup is used in a provisioner on datadog-agent side: https://github.com/DataDog/datadog-agent/blob/main/test/new-e2e/pkg/provisioners/local/kubernetes/kind.go#L132
The goal is provide a local setup that allows you to iterate faster when creating a test for the first time
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Ah, I see. I’ve implemented functionality for passing the pull secret as a parameter for both the local cluster and cluster on GCP. However, I didn’t add an invoke command for the local cluster creation to keep consistency with other cluster types. For the datadog-agent integration, I anticipate needing to adapt the pull secret config approach. Currently, the GCP OpenShift cluster receives the pull secret through the invoke setup process, but the datadog-agent environment may require a different configuration pattern. I’d appreciate any guidance for how to handle that…