Skip to content

Commit 16e9d0d

Browse files
shaina-patel-23shaina-patel-23
committed
WIP
1 parent 22dab10 commit 16e9d0d

11 files changed

Lines changed: 353 additions & 146 deletions

File tree

‎components/kubernetes/openshift.go‎

Lines changed: 12 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package kubernetes
22

33
import (
4-
_ "embed"
5-
"fmt"
6-
"os"
74
"regexp"
85
"strings"
96

@@ -14,22 +11,17 @@ import (
1411
oscomp "github.com/DataDog/test-infra-definitions/components/os"
1512
"github.com/DataDog/test-infra-definitions/components/remote"
1613
"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
17-
// sdkconfig "github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
1814
)
1915

20-
func NewLocalOpenShiftCluster(env config.Env, name string, opts ...pulumi.ResourceOption) (*Cluster, error) {
16+
func NewLocalOpenShiftCluster(env config.Env, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) {
2117
return components.NewComponent(env, name, func(clusterComp *Cluster) error {
18+
openShiftClusterName := env.CommonNamer().DisplayName(49)
2219
opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
2320
commonEnvironment := env
2421
runner := command.NewLocalRunner(env, command.LocalRunnerArgs{
2522
OSCommand: command.NewUnixOSCommand(),
2623
})
2724

28-
pullSecretPath := os.Getenv("PULL_SECRET_PATH")
29-
if pullSecretPath == "" {
30-
return fmt.Errorf("PULL_SECRET_PATH environment variable is not set")
31-
}
32-
3325
crcSetup, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{
3426
Create: pulumi.String("crc setup"),
3527
}, opts...)
@@ -55,25 +47,18 @@ func NewLocalOpenShiftCluster(env config.Env, name string, opts ...pulumi.Resour
5547
}
5648

5749
clusterComp.KubeConfig = kubeConfigCmd.StdoutOutput()
58-
clusterComp.ClusterName = pulumi.String("openshift").ToStringOutput()
50+
clusterComp.ClusterName = openShiftClusterName.ToStringOutput()
5951
return nil
6052
}, opts...)
6153
}
6254

63-
func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, opts ...pulumi.ResourceOption) (*Cluster, error) {
55+
func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, pullSecretPath string, opts ...pulumi.ResourceOption) (*Cluster, error) {
6456
return components.NewComponent(env, name, func(clusterComp *Cluster) error {
6557
openShiftClusterName := env.CommonNamer().DisplayName(49)
6658
opts = utils.MergeOptions[pulumi.ResourceOption](opts, pulumi.Parent(clusterComp))
6759
runner := vm.OS.Runner()
6860
commonEnvironment := env
6961

70-
// infraConfig := sdkconfig.New(env.Ctx(), "ddinfra")
71-
// pullSecretPath := infraConfig.Require("openShiftPullSecretPath")
72-
pullSecretPath := os.Getenv("PULL_SECRET_PATH")
73-
if pullSecretPath == "" {
74-
return fmt.Errorf("PULL_SECRET_PATH environment variable is not set")
75-
}
76-
7762
openShiftInstallBinary, err := InstallOpenShiftBinary(env, vm, opts...)
7863
if err != nil {
7964
return err
@@ -107,7 +92,7 @@ func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, opts ...p
10792
}
10893

10994
setupCRC, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("crc-setup"), &command.Args{
110-
Create: pulumi.String("crc setup"),
95+
Create: pulumi.String("crc config set disk-size 100 && crc config set cpus 6 && crc config set memory 12288 && crc setup"),
11196
}, utils.MergeOptions(opts, utils.PulumiDependsOn(enableLinger, pullSecretFile))...)
11297
if err != nil {
11398
return err
@@ -123,111 +108,25 @@ func NewOpenShiftCluster(env config.Env, vm *remote.Host, name string, opts ...p
123108
if err != nil {
124109
return err
125110
}
126-
// Option 1: Using socat (current approach, works but on next try try this without the nohup)
127-
// socatInstall, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-socat"), &command.Args{
128-
// Create: pulumi.String("sudo dnf install -y socat"),
129-
// }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
130-
// if err != nil {
131-
// return err
132-
// }
133-
134-
// socatForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("socat-kubeapi-proxy"), &command.Args{
135-
// Create: pulumi.String(`
136-
// sudo nohup socat TCP-LISTEN:8443,bind=0.0.0.0,fork TCP:127.0.0.1:6443 > /tmp/socat.log 2>&1 &
137-
// `),
138-
// Delete: pulumi.String(`
139-
// sudo pkill -f "socat TCP-LISTEN:8443" || true
140-
// `),
141-
// }, utils.MergeOptions(opts, utils.PulumiDependsOn(socatInstall))...)
142-
143-
// Option 2: Using UFW (haven't tried this yet)
144-
ufwInstall, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-ufw"), &command.Args{
145-
Create: pulumi.String("sudo dnf install -y ufw"),
111+
socatInstall, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("install-socat"), &command.Args{
112+
Create: pulumi.String("sudo dnf install -y socat"),
146113
}, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
147114
if err != nil {
148115
return err
149116
}
150117

151-
ufwForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("ufw-forwarding"), &command.Args{
118+
socatForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("socat-kubeapi-proxy"), &command.Args{
152119
Create: pulumi.String(`
153-
sudo ufw enable && \
154-
sudo ufw allow 8443 && \
155-
sudo ufw route allow from any port 8443 to 127.0.0.1 port 6443 && \
156-
sudo su -c 'ufw export save > /etc/ufw/user.rules'
120+
sudo nohup socat TCP-LISTEN:8443,bind=0.0.0.0,fork TCP:127.0.0.1:6443 > /tmp/socat.log 2>&1 &
157121
`),
158122
Delete: pulumi.String(`
159-
sudo ufw route delete allow from any port 8443 to 127.0.0.1 port 6443 || true
123+
sudo pkill -f "socat TCP-LISTEN:8443" || true
160124
`),
161-
}, utils.MergeOptions(opts, utils.PulumiDependsOn(ufwInstall))...)
162-
if err != nil {
163-
return err
164-
}
165-
166-
// Option 3: Using firewalld (haven't tried this yet)
167-
// firewalldForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("firewalld-forwarding"), &command.Args{
168-
// Create: pulumi.String(`
169-
// sudo systemctl start firewalld && \
170-
// sudo systemctl enable firewalld && \
171-
// sudo firewall-cmd --zone=public --add-port=8443/tcp --permanent && \
172-
// sudo firewall-cmd --zone=public --add-forward-port=port=8443:proto=tcp:toport=6443:toaddr=127.0.0.1 --permanent && \
173-
// sudo firewall-cmd --reload
174-
// `),
175-
// Delete: pulumi.String(`
176-
// sudo firewall-cmd --zone=public --remove-port=8443/tcp --permanent || true && \
177-
// sudo firewall-cmd --zone=public --remove-forward-port=port=8443:proto=tcp:toport=6443:toaddr=127.0.0.1 --permanent || true && \
178-
// sudo firewall-cmd --reload || true
179-
// `),
180-
// }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
181-
// if err != nil {
182-
// return err
183-
// }
184-
185-
// Option 4: Using nftables (haven't tried this yet)
186-
// nftablesForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("nftables-forwarding"), &command.Args{
187-
// Create: pulumi.String(`
188-
// sudo systemctl start nftables && \
189-
// sudo systemctl enable nftables && \
190-
// sudo nft add table ip nat && \
191-
// sudo nft add chain ip nat PREROUTING { type nat hook prerouting priority 0 \; } && \
192-
// sudo nft add rule ip nat PREROUTING tcp dport 8443 dnat to 127.0.0.1:6443 && \
193-
// sudo nft add table ip filter && \
194-
// sudo nft add chain ip filter FORWARD && \
195-
// sudo nft add rule ip filter FORWARD tcp daddr 127.0.0.1 dport 6443 accept
196-
// `),
197-
// Delete: pulumi.String(`
198-
// sudo nft delete rule ip nat PREROUTING tcp dport 8443 dnat to 127.0.0.1:6443 || true && \
199-
// sudo nft delete rule ip filter FORWARD tcp daddr 127.0.0.1 dport 6443 accept || true && \
200-
// sudo nft delete chain ip nat PREROUTING || true && \
201-
// sudo nft delete chain ip filter FORWARD || true && \
202-
// sudo nft delete table ip nat || true && \
203-
// sudo nft delete table ip filter || true
204-
// `),
205-
// }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
206-
// if err != nil {
207-
// return err
208-
// }
209-
210-
//Option 5: Using iptables (tried and didn't work so far)
211-
// iptablesForwarding, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("iptables-forwarding"), &command.Args{
212-
// Create: pulumi.Sprintf(`
213-
// sudo iptables -t nat -A PREROUTING -p tcp --dport 8443 -j DNAT --to-destination 127.0.0.1:6443 && \
214-
// sudo iptables -t nat -A OUTPUT -p tcp -d %[1]s --dport 8443 -j DNAT --to-destination 127.0.0.1:6443 && \
215-
// sudo iptables -A FORWARD -p tcp -d 127.0.0.1 --dport 6443 -j ACCEPT && \
216-
// sudo iptables -t nat -A POSTROUTING -j MASQUERADE
217-
// `, vm.Address),
218-
// Delete: pulumi.String(`
219-
// sudo iptables -t nat -D PREROUTING -p tcp --dport 8443 -j DNAT --to-destination 127.0.0.1:6443 || true && \
220-
// sudo iptables -D FORWARD -p tcp -d 127.0.0.1 --dport 6443 -j ACCEPT || true && \
221-
// sudo iptables -t nat -D POSTROUTING -j MASQUERADE || true
222-
// `),
223-
// }, utils.MergeOptions(opts, utils.PulumiDependsOn(startCRC))...)
224-
// if err != nil {
225-
// return err
226-
// }
125+
}, utils.MergeOptions(opts, utils.PulumiDependsOn(socatInstall))...)
227126

228127
kubeConfig, err := runner.Command(commonEnvironment.CommonNamer().ResourceName("get-kubeconfig"), &command.Args{
229128
Create: pulumi.String("cat ~/.crc/machines/crc/kubeconfig"),
230-
}, utils.MergeOptions(opts, utils.PulumiDependsOn(ufwForwarding))...)
129+
}, utils.MergeOptions(opts, utils.PulumiDependsOn(socatForwarding))...)
231130
if err != nil {
232131
return err
233132
}

‎resources/gcp/compute/vm.go‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -24,13 +24,17 @@ func NewLinuxInstance(e gcp.Environment, name string, imageName string, instance
2424
Subnetwork: pulumi.String(e.DefaultSubnet()),
2525
},
2626
},
27-
Name: e.Namer.DisplayName(63, pulumi.String(name)),
28-
MachineType: pulumi.String(instanceType),
29-
AllowStoppingForUpdate: pulumi.Bool(true),
30-
CanIpForward: pulumi.Bool(true),
31-
AdvancedMachineFeatures: &compute.InstanceAdvancedMachineFeaturesArgs{
32-
EnableNestedVirtualization: pulumi.Bool(true),
33-
},
27+
Name: e.Namer.DisplayName(63, pulumi.String(name)),
28+
MachineType: pulumi.String(instanceType),
29+
CanIpForward: pulumi.Bool(true),
30+
AdvancedMachineFeatures: func() *compute.InstanceAdvancedMachineFeaturesArgs {
31+
if e.EnableNestedVirtualization() {
32+
return &compute.InstanceAdvancedMachineFeaturesArgs{
33+
EnableNestedVirtualization: pulumi.Bool(true),
34+
}
35+
}
36+
return nil
37+
}(),
3438
Tags: pulumi.StringArray{
3539
pulumi.String("appgate-gateway"),
3640
pulumi.String("nat-us-central1"),

‎resources/gcp/environment.go‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,8 @@ const (
2626
DDInfraDefaultZoneNameParamName = "gcp/defaultZone"
2727
DDInfraDefautVMServiceAccountParamName = "gcp/defaultVMServiceAccount"
2828
DDInfraGKEEnableAutopilot = "gcp/gke/enableAutopilot"
29+
DDInfraOpenShiftPullSecretPath = "gcp/openshift/pullSecretPath"
30+
DDInfraEnableNestedVirtualization = "gcp/enableNestedVirtualization"
2931
)
3032

3133
type Environment struct {
@@ -170,3 +172,13 @@ func (e *Environment) Region() string {
170172
func (e *Environment) Zone() string {
171173
return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultZoneNameParamName, e.envDefault.gcp.zone)
172174
}
175+
176+
// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file
177+
func (e *Environment) OpenShiftPullSecretPath() string {
178+
return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath)
179+
}
180+
181+
// EnableNestedVirtualization returns whether to enable nested virtualization
182+
func (e *Environment) EnableNestedVirtualization() bool {
183+
return e.GetBoolWithDefault(e.InfraConfig, DDInfraEnableNestedVirtualization, e.envDefault.ddInfra.openshift.nestedVirtualization)
184+
}

‎resources/gcp/environmentDefaults.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,12 +22,17 @@ type ddInfra struct {
2222
defaultSubnetName string
2323
defaultVMServiceAccount string
2424
gke ddInfraGKE
25+
openshift ddInfraOpenShift
2526
}
2627

2728
type ddInfraGKE struct {
2829
autopilot bool
2930
}
3031

32+
type ddInfraOpenShift struct {
33+
nestedVirtualization bool
34+
}
35+
3136
func getEnvironmentDefault(envName string) environmentDefault {
3237
switch envName {
3338
case agentSandboxEnv:
@@ -52,6 +57,7 @@ func agentSandboxDefault() environmentDefault {
5257
defaultSubnetName: "datadog-agent-sandbox-us-central1-private",
5358
defaultVMServiceAccount: "vmserviceaccount@datadog-agent-sandbox.iam.gserviceaccount.com",
5459
gke: ddInfraGKE{autopilot: false},
60+
openshift: ddInfraOpenShift{nestedVirtualization: false},
5561
},
5662
}
5763
}
@@ -69,6 +75,7 @@ func agentQaDefault() environmentDefault {
6975
defaultSubnetName: "datadog-agent-qa-us-central1-private",
7076
defaultVMServiceAccount: "vmserviceaccount@datadog-agent-qa.iam.gserviceaccount.com",
7177
gke: ddInfraGKE{autopilot: false},
78+
openshift: ddInfraOpenShift{nestedVirtualization: false},
7279
},
7380
}
7481
}

‎resources/local/environment.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@ import (
1010
const (
1111
localNamerNamespace = "local"
1212
// local Infra (local)
13-
DDInfraDefaultPublicKeyPath = "local/defaultPublicKeyPath"
13+
DDInfraDefaultPublicKeyPath = "local/defaultPublicKeyPath"
14+
DDInfraOpenShiftPullSecretPath = "local/openshift/pullSecretPath"
1415
)
1516

1617
type Environment struct {
@@ -62,3 +63,8 @@ func (e *Environment) InternalRegistryFullImagePathExists(_ string) (bool, error
6263
func (e *Environment) DefaultPublicKeyPath() string {
6364
return e.InfraConfig.Get(DDInfraDefaultPublicKeyPath)
6465
}
66+
67+
// OpenShiftPullSecretPath returns the path to the OpenShift pull secret file
68+
func (e *Environment) OpenShiftPullSecretPath() string {
69+
return e.InfraConfig.Get(DDInfraOpenShiftPullSecretPath)
70+
}

‎scenarios/gcp/compute/vmargs.go‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,3 @@ func WithOSArch(osDesc os.Descriptor, arch os.Architecture) VMOption {
3434
return nil
3535
}
3636
}
37-
38-
// WithInstanceType set the instance type
39-
func WithInstanceType(instanceType string) VMOption {
40-
return func(p *vmArgs) error {
41-
p.instanceType = instanceType
42-
return nil
43-
}
44-
}

‎scenarios/gcp/fakeintake/params.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import "github.com/DataDog/test-infra-definitions/common"
55
type Params struct {
66
DDDevForwarding bool
77
ImageURL string
8+
Memory int
89
}
910

1011
type Option = func(*Params) error
@@ -14,6 +15,7 @@ func NewParams(options ...Option) (*Params, error) {
1415
params := &Params{
1516
ImageURL: "gcr.io/datadoghq/fakeintake:latest",
1617
DDDevForwarding: true,
18+
Memory: 1024,
1719
}
1820
return common.ApplyOption(params, options)
1921
}
@@ -33,3 +35,11 @@ func WithoutDDDevForwarding() Option {
3335
return nil
3436
}
3537
}
38+
39+
// WithMemory sets the amount (in MiB) of memory to allocate to the fakeintake
40+
func WithMemory(memory int) Option {
41+
return func(p *Params) error {
42+
p.Memory = memory
43+
return nil
44+
}
45+
}

0 commit comments

Comments
 (0)