Skip to content

Commit 691a63b

Browse files
KevinFairise2chouetz
KevinFairise2
and
chouetz
authored
[ACIX-1891] Disable internet in all host tests by default, add an option to enable it (#54158)
### What does this PR do? Use a dedicated security group for hosts, that will not allow reaching internet by default, tests that still need internet must call it explicitly in the test definition. Long term goal is to reduce the number of tests that need to reach internet, to improve reliability of those tests ### Motivation Improve reliability enforce best practices ### Describe how you validated your changes ### Additional Notes Co-authored-by: nicolas.schweitzer <nicolas.schweitzer@datadoghq.com>
1 parent e98625f commit 691a63b

72 files changed

Lines changed: 181 additions & 88 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎test/e2e-framework/components/datadog/agent/host_linuxos.go‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -35,16 +35,17 @@ func (am *agentLinuxManager) getInstallCommand(version agentparams.PackageVersio
3535
testEnvVars := []string{}
3636

3737
if version.PipelineID != "" {
38-
testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_APT_URL=apttesting.datad0g.com/datadog-agent/pipeline-%v-a%v", version.PipelineID, version.Major))
38+
testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_APT_URL=s3.amazonaws.com/apttesting.datad0g.com/datadog-agent/pipeline-%v-a%v", version.PipelineID, version.Major))
3939
// apt testing repo
4040
// TESTING_APT_REPO_VERSION="pipeline-xxxxx-a7 7"
4141
testEnvVars = append(testEnvVars, fmt.Sprintf(`TESTING_APT_REPO_VERSION="stable-%[1]s %[2]v"`, am.targetOS.Descriptor().Architecture, version.Major))
42-
testEnvVars = append(testEnvVars, "TESTING_YUM_URL=yumtesting.datad0g.com")
42+
testEnvVars = append(testEnvVars, "TESTING_YUM_URL=s3.amazonaws.com/yumtesting.datad0g.com")
4343
// yum testing repo
4444
// TESTING_YUM_VERSION_PATH="testing/pipeline-xxxxx-a7/7"
4545
testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_YUM_VERSION_PATH=testing/pipeline-%[1]v-a%[2]v/%[2]v", version.PipelineID, version.Major))
4646
// target testing keys
47-
testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_KEYS_URL=apttesting.datad0g.com/test-keys"))
47+
testEnvVars = append(testEnvVars, fmt.Sprintf("TESTING_KEYS_URL=s3.amazonaws.com/apttesting.datad0g.com/test-keys"))
48+
testEnvVars = append(testEnvVars, "TESTING_REPORT_URL=undefined")
4849
} else {
4950
testEnvVars = append(testEnvVars, fmt.Sprintf("DD_AGENT_MAJOR_VERSION=%v", version.Major))
5051

@@ -53,8 +54,24 @@ func (am *agentLinuxManager) getInstallCommand(version agentparams.PackageVersio
5354
}
5455

5556
if version.Channel != "" && version.Channel != agentparams.StableChannel {
56-
testEnvVars = append(testEnvVars, "REPO_URL=datad0g.com")
57+
// Non-stable channel (e.g. beta): packages are on datad0g.com. Use S3 bucket
58+
// URLs directly instead of REPO_URL so that VMs without public internet access
59+
// can reach them via the S3 VPC gateway endpoint.
60+
testEnvVars = append(testEnvVars, "DD_REPO_URL=datad0g.com")
61+
testEnvVars = append(testEnvVars, "TESTING_REPORT_URL=undefined")
62+
testEnvVars = append(testEnvVars, "TESTING_APT_URL=s3.amazonaws.com/apt.datad0g.com")
63+
testEnvVars = append(testEnvVars, "TESTING_YUM_URL=s3.amazonaws.com/yum.datad0g.com")
64+
testEnvVars = append(testEnvVars, "TESTING_KEYS_URL=s3.amazonaws.com/public-signing-keys")
5765
testEnvVars = append(testEnvVars, fmt.Sprintf("DD_AGENT_DIST_CHANNEL=%s", version.Channel))
66+
} else {
67+
// No pipeline ID and stable channel: installing from the default datadoghq.com repos.
68+
// Use S3 bucket URLs directly instead of the CloudFront-backed domains so that
69+
// VMs without public internet access can reach them via the S3 VPC gateway endpoint.
70+
testEnvVars = append(testEnvVars, "TESTING_APT_URL=s3.amazonaws.com/apt.datadoghq.com")
71+
testEnvVars = append(testEnvVars, "TESTING_YUM_URL=s3.amazonaws.com/yum.datadoghq.com")
72+
testEnvVars = append(testEnvVars, "TESTING_KEYS_URL=s3.amazonaws.com/public-signing-keys")
73+
testEnvVars = append(testEnvVars, "TESTING_REPORT_URL=undefined")
74+
5875
}
5976
}
6077

‎test/e2e-framework/components/datadog/updater/install_script.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,4 +20,4 @@ TESTING_APT_URL="apttesting.datad0g.com/datadog-agent/pipeline-${DD_PIPELINE_ID}
2020
TESTING_APT_REPO_VERSION="stable-$(uname -m | sed 's/aarch64/arm64/; s/amd64/x86_64/') 7" \
2121
TESTING_YUM_URL="yumtesting.datad0g.com" \
2222
TESTING_YUM_VERSION_PATH="testing/pipeline-${DD_PIPELINE_ID}-a7/7" \
23-
bash -c "$(curl -L https://install.datadoghq.com/scripts/install_script_agent7.sh)"
23+
bash -c "$(curl -L https://s3.amazonaws.com/dd-agent/scripts/install_script_agent7.sh)"

‎test/e2e-framework/resources/aws/ec2/vm.go‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,8 @@
66
package ec2
77

88
import (
9+
"fmt"
10+
911
"github.com/DataDog/datadog-agent/test/e2e-framework/common/config"
1012
"github.com/DataDog/datadog-agent/test/e2e-framework/common/utils"
1113
"github.com/DataDog/datadog-agent/test/e2e-framework/resources/aws"
@@ -26,10 +28,11 @@ type InstanceArgs struct {
2628
InstanceProfile string
2729

2830
// Optional
29-
UserData string
30-
HTTPTokensRequired bool
31-
HostID pulumi.StringInput // For dedicated host tenancy
32-
VolumeThroughput int // GP3 volume throughput in MiB/s (125-1000)
31+
UserData string
32+
HTTPTokensRequired bool
33+
HostID pulumi.StringInput // For dedicated host tenancy
34+
VolumeThroughput int // GP3 volume throughput in MiB/s (125-1000)
35+
WithoutInternetAccess bool // Replaces the account's default security groups with groups resolved from e.NoInternetSecurityGroupNames()
3336
}
3437

3538
func NewInstance(e aws.Environment, name string, args InstanceArgs, opts ...pulumi.ResourceOption) (*ec2.Instance, error) {
@@ -43,12 +46,31 @@ func NewInstance(e aws.Environment, name string, args InstanceArgs, opts ...pulu
4346
rootBlockDevice.VolumeType = pulumi.String("gp3")
4447
rootBlockDevice.Throughput = pulumi.Int(args.VolumeThroughput)
4548
}
49+
var securityGroups []string
50+
if args.WithoutInternetAccess {
51+
sgNames := e.NoInternetSecurityGroupNames()
52+
if len(sgNames) == 0 {
53+
return nil, fmt.Errorf("no security groups configured to block internet access, set the %s config parameter", aws.DDInfraNoInternetSecurityGroupNamesParamName)
54+
}
55+
for _, sgName := range sgNames {
56+
sg, err := ec2.LookupSecurityGroup(e.Ctx(), &ec2.LookupSecurityGroupArgs{
57+
Name: pulumi.StringRef(sgName),
58+
VpcId: pulumi.StringRef(e.DefaultVPCID()),
59+
}, e.WithProvider(config.ProviderAWS))
60+
if err != nil {
61+
return nil, fmt.Errorf("failed to lookup no-internet security group %q: %w", sgName, err)
62+
}
63+
securityGroups = append(securityGroups, sg.Id)
64+
}
65+
} else {
66+
securityGroups = e.DefaultSecurityGroups()
67+
}
4668
instanceArgs := &ec2.InstanceArgs{
4769
Ami: pulumi.StringPtr(args.AMI),
4870
SubnetId: e.RandomSubnets().Index(pulumi.Int(0)),
4971
IamInstanceProfile: pulumi.StringPtr(args.InstanceProfile),
5072
InstanceType: pulumi.StringPtr(args.InstanceType),
51-
VpcSecurityGroupIds: pulumi.ToStringArray(e.DefaultSecurityGroups()),
73+
VpcSecurityGroupIds: pulumi.ToStringArray(securityGroups),
5274
KeyName: pulumi.StringPtr(args.KeyPairName),
5375
UserData: pulumi.StringPtr(args.UserData),
5476
UserDataReplaceOnChange: pulumi.BoolPtr(true),

‎test/e2e-framework/resources/aws/environment.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ const (
3131
DDInfraDefaultVPCIDParamName = "aws/defaultVPCID"
3232
DDInfraDefaultSubnetsParamName = "aws/defaultSubnets"
3333
DDInfraDefaultSecurityGroupsParamName = "aws/defaultSecurityGroups"
34+
DDInfraNoInternetSecurityGroupNamesParamName = "aws/noInternetSecurityGroupNames"
3435
DDInfraDefaultInstanceTypeParamName = "aws/defaultInstanceType"
3536
DDInfraDefaultInstanceProfileParamName = "aws/defaultInstanceProfile"
3637
DDInfraDefaultARMInstanceTypeParamName = "aws/defaultARMInstanceType"
@@ -258,6 +259,14 @@ func (e *Environment) DefaultSecurityGroups() []string {
258259
return e.GetStringListWithDefault(e.InfraConfig, DDInfraDefaultSecurityGroupsParamName, e.envDefault.ddInfra.defaultSecurityGroups)
259260
}
260261

262+
// NoInternetSecurityGroupNames returns the names of security groups to attach to a VM instead of
263+
// the account's default security groups when internet access must be blocked. The groups are
264+
// resolved to IDs at provisioning time via LookupSecurityGroup; they must already exist in the
265+
// account's VPC and be configured accordingly.
266+
func (e *Environment) NoInternetSecurityGroupNames() []string {
267+
return e.GetStringListWithDefault(e.InfraConfig, DDInfraNoInternetSecurityGroupNamesParamName, e.envDefault.ddInfra.noInternetSecurityGroupNames)
268+
}
269+
261270
func (e *Environment) DefaultInstanceType() string {
262271
return e.GetStringWithDefault(e.InfraConfig, DDInfraDefaultInstanceTypeParamName, e.envDefault.ddInfra.defaultInstanceType)
263272
}

‎test/e2e-framework/resources/aws/environmentDefaults.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ type ddInfra struct {
4040
defaultVPCID string
4141
defaultSubnets []SubnetConfig
4242
defaultSecurityGroups []string
43+
noInternetSecurityGroupNames []string
4344
defaultInstanceType string
4445
defaultInstanceProfileName string
4546
defaultARMInstanceType string
@@ -168,6 +169,7 @@ func agentSandboxDefault() environmentDefault {
168169
{ID: "subnet-003831c49a10df3dd", MacOSCompatible: false},
169170
},
170171
defaultSecurityGroups: []string{"sg-038231b976eb13d44", "sg-05466e7ce253d21b1"},
172+
noInternetSecurityGroupNames: []string{"no-internet-access"},
171173
defaultInstanceType: "t3.medium",
172174
defaultInstanceProfileName: "ec2InstanceRole",
173175
defaultARMInstanceType: "t4g.medium",
@@ -238,6 +240,7 @@ func agentQADefault() environmentDefault {
238240
{ID: "subnet-0dabe4bab92b2b9a7", MacOSCompatible: true}, // us-east-1b
239241
},
240242
defaultSecurityGroups: []string{"sg-05e9573fcc582f22c", "sg-0498c960a173dff1e"},
243+
noInternetSecurityGroupNames: []string{"no-internet-access"},
241244
defaultInstanceType: "t3.medium",
242245
defaultInstanceProfileName: "ec2InstanceRole",
243246
defaultARMInstanceType: "t4g.medium",

‎test/e2e-framework/scenarios/aws/ec2/vm.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -51,14 +51,15 @@ func NewVM(e aws.Environment, name string, params ...VMOption) (*remote.Host, er
5151
c.CloudProvider = pulumi.String(components.CloudProviderAWS).ToStringOutput()
5252

5353
instanceArgs := ec2.InstanceArgs{
54-
AMI: amiInfo.id,
55-
InstanceType: vmArgs.instanceType,
56-
UserData: vmArgs.userData,
57-
InstanceProfile: vmArgs.instanceProfile,
58-
HTTPTokensRequired: vmArgs.httpTokensRequired,
59-
Tenancy: vmArgs.tenancy,
60-
HostID: pulumi.String(vmArgs.hostID),
61-
VolumeThroughput: vmArgs.volumeThroughput,
54+
AMI: amiInfo.id,
55+
InstanceType: vmArgs.instanceType,
56+
UserData: vmArgs.userData,
57+
InstanceProfile: vmArgs.instanceProfile,
58+
HTTPTokensRequired: vmArgs.httpTokensRequired,
59+
Tenancy: vmArgs.tenancy,
60+
HostID: pulumi.String(vmArgs.hostID),
61+
VolumeThroughput: vmArgs.volumeThroughput,
62+
WithoutInternetAccess: vmArgs.withoutInternetAccess,
6263
StorageSize: vmArgs.storageSize,
6364
}
6465

‎test/e2e-framework/scenarios/aws/ec2/vmargs.go‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
// - [WithHostID]
2727
// - [WithTenancy]
2828
// - [WithVolumeThroughput]
29+
// - [WithoutInternetAccess]
2930
// - [WithPulumiResourceOptions]
3031
//
3132
// [Functional options pattern]: https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis
@@ -40,6 +41,7 @@ type vmArgs struct {
4041
tenancy string
4142
hostID string
4243

44+
withoutInternetAccess bool
4345
httpTokensRequired bool
4446
volumeThroughput int // GP3 volume throughput in MiB/s (125-1000, default 125)
4547
storageSize int // root volume size in GiB (0 = account default)
@@ -149,6 +151,29 @@ func WithVolumeThroughput(throughput int) VMOption {
149151
}
150152
}
151153

154+
// WithoutInternetAccess replaces the account's default security groups with the ones
155+
// configured to block internet access (see aws.Environment.NoInternetSecurityGroupNames).
156+
// These security groups must already exist and be configured accordingly, since AWS
157+
// security group rules are additive across all groups attached to an instance: attaching
158+
// a restrictive group in addition to the permissive defaults would not restrict anything.
159+
func WithoutInternetAccess() VMOption {
160+
return func(p *vmArgs) error {
161+
p.withoutInternetAccess = true
162+
return nil
163+
}
164+
}
165+
166+
// WithInternetAccess explicitly opts the VM into internet access, overriding any
167+
// no-internet default set by the provisioner. Tests that need to reach external
168+
// hosts (e.g. to download packages, pull images, or contact external APIs) must
169+
// call this option explicitly.
170+
func WithInternetAccess() VMOption {
171+
return func(p *vmArgs) error {
172+
p.withoutInternetAccess = false
173+
return nil
174+
}
175+
}
176+
152177
// WithStorageSize sets the root volume size in GiB. When unset (0) the account
153178
// default is used. Needed for hosts whose root must exceed the default (e.g. a
154179
// bare-metal host storing large nested-VM qcow2 images on its EBS root).

‎test/e2e-framework/testing/provisioners/aws/host/host.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,8 @@ func Provisioner(opts ...ProvisionerOption) provisioners.TypedProvisioner[enviro
7474
provisioner := provisioners.NewTypedPulumiProvisioner(provisionerBaseID+runParams.Name, func(ctx *pulumi.Context, env *environments.Host) error {
7575
// We ALWAYS need to make a deep copy of `params`, as the provisioner can be called multiple times.
7676
// and it's easy to forget about it, leading to hard to debug issues.
77+
// Prepend the no-internet default so that caller-provided opts (e.g. ec2.WithInternetAccess()) can override it.
78+
opts = append([]ProvisionerOption{WithRunOptions(ec2.WithEC2InstanceOptions(ec2.WithoutInternetAccess()))}, opts...)
7779
params := getProvisionerParams(opts...)
7880
runParams := ec2.GetParams(params.runOptions...)
7981

‎test/new-e2e/examples/agentenv_logs_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ func TestVMLogsExampleSuite(t *testing.T) {
3232
e2e.Run(t, &vmLogsExampleSuite{}, e2e.WithProvisioner(
3333
awshost.Provisioner(
3434
awshost.WithRunOptions(
35+
ec2.WithEC2InstanceOptions(ec2.WithoutInternetAccess()),
3536
ec2.WithAgentOptions(
3637
agentparams.WithIntegration("custom_logs.d", customLogsConfig),
3738
agentparams.WithLogs(),

‎test/new-e2e/examples/vm_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package examples
88
import (
99
"testing"
1010

11+
"github.com/DataDog/datadog-agent/test/e2e-framework/scenarios/aws/ec2"
1112
"github.com/DataDog/datadog-agent/test/e2e-framework/testing/e2e"
1213
"github.com/DataDog/datadog-agent/test/e2e-framework/testing/environments"
1314
awshost "github.com/DataDog/datadog-agent/test/e2e-framework/testing/provisioners/aws/host"
@@ -19,7 +20,7 @@ type vmSuite struct {
1920

2021
// TestVMSuite runs tests for the VM interface to ensure its implementation is correct.
2122
func TestVMSuite(t *testing.T) {
22-
suiteParams := []e2e.SuiteOption{e2e.WithProvisioner(awshost.ProvisionerNoAgentNoFakeIntake())}
23+
suiteParams := []e2e.SuiteOption{e2e.WithProvisioner(awshost.ProvisionerNoAgentNoFakeIntake(awshost.WithRunOptions(ec2.WithEC2InstanceOptions(ec2.WithoutInternetAccess()))))}
2324

2425
e2e.Run(t, &vmSuite{}, suiteParams...)
2526
}

0 commit comments

Comments
 (0)