Use Linux image to execute E2E tests (#42832)

### What does this PR do?

Use classic Linux image for E2E tests. This will make the maintenance easier by no longer need to maintain a dedicated e2e tests image.
As a consequences we need to fetch the Pulumi plugins in the CI. It can be done easily in the go_e2e_deps jobs and the Pulumi are then cached.

### Motivation

Ease maintenance of e2e tests image. This is also an unblocker for merging test-infra-definitions inside datadog-agent

### Describe how you validated your changes

Complete CI running.

### Additional Notes


Co-authored-by: kevin.fairise <kevin.fairise@datadoghq.com>

Author: KevinFairise2

.github/CODEOWNERS

@@ -154,7 +154,6 @@
 /.gitlab/e2e_install_packages/windows.yml            @DataDog/agent-onboarding @DataDog/windows-products
 
 /.gitlab/common/                                     @DataDog/agent-devx
-/.gitlab/common/test_infra_version.yml               @DataDog/agent-devx
 
 /.gitlab/e2e/e2e.yml                                 @DataDog/container-integrations @DataDog/agent-devx @DataDog/fleet
 /.gitlab/container_build/fakeintake.yml              @DataDog/agent-e2e-testing @DataDog/agent-devx

.github/chainguard/self.gitlab.read.sts.yaml

@@ -0,0 +1,10 @@
+issuer: https://gitlab.ddbuild.io
+
+subject_pattern: "project_path:DataDog/.*"
+
+claim_pattern:
+  project_id: "4670"
+  ref: "*"
+  ref_protected: "true"
+permissions:
+  contents: read

.gitlab-ci.yml

@@ -9,7 +9,6 @@ include:
   - .gitlab/choco_build/choco_build.yml
   - .gitlab/common/shared.yml
   - .gitlab/common/skip_ci_check.yml
-  - .gitlab/common/test_infra_version.yml
   - .gitlab/container_build/include.yml
   - .gitlab/container_scan/container_scan.yml
   - .gitlab/deploy_packages/include.yml

.gitlab/check_merge/do_not_merge.yml

@@ -19,8 +19,7 @@ do-not-merge:
          [ -n "$CI_IMAGE_LINUX_RPM_X64_SUFFIX" ] ||
          [ -n "$CI_IMAGE_LINUX_RPM_ARM64_SUFFIX" ] ||
          [ -n "$CI_IMAGE_LINUX_RPM_ARMHF_SUFFIX" ] ||
-         [ -n "$CI_IMAGE_LINUX_WIN_LTSC2022_X64_SUFFIX" ] ||
-         [ -n "$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX" ]; then
+         [ -n "$CI_IMAGE_LINUX_WIN_LTSC2022_X64_SUFFIX" ]; then
         echo "Pull request uses non-empty BUILDIMAGES_SUFFIX variable"
         echo "This workflow fails so that the pull request cannot be merged"
         exit 1

.gitlab/common/test_infra_version.yml

@@ -15,6 +15,10 @@
   - mkdir -p $GOPATH/pkg/mod/cache && tar xJf modcache_e2e.tar.xz -C $GOPATH/pkg/mod/cache
   - rm -f modcache_e2e.tar.xz
 
+.retrieve_linux_pulumi_plugins:
+  - mkdir -p ~/.pulumi && tar xJf pulumi_plugins.tar.xz -C ~/.pulumi
+  - rm -f pulumi_plugins.tar.xz
+
 .cache:
   stage: deps_fetch
   image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
@@ -89,15 +93,33 @@ go_tools_deps:
         - modcache_tools.tar.xz
 
 go_e2e_deps:
+  id_tokens:
+    DDOCTOSTS_ID_TOKEN:
+      aud: dd-octo-sts
   extends: .cache
+  before_script:
+    - dd-octo-sts version
+    - dd-octo-sts debug --scope DataDog/datadog-agent --policy self.gitlab.deps-fetch
+    - export GITHUB_TOKEN=$(dd-octo-sts token --scope DataDog/datadog-agent --policy self.gitlab.deps-fetch)
   script:
-    - if [ -f modcache_e2e.tar.xz  ]; then exit 0; fi
-    - dda inv -- -e new-e2e-tests.deps
-    - cd $GOPATH/pkg/mod/cache/ && tar c -I "xz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz .
+    - |
+      if [ ! -f modcache_e2e.tar.xz ]; then
+        dda inv -- -e new-e2e-tests.deps
+        tar c -I "xz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/modcache_e2e.tar.xz -C $GOPATH/pkg/mod/cache/ .
+      fi
+    - |
+      if [ ! -f pulumi_plugins.tar.xz ]; then
+        # Pre-download Pulumi plugins used by e2e scenarios and components, should no longer be needed when we move test-infra-definitions to datadog-agent
+        pushd ./test/new-e2e
+        PULUMI_CONFIG_PASSPHRASE=dummy pulumi --non-interactive plugin install
+        popd
+        tar c -I "xz -T${KUBERNETES_CPU_REQUEST}" -f $CI_PROJECT_DIR/pulumi_plugins.tar.xz -C ~/.pulumi plugins
+      fi
   artifacts:
     expire_in: 1 day
     paths:
       - $CI_PROJECT_DIR/modcache_e2e.tar.xz
+      - $CI_PROJECT_DIR/pulumi_plugins.tar.xz
   cache:
     - key:
         files:
@@ -106,6 +128,13 @@ go_e2e_deps:
         prefix: "go_e2e_deps_modcache"
       paths:
         - modcache_e2e.tar.xz
+    - key:
+        files:
+          - ./test/new-e2e/go.mod
+          - .gitlab/deps_fetch/deps_fetch.yml
+        prefix: "pulumi_plugins"
+      paths:
+        - pulumi_plugins.tar.xz
 
 fetch_openjdk:
   needs: []

.gitlab/deps_fetch/deps_fetch.yml

@@ -3,12 +3,15 @@
 # Contains test jobs based on the new-e2e tests framework
 .new_e2e_template:
   stage: e2e
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   tags: ["arch:amd64", "specific:true"]
   needs:
     - !reference [.needs_new_e2e_template]
   before_script:
     - !reference [.retrieve_linux_go_e2e_deps]
+    - !reference [.retrieve_linux_pulumi_plugins]
+    - !reference [.retrieve_linux_go_tools_deps]
+    - dda inv -- -e install-tools
     # Setup AWS Credentials
     - mkdir -p ~/.aws
     - |
@@ -121,6 +124,7 @@
 .needs_new_e2e_template:
   - go_e2e_deps
   - go_e2e_test_binaries
+  - go_tools_deps
   - job: new-e2e-base-coverage
     optional: true
 
@@ -142,6 +146,7 @@ new-e2e-base:
   extends: .new_e2e_template
   needs:
     - go_e2e_deps
+    - go_tools_deps
     - go_e2e_test_binaries
     - agent_deb-x64-a7
   rules:
@@ -157,7 +162,7 @@ new-e2e-base:
 # This job runs early in the pipeline and creates artifacts that can be reused by test jobs
 go_e2e_test_binaries:
   stage: binary_build
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   tags: ["arch:amd64", "specific:true"]
   needs:
     - go_e2e_deps
@@ -246,6 +251,7 @@ new-e2e-containers-eks-init:
   extends: .new_e2e_template
   needs:
     - go_e2e_deps
+    - go_tools_deps
   rules:
     - !reference [.on_container_or_e2e_changes]
     - !reference [.manual]
@@ -454,6 +460,7 @@ new-e2e-npm-eks-init:
   extends: .new_e2e_template
   needs:
     - go_e2e_deps
+    - go_tools_deps
   rules:
     - !reference [.on_npm_or_e2e_changes]
     - !reference [.manual]
@@ -844,6 +851,7 @@ new-e2e-otel-eks-init:
     - !reference [.manual]
   needs:
     - go_e2e_deps
+    - go_tools_deps
   variables:
     TARGETS: ./tests/otel
     TEAM: otel
@@ -1068,7 +1076,7 @@ trigger-fips-e2e:
     strategy: depend
 
 new-e2e-cleanup-on-failure:
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   tags: ["arch:amd64", "specific:true"]
   stage: e2e_cleanup
   dependencies: [go_e2e_deps]

.gitlab/e2e/e2e.yml

@@ -8,6 +8,7 @@ e2e_pre_test:
   needs:
     - go_e2e_deps
     - go_e2e_test_binaries
+    - go_tools_deps
     - job: publish_fakeintake
       optional: true
   script:

.gitlab/e2e_pre_test/e2e_pre_test.yml

@@ -133,7 +133,7 @@
   extends:
     - .kmt_ec2_location_us_east_1
   stage: kernel_matrix_testing_prepare
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   needs: ["go_deps", "go_tools_deps"]
   tags: ["arch:amd64", "specific:true"]
   variables:
@@ -206,7 +206,7 @@
 
 .kmt_cleanup:
   stage: kernel_matrix_testing_cleanup
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   tags: ["arch:amd64", "specific:true"]
   before_script:
     - !reference [.kmt_new_profile]

.gitlab/kernel_matrix_testing/common.yml

@@ -184,7 +184,7 @@ go_mod_tidy_check:
 
 new-e2e-unit-tests:
   extends: .linux_tests
-  image: registry.ddbuild.io/ci/test-infra-definitions/runner$TEST_INFRA_DEFINITIONS_BUILDIMAGES_SUFFIX:$TEST_INFRA_DEFINITIONS_BUILDIMAGES
+  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
   tags: ["arch:amd64", "specific:true"]
   needs:
     - go_e2e_deps
@@ -201,6 +201,7 @@ new-e2e-unit-tests:
     # Use S3 backend
     - pulumi login "s3://dd-pulumi-state?region=us-east-1&awssdk=v2&profile=$AWS_PROFILE"
   script:
+    - dda inv -- -e install-tools
     - dda inv -- -e gitlab.generate-ci-visibility-links --output=$EXTERNAL_LINKS_PATH
     - dda inv -- -e new-e2e-tests.run --targets ./pkg/utils --junit-tar junit-${CI_JOB_ID}.tgz ${EXTRA_PARAMS}
   after_script:
@@ -209,6 +210,4 @@ new-e2e-unit-tests:
     KUBERNETES_MEMORY_REQUEST: 12Gi
     KUBERNETES_MEMORY_LIMIT: 16Gi
     KUBERNETES_CPU_REQUEST: 6
-    # Not using the entrypoint script for the e2e runner image
-    FF_KUBERNETES_HONOR_ENTRYPOINT: false
   timeout: 20m  # Not less than 20m because job startup can take time.