system_probe.yml

# System probe overrides
upload_dependencies_sysprobe_x64:
  extends:
    - .package_dependencies
  needs:
    - !reference [.kmt_setup_needs]
    - pull_test_dockers_x64
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    ARCH: x86_64
    INSTANCE_TYPE: r5d.metal
    TEST_COMPONENT: system-probe

upload_dependencies_sysprobe_arm64:
  extends:
    - .package_dependencies
  needs:
    - !reference [.kmt_setup_needs]
    - pull_test_dockers_arm64
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    ARCH: arm64
    INSTANCE_TYPE: r6gd.metal
    TEST_COMPONENT: system-probe

# --- Docker test images
.pull_test_dockers:
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/docker_x64$CI_IMAGE_DOCKER_X64_SUFFIX:$CI_IMAGE_DOCKER_X64
  needs: []
  tags: ["arch:amd64", "specific:true"]
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  stage: kernel_matrix_testing_prepare
  script:
    - !reference [.login_to_docker_readonly_crane]
    # Pull base images
    - mkdir $KMT_DOCKERS
    - dda inv -- -e system-probe.save-test-dockers --use-crane --output-dir $KMT_DOCKERS --arch $ARCH
  after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
    - !reference [.tag_kmt_ci_job]
  artifacts:
    expire_in: 1 day
    paths:
      - $KMT_DOCKERS
  variables:
    KMT_DOCKERS: $DD_AGENT_TESTING_DIR/kmt-dockers-$ARCH

pull_test_dockers_x64:
  extends: .pull_test_dockers
  variables:
    ARCH: x86_64

pull_test_dockers_arm64:
  extends: .pull_test_dockers
  variables:
    ARCH: arm64

# -- BTF
.upload_minimized_btfs:
  stage: kernel_matrix_testing_prepare
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
  tags: ["arch:amd64", "specific:true"]
  script:
    # Build dependencies directory
    - mkdir -p $DEPENDENCIES
    - pushd $DEPENDENCIES
    # download and copy btf files
    - mkdir -p $BTF_DIR
    - cp $CI_PROJECT_DIR/minimized-btfs.tar.xz $BTF_DIR/minimized-btfs.tar.xz
    - popd
    # package all the dependencies
    - ls -la $DEPENDENCIES
    - pushd $CI_PROJECT_DIR/kmt-deps/ci/$ARCH
    - tar czvf $ARCHIVE_NAME btfs
    - popd
    # upload tests
    # Switch to new profile after the btfs have been downloaded. Switching before
    # causes permission issues.
    - !reference [.kmt_new_profile]
    - !reference [.write_ssh_key_file]
    - !reference [.wait_for_instance]
    - !reference [.get_instance_ip_by_type]
    - !reference [.setup_ssh_config]
    - scp $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/$ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/
  after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
    - !reference [.tag_kmt_ci_job]
  variables:
    DEPENDENCIES: $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/btfs
    BTF_DIR: opt/kmt-ramfs/system-probe-tests/pkg/ebpf/bytecode/build/${ARCH}/co-re/btf
    AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key

upload_minimized_btfs_sysprobe_x64:
  extends:
    - .upload_minimized_btfs
  needs:
    - !reference [.kmt_setup_needs]
    - generate_minimized_btfs_x64
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    ARCHIVE_NAME: btfs-x86_64.tar.gz
    ARCH: x86_64
    INSTANCE_TYPE: r5d.metal
    TEST_COMPONENT: system-probe

upload_minimized_btfs_sysprobe_arm64:
  extends:
    - .upload_minimized_btfs
  needs:
    - !reference [.kmt_setup_needs]
    - generate_minimized_btfs_arm64
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    ARCHIVE_NAME: btfs-arm64.tar.gz
    ARCH: arm64
    INSTANCE_TYPE: r6gd.metal
    TEST_COMPONENT: system-probe

kmt_setup_env_sysprobe_arm64:
  extends:
    - .kmt_setup_env
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    INSTANCE_TYPE: "r6gd.metal"
    INSTANCE_TYPE_ARG: "--instance-type-arm=$INSTANCE_TYPE"
    ARCH: arm64
    AMI_ID_ARG: "--arm-ami-id=$KERNEL_MATRIX_TESTING_ARM_AMI_ID"
    LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-arm
    TEST_COMPONENT: system-probe

kmt_setup_env_sysprobe_x64:
  extends:
    - .kmt_setup_env
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    INSTANCE_TYPE: "r5d.metal"
    INSTANCE_TYPE_ARG: "--instance-type-x86=$INSTANCE_TYPE"
    ARCH: x86_64
    AMI_ID_ARG: "--x86-ami-id=$KERNEL_MATRIX_TESTING_X86_AMI_ID"
    LibvirtSSHKey: $CI_PROJECT_DIR/libvirt_rsa-x86
    TEST_COMPONENT: system-probe

.upload_sysprobe_tests:
  stage: kernel_matrix_testing_prepare
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  before_script:
    - !reference [.retrieve_linux_go_deps]
    - !reference [.kmt_new_profile]
    - !reference [.write_ssh_key_file]
  script:
    # copy system probe tests
    - pushd $CI_PROJECT_DIR/kmt-deps/ci/$ARCH
    - tar czvf $TEST_ARCHIVE_NAME opt
    - popd
    # upload tests
    - !reference [.wait_for_instance]
    - !reference [.get_instance_ip_by_type]
    - !reference [.setup_ssh_config]
    - scp $CI_PROJECT_DIR/kmt-deps/ci/$ARCH/$TEST_ARCHIVE_NAME metal_instance:/opt/kernel-version-testing/
    # build connector
    - pushd $CI_PROJECT_DIR/test/new-e2e
    - go build -o $CI_PROJECT_DIR/connector-${ARCH} $CI_PROJECT_DIR/test/new-e2e/system-probe/connector/main.go
    - popd
    # upload connector to metal instance
    - scp $CI_PROJECT_DIR/connector-${ARCH} metal_instance:/home/ubuntu/connector
  after_script:
    - DD_API_KEY=$($CI_PROJECT_DIR/tools/ci/fetch_secret.sh $AGENT_API_KEY_ORG2 token) || exit $?; export DD_API_KEY
    - !reference [.tag_kmt_ci_job]
  variables:
    AWS_EC2_SSH_KEY_FILE: $CI_PROJECT_DIR/ssh_key
    TEST_COMPONENT: system-probe
    TEST_ARCHIVE_NAME: tests-$ARCH.tar.gz
  artifacts:
    when: always
    paths:
      - $CI_PROJECT_DIR/connector-${ARCH}

upload_sysprobe_tests_x64:
  extends:
    - .upload_sysprobe_tests
  needs:
    - !reference [.kmt_setup_needs]
    - prepare_sysprobe_ebpf_functional_tests_x64
    - tests_ebpf_x64
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
  tags: ["arch:amd64", "specific:true"]
  variables:
    ARCH: x86_64
    INSTANCE_TYPE: r5d.metal

upload_sysprobe_tests_arm64:
  extends:
    - .upload_sysprobe_tests
  needs:
    - !reference [.kmt_setup_needs]
    - prepare_sysprobe_ebpf_functional_tests_arm64
    - tests_ebpf_arm64
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
  tags: ["arch:arm64", "specific:true"]
  variables:
    ARCH: arm64
    INSTANCE_TYPE: r6gd.metal

.kmt_run_sysprobe_tests:
  extends: .kmt_run_tests
  stage: kernel_matrix_testing_system_probe
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  timeout: 1h 30m
  variables:
    TEST_COMPONENT: system-probe

kmt_run_sysprobe_tests_x64:
  extends:
    - .kmt_run_sysprobe_tests
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
  tags: ["arch:amd64", "specific:true"]
  needs:
    - kmt_setup_env_sysprobe_x64
    - upload_dependencies_sysprobe_x64
    - upload_sysprobe_tests_x64
    - upload_minimized_btfs_sysprobe_x64
  variables:
    ARCH: "x86_64"
  parallel:
    matrix:
      - TAG:
          - "ubuntu_16.04"
          - "ubuntu_18.04"
          - "ubuntu_20.04"
          - "ubuntu_22.04"
          - "ubuntu_24.04"
          - "ubuntu_25.10"
          - "amazon_4.14"
          - "amazon_5.4"
          - "amazon_5.10"
          - "amazon_2023"
          - "fedora_37"
          - "fedora_38"
          - "debian_9"
          - "debian_10"
          - "debian_11"
          - "debian_12"
          - "centos_7.9"
          - "centos_8"
          - "rocky_9.4"
          - "rocky_8.4"
        TEST_SET: ["only_usm", "no_usm"]
  after_script:
    - !reference [.collect_outcomes_kmt]
    - !reference [.upload_junit_kmt]
    - !reference [.suggest_kmt_retry_command]

kmt_run_sysprobe_tests_arm64:
  extends:
    - .kmt_run_sysprobe_tests
  image: registry.ddbuild.io/ci/datadog-agent-buildimages/linux$CI_IMAGE_LINUX_SUFFIX:$CI_IMAGE_LINUX
  tags: ["arch:arm64", "specific:true"]
  needs:
    - kmt_setup_env_sysprobe_arm64
    - upload_dependencies_sysprobe_arm64
    - upload_sysprobe_tests_arm64
    - upload_minimized_btfs_sysprobe_arm64
  variables:
    ARCH: "arm64"
  parallel:
    matrix:
      - TAG:
          - "ubuntu_18.04"
          - "ubuntu_20.04"
          - "ubuntu_22.04"
          - "ubuntu_24.04"
          - "ubuntu_25.10"
          - "amazon_4.14"
          - "amazon_5.4"
          - "amazon_5.10"
          - "amazon_2023"
          - "fedora_37"
          - "fedora_38"
          - "debian_10"
          - "debian_11"
          - "debian_12"
          - "centos_7.9"
          - "centos_8"
          - "rocky_9.4"
        TEST_SET: ["only_usm", "no_usm"]
  after_script:
    - !reference [.collect_outcomes_kmt]
    - !reference [.upload_junit_kmt]
    - !reference [.suggest_kmt_retry_command]
.kmt_sysprobe_cleanup:
  extends:
    - .kmt_cleanup
  rules: !reference [.on_system_probe_or_e2e_changes_or_manual]
  variables:
    TEST_COMPONENT: system-probe

kmt_sysprobe_cleanup_arm64:
  when: always
  extends:
    - .kmt_sysprobe_cleanup
  needs:
    - kmt_setup_env_sysprobe_arm64
    - kmt_run_sysprobe_tests_arm64
    - upload_minimized_btfs_sysprobe_arm64
    - upload_sysprobe_tests_arm64
    - upload_dependencies_sysprobe_arm64
  variables:
    ARCH: arm64
    INSTANCE_TYPE: "r6gd.metal"

kmt_sysprobe_cleanup_x64:
  when: always
  extends:
    - .kmt_sysprobe_cleanup
  needs:
    - kmt_setup_env_sysprobe_x64
    - kmt_run_sysprobe_tests_x64
    - upload_minimized_btfs_sysprobe_x64
    - upload_sysprobe_tests_x64
    - upload_dependencies_sysprobe_x64
  variables:
    ARCH: x86_64
    INSTANCE_TYPE: "r5d.metal"

kmt_sysprobe_cleanup_arm64_manual:
  extends:
    - .kmt_sysprobe_cleanup
    - .kmt_cleanup_manual
  variables:
    ARCH: arm64
    INSTANCE_TYPE: "r6gd.metal"

kmt_sysprobe_cleanup_x64_manual:
  extends:
    - .kmt_sysprobe_cleanup
    - .kmt_cleanup_manual
  variables:
    ARCH: x86_64
    INSTANCE_TYPE: "r5d.metal"