Merge branch 'main' into rayz/trigger-otlp-ingest-metrics-experiment

Author: rayz

.github/workflows/test-devcontainer-update.yml

@@ -1,18 +1,74 @@
 name: test devcontainer
 
 on:
-  workflow_call:
-  workflow_dispatch:
+  push:
+    branches:
+    - main
   pull_request:
     branches:
     - main
-    paths:
-    - .github/workflows/test-devcontainer.yml
+  workflow_dispatch:
 
-permissions: {}
+permissions:
+  contents: read
 
 jobs:
+  detect-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      changes: "${{ steps.images.outcome == 'success' || steps.files.outcome == 'success' }}"
+
+    steps:
+    - name: Fetch changed files (PR)
+      if: github.event_name == 'pull_request'
+      env:
+        GH_TOKEN: "${{ github.token }}"
+      run: |
+        gh api repos/${{ github.repository }}/pulls/${{ github.event.pull_request.number }}/files --paginate \
+        | jq -s 'add' > pr-files.json
+
+    - name: Fetch changed files (push)
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      if: github.event_name == 'push'
+      with:
+        persist-credentials: false
+        fetch-depth: 2
+
+    - name: Build changed files list
+      run: |
+        if [ "${{ github.event_name }}" = "pull_request" ]; then
+          jq -r '.[].filename' pr-files.json | sort -u > changed-files.txt
+          jq -r '.[] | select(.filename==".gitlab-ci.yml") | .patch // empty' pr-files.json > gitlab-ci.patch
+        else
+          git diff --name-only HEAD~1 HEAD | sort -u > changed-files.txt
+          git diff HEAD~1 HEAD -- .gitlab-ci.yml > gitlab-ci.patch || true
+        fi
+
+    - name: Detect CI image changes
+      id: images
+      run: |
+        if grep -E '^[-+] +CI_IMAGE_[^:]+: v' gitlab-ci.patch >/dev/null; then
+          echo "changed=true"  >> "$GITHUB_OUTPUT"
+        else
+          echo "changed=false" >> "$GITHUB_OUTPUT"
+        fi
+
+    - name: Detect files of interest changes
+      id: files
+      run: |
+        cat > files-of-interest.txt <<'EOF'
+        .github/workflows/test-devcontainer.yml
+        EOF
+
+        if grep -Fx -f files-of-interest.txt changed-files.txt >/dev/null; then
+          echo "changed=true"  >> "$GITHUB_OUTPUT"
+        else
+          echo "changed=false" >> "$GITHUB_OUTPUT"
+        fi
+
   test:
+    needs: detect-changes
+    if: ${{ needs.detect-changes.outputs.changes == 'true' }}
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -23,7 +79,8 @@ jobs:
     - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       with:
         persist-credentials: false
-        fetch-depth: 0
+        fetch-depth: 1000
+        fetch-tags: true
 
     - name: Setup Node.js
       uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5.0.0
@@ -33,6 +90,23 @@ jobs:
       with:
         features: legacy-tasks
 
+    - name: Free disk space
+      run: |
+        df -h
+        echo "::group::Android"
+        sudo rm -rf /usr/local/lib/android
+        echo "::endgroup::"
+        echo "::group::.NET"
+        sudo rm -rf /usr/share/dotnet
+        echo "::endgroup::"
+        echo "::group::Haskell"
+        sudo rm -rf /opt/ghc
+        echo "::endgroup::"
+        echo "::group::CodeQL"
+        sudo rm -rf /opt/hostedtoolcache/CodeQL
+        echo "::endgroup::"
+        df -h
+
     - name: Build image
       run: |
         git clone https://github.com/DataDog/datadog-agent-buildimages.git
@@ -52,5 +126,16 @@ jobs:
     - name: Start Dev Container
       run: devcontainer up --workspace-folder .
 
+    - name: Grant the datadog user access to host Docker socket
+      run: |
+        docker exec -u root datadog_agent_devcontainer sh -lc '
+          set -e
+          sock=/var/run/docker.sock
+          gid=$(stat -c %g "$sock")
+          getent group "$gid" >/dev/null || groupadd -g "$gid" dockersock
+          gname=$(getent group "$gid" | cut -d: -f1)
+          id -nG datadog | tr " " "\n" | grep -qx "$gname" || usermod -aG "$gname" datadog
+        '
+
     - name: Test Agent build
       run: devcontainer exec --workspace-folder . dda inv agent.hacky-dev-image-build --target-image=agent

.github/workflows/test-devcontainer.yml

@@ -164,21 +164,21 @@ variables:
   # Build images versions
   # To use images from datadog-agent-buildimages dev branches, set the corresponding
   # SUFFIX variable to
-  CI_IMAGE_BTF_GEN: v79330459-0f2eff53
+  CI_IMAGE_BTF_GEN: v79781370-ed414087
   CI_IMAGE_BTF_GEN_SUFFIX: ""
-  CI_IMAGE_DOCKER_X64: v79330459-0f2eff53
+  CI_IMAGE_DOCKER_X64: v79781370-ed414087
   CI_IMAGE_DOCKER_X64_SUFFIX: ""
-  CI_IMAGE_DOCKER_ARM64: v79330459-0f2eff53
+  CI_IMAGE_DOCKER_ARM64: v79781370-ed414087
   CI_IMAGE_DOCKER_ARM64_SUFFIX: ""
-  CI_IMAGE_GITLAB_AGENT_DEPLOY: v79330459-0f2eff53
+  CI_IMAGE_GITLAB_AGENT_DEPLOY: v79781370-ed414087
   CI_IMAGE_GITLAB_AGENT_DEPLOY_SUFFIX: ""
-  CI_IMAGE_LINUX: v79330459-0f2eff53
+  CI_IMAGE_LINUX: v79781370-ed414087
   CI_IMAGE_LINUX_SUFFIX: ""
-  CI_IMAGE_RPM_X64: v79330459-0f2eff53
+  CI_IMAGE_RPM_X64: v79781370-ed414087
   CI_IMAGE_RPM_X64_SUFFIX: ""
-  CI_IMAGE_RPM_ARM64: v79330459-0f2eff53
+  CI_IMAGE_RPM_ARM64: v79781370-ed414087
   CI_IMAGE_RPM_ARM64_SUFFIX: ""
-  CI_IMAGE_RPM_ARMHF: v79330459-0f2eff53
+  CI_IMAGE_RPM_ARMHF: v79781370-ed414087
   CI_IMAGE_RPM_ARMHF_SUFFIX: ""
   CI_IMAGE_WIN_LTSC2022_X64: v79330459-0f2eff53
   CI_IMAGE_WIN_LTSC2022_X64_SUFFIX: ""
@@ -1115,7 +1115,7 @@ workflow:
   - !reference [.except_mergequeue]
   - changes:
       paths:
-        - pkg/fleet/installer/packages/embedded/templates/**/*.service
+        - pkg/fleet/installer/packages/embedded/tmpl/**/*.service
       compare_to: $COMPARE_TO_BRANCH
   - when: manual
     allow_failure: true

.gitlab-ci.yml

@@ -46,6 +46,6 @@ bazel:build-deps:windows-amd64:
   variables:
     ARCH: x64
     SCRIPT: |-
-      echo "🟡 TODO(regis): compilation errors remain - limiting to a working subset for the time being"
-      bazel build '@zlib//...' '@libffi//:ffi'
+      [Console]::WriteLine("🟡 TODO(regis): compilation errors remain - limiting to a working subset for the time being")
+      bazel build '@zlib//...' '@libffi//:ffi' '@xz//...'
       # bazel build @bzip2//...

.gitlab/bazel/build-deps.yaml

@@ -26,8 +26,6 @@
 .bazel:ext:windows:
   extends: .windows_docker_default
   timeout: 60m # pulling images alone can take more than 30m
-  allow_failure: true
-  retry: 0
   variables:
     GIT_DEPTH: 2
   script:
@@ -53,4 +51,4 @@
       --volume="${RUNNER_TEMP_PROJECT_DIR}:$RUNNER_TEMP_PROJECT_DIR"
       --workdir="$CI_PROJECT_DIR"
       "$WINBUILDIMAGE"
-      powershell -NonInteractive -NoLogo -NoProfile -OutputFormat Text -EncodedCommand $EncodedCommand
+      powershell -NonInteractive -NoLogo -NoProfile -InputFormat Text -OutputFormat Text -EncodedCommand $EncodedCommand

.gitlab/bazel/defs.yaml

@@ -3,14 +3,14 @@ benchmark:
   # This base image is created here: https://gitlab.ddbuild.io/DataDog/apm-reliability/benchmarking-platform
   image: 486234852809.dkr.ecr.us-east-1.amazonaws.com/ci/benchmarking-platform:trace-agent_microbenchmarks
   timeout: 1h
-  rules:
-    !reference [.on_trace_agent_changes_or_manual]
+  rules: !reference [.on_trace_agent_changes_or_manual]
   interruptible: true
   needs: ["setup_agent_version"]
   tags: ["team:apm-k8s-tweaked-metal-datadog-agent", "specific:true"]
   script:
     - export ARTIFACTS_DIR="$(pwd)/artifacts" && mkdir -p $ARTIFACTS_DIR
     - export BUILDIMAGES_COMMIT="${CI_IMAGE_LINUX#*-}"
+    - export BASE_BRANCH="$(cat release.json | jq ".base_branch" | tr -d '"')"
     - export DDA_VERSION="$(curl -s https://raw.githubusercontent.com/DataDog/datadog-agent-buildimages/${BUILDIMAGES_COMMIT}/dda.env | awk -F= '/^DDA_VERSION=/ {print $2}')"
     - pip install "git+https://github.com/DataDog/datadog-agent-dev.git@${DDA_VERSION}"
     - dda self dep sync -f legacy-tasks

.gitlab/benchmarks/benchmarks.yml

@@ -0,0 +1,41 @@
+---
+.static_quality_gate_docker: |
+  # Measure Docker image size and generate in-place report
+  # This runs after the main script and won't fail the job if there are issues.
+  # This is common for all Docker image builds
+
+  if [[ -n "$STATIC_QUALITY_GATE_NAME" ]]; then
+    echo "📊 Starting Docker image measurement..."
+
+    # Construct the built image reference from build variables
+    # This follows the same pattern used in docker build jobs
+    ECR_RELEASE_SUFFIX=${ECR_RELEASE_SUFFIX:-}
+    TAG_SUFFIX=${TAG_SUFFIX:-}
+    IMAGE_REF="${IMAGE}${ECR_RELEASE_SUFFIX}:v${CI_PIPELINE_ID}-${CI_COMMIT_SHORT_SHA}${TAG_SUFFIX}-$ARCH"
+    
+    echo "🔍 Measuring Docker image: $IMAGE_REF"
+
+    # Extract report prefix from gate name (e.g. static_quality_gate_docker_agent_amd64 -> docker_agent_amd64)
+    REPORT_PREFIX="${STATIC_QUALITY_GATE_NAME#static_quality_gate_}"
+
+    # Generate measurement report using STATIC_QUALITY_GATE_NAME variable
+    dda inv quality-gates.measure-image-local \
+      --image-ref "$IMAGE_REF" \
+      --gate-name "$STATIC_QUALITY_GATE_NAME" \
+      --build-job-name "$CI_JOB_NAME" \
+      --output-path "${REPORT_PREFIX}_size_report_${CI_PIPELINE_ID}_${CI_COMMIT_SHA:0:8}.yml" \
+      --debug || { echo "⚠️  Docker image measurement failed for $IMAGE_REF"; exit 0; }
+
+    echo "✅ Docker image measurement completed"
+
+    # Upload the report to S3 (same bucket structure as packages)
+    BUCKET_BASE_PATH="s3://dd-ci-artefacts-build-stable/datadog-agent/static_quality_gates/GATE_REPORTS/${CI_COMMIT_SHA}"
+    echo "Uploading report to ${BUCKET_BASE_PATH}"
+    aws s3 cp --only-show-errors --region us-east-1 --sse AES256 \
+      "${REPORT_PREFIX}_size_report_${CI_PIPELINE_ID}_${CI_COMMIT_SHA:0:8}.yml" \
+      "${BUCKET_BASE_PATH}/${REPORT_PREFIX}_size_report_${CI_PIPELINE_ID}_${CI_COMMIT_SHA:0:8}.yml"
+  else
+    echo "ℹ️  Skipping Docker image measurement (no STATIC_QUALITY_GATE_NAME defined)"
+  fi
+
+.static_quality_gate_report_path: "**/*_size_report_*.yml"

.gitlab/container_build/common.yml

@@ -88,6 +88,7 @@
       if [[ "$FLATTEN_IMAGE" == "true" ]]; then
         crane flatten -t ${TARGET_TAG} ${TARGET_TAG}
       fi
+    - !reference [.static_quality_gate_docker]
   # Workaround for temporary network failures
   retry: 2
   timeout: 30m
@@ -140,6 +141,7 @@ docker_build_agent7:
   variables:
     TAG_SUFFIX: -7
     BUILD_ARG: --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-amd64.tar.xz
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_agent_amd64"
 
 # TODO: Move these single-machine-performance jobs to
 # .gitlab/deploy_containers/deploy_containers_a7.yml.
@@ -172,6 +174,7 @@ docker_build_agent7_arm64:
   variables:
     TAG_SUFFIX: -7
     BUILD_ARG: --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_agent_arm64"
 
 # build agent7 fips image
 docker_build_fips_agent7:
@@ -198,6 +201,7 @@ docker_build_agent7_jmx:
   variables:
     TAG_SUFFIX: -7-jmx
     BUILD_ARG: --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-amd64.tar.xz
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_agent_jmx_amd64"
 
 docker_build_agent7_jmx_arm64:
   extends: [.docker_build_arm64, .docker_build_agent7]
@@ -206,6 +210,7 @@ docker_build_agent7_jmx_arm64:
   variables:
     TAG_SUFFIX: -7-jmx
     BUILD_ARG: --build-arg WITH_JMX=true --build-arg DD_AGENT_ARTIFACT=datadog-agent-7*-arm64.tar.xz
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_agent_jmx_arm64"
 
 docker_build_fips_agent7_jmx:
   extends: [.docker_build_amd64, .docker_build_agent7]
@@ -316,6 +321,8 @@ docker_build_cluster_agent_amd64:
       artifacts: true
     - job: cws_instrumentation-build_arm64
       artifacts: true
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_cluster_agent_amd64"
 
 docker_build_cluster_agent_arm64:
   extends: [.docker_build_arm64, .docker_build_cluster_agent]
@@ -326,6 +333,8 @@ docker_build_cluster_agent_arm64:
       artifacts: true
     - job: cws_instrumentation-build_arm64
       artifacts: true
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_cluster_agent_arm64"
 
 .docker_build_cluster_agent_fips:
   extends: .docker_build_cluster_agent
@@ -372,12 +381,16 @@ docker_build_cws_instrumentation_amd64:
   needs:
     - job: cws_instrumentation-build_amd64
       artifacts: true
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_cws_instrumentation_amd64"
 
 docker_build_cws_instrumentation_arm64:
   extends: [.docker_build_cws_instrumentation, .docker_build_arm64]
   needs:
     - job: cws_instrumentation-build_arm64
       artifacts: true
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_cws_instrumentation_arm64"
 
 # build the dogstatsd image
 .docker_build_dogstatsd:
@@ -396,10 +409,14 @@ docker_build_dogstatsd_amd64:
     - job: build_dogstatsd_static-binary_x64
       artifacts: false
   timeout: 20m
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_dogstatsd_amd64"
 
 docker_build_dogstatsd_arm64:
   extends: [.docker_build_dogstatsd, .docker_build_arm64, .docker_build_s3]
   needs:
     - job: build_dogstatsd_static-binary_arm64
       artifacts: false
   timeout: 20m
+  variables:
+    STATIC_QUALITY_GATE_NAME: "static_quality_gate_docker_dogstatsd_arm64"

.gitlab/container_build/docker_linux.yml

@@ -3,6 +3,7 @@
 # Contains jobs to build container images of the Agent.
 
 include:
+  - .gitlab/container_build/common.yml
   - .gitlab/container_build/docker_linux.yml
   - .gitlab/container_build/docker_windows.yml
   - .gitlab/container_build/fakeintake.yml