resolve-build-deps.yaml

name: Resolve Dependencies and Build Wheels

on:
  workflow_dispatch:
  pull_request:
    branches:
    - master
    - 7.*.*

  push:
    branches:
    - master
    - 7.*.*

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }}

defaults:
  run:
    shell: bash

env:
  PYTHONUNBUFFERED: "1"
  PYTHON_VERSION: "3.13"
  DIRECT_DEPENDENCY_FILE: agent_requirements.in
  # https://reproducible-builds.org/specs/source-date-epoch/
  SOURCE_DATE_EPOCH: "1580601600"

jobs:
  # measure-disk-usage.yml depends on this workflow being triggered and completed,
  # so it can wait for the build to calculate dependency sizes.
  # The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected.

  check-should-run:
    name: Check if build should run
    runs-on: ubuntu-22.04
    permissions:
      actions: write
      contents: read
    outputs:
      builder_changed: ${{ steps.dependency-check.outputs.builder_changed }}
      should_run_build: ${{ steps.dependency-check.outputs.should_run_build }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Define diff commits
        id: set_sha
        if: github.event_name != 'workflow_dispatch'
        run: .github/workflows/scripts/resolve_deps_define_diff_commits.sh
        env:
          PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
          PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          EVENT_BEFORE: ${{ github.event.before }}

      - name: Get changed files
        id: changed-files
        if: github.event_name != 'workflow_dispatch'
        run: |
          REPO="${{ github.repository }}"

          CHANGED_FILES=$(gh api "repos/$REPO/compare/${{ steps.set_sha.outputs.prev_sha }}...${{ steps.set_sha.outputs.curr_sha }}" --paginate | \
          jq -r 'select(.files != null) | .files | map(.filename) | join(" ")')

          echo "files_changed=$CHANGED_FILES" >> $GITHUB_OUTPUT
          echo "files_changed=$CHANGED_FILES"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Check if build should run
        id: dependency-check
        run: .github/workflows/scripts/resolve_deps_check_should_run.sh
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          FILES_CHANGED: ${{ steps.changed-files.outputs.files_changed }}

  test:
    name: Run tests
    needs:
      - check-should-run
    if: needs.check-should-run.outputs.should_run_build == 'true'
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        run: |
          pip install -r .builders/deps/host_dependencies.txt
          pip install -r .builders/test_dependencies.txt
      - name: Run tests
        run: |
          cd .builders
          pytest -vvv
      - name: Run mypy type checking
        run: |
          cd .builders
          python -m mypy --config-file pyproject.toml .

  # Pin transitive dependency versions before parallel builds to avoid race
  # conditions with new package releases on PyPI during the build window.
  pre-resolve:
    name: Pre-resolve dependency versions
    needs:
      - check-should-run
      - test
    if: needs.check-should-run.outputs.should_run_build == 'true'
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Determine resolution cutoff timestamp
        id: timestamp
        run: |
          # Use the checked-out commit timestamp for reproducibility on re-runs
          echo "cutoff=$(git log -1 --format=%cI)" >> "${GITHUB_OUTPUT}"

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0

      - name: Generate dependency constraints
        run: |
          # We only resolve against PyPI here. The custom index
          # (agent-int-packages.datadoghq.com) hosts direct dependencies that
          # are already pinned in agent_requirements.in, so they don't need
          # constraining. We only need to pin transitive deps, which all come
          # from PyPI.
          uv pip compile ${{ env.DIRECT_DEPENDENCY_FILE }} \
            --exclude-newer "${{ steps.timestamp.outputs.cutoff }}" \
            --python-version ${{ env.PYTHON_VERSION }} \
            --output-file constraints.txt
          echo "Generated constraints:"
          cat constraints.txt

      - name: Upload constraints
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: resolution-constraints
          path: constraints.txt

  build:
    name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
    needs:
      - check-should-run
      - pre-resolve
    if: needs.check-should-run.outputs.should_run_build == 'true'
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
        - os: ubuntu-22.04-arm
          image: linux-aarch64
        - os: ubuntu-22.04
          image: linux-x86_64
        - os: windows-2022
          image: windows-x86_64

    permissions:
      packages: write

    env:
      OUT_DIR: output/${{ matrix.job.image }}
      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
      DOCKER: docker

    steps:
    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download dependency constraints
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        name: resolution-constraints
        path: .constraints

    - name: Ensure Docker is running
      run: bash .github/actions/setup-test-target-scripts/src/ensure-docker.sh

    - name: Log in to GitHub Packages
      uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Build image and wheels
      if: needs.check-should-run.outputs.builder_changed == 'true'
      run: python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --constraints .constraints/constraints.txt

    - name: Pull image and build wheels
      if: needs.check-should-run.outputs.builder_changed == 'false'
      run: |
        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest --constraints .constraints/constraints.txt

    - name: Publish image
      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
      run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}

    - name: Save new image digest
      if: github.event_name == 'push' && needs.check-should-run.outputs.builder_changed == 'true'
      run: >-
        ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
        | cut -d '@' -f 2
        > ${{ env.OUT_DIR }}/image_digest

    - name: Persist current image digest
      if: needs.check-should-run.outputs.builder_changed == 'false'
      run: >-
        jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
        > ${{ env.OUT_DIR }}/image_digest

    - name: Upload artifacts
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: target-${{ matrix.job.image }}
        path: output

  build-macos:
    name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }}
    needs:
      - check-should-run
      - pre-resolve
    if: needs.check-should-run.outputs.should_run_build == 'true'
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
        - arch: x86_64
          os: macos-14-large
        - arch: aarch64
          os: macos-14 # "macOS 14 Arm64" as per https://github.com/actions/runner-images/blob/main/README.md
    env:
      TARGET_NAME: macos-${{ matrix.job.arch }}
      OUT_DIR: output/macos-${{ matrix.job.arch }}
      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.13/bin/python"

    permissions:
      packages: write

    steps:
    - name: Set up environment
      run: |
        # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
        # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
        brew remove --force --ignore-dependencies $(brew list --formula)
        brew install coreutils

    - name: Set up Python
      # Use PBS as suggested by: https://github.com/DataDog/integrations-core/pull/21692#pullrequestreview-3358660684
      # PBS stands for "Python Build Standalone": https://astral.sh/blog/python-build-standalone
      env:
        PYTHON_PATCH: 12
        PBS_RELEASE: 20260203
        PBS_SHA256__aarch64: 146d011e9246790659d86c729a9bb37dc423545d0ed8e542ba1dfe93700aa0f2
        PBS_SHA256__x86_64: 5fb24d5a82f248e985bdc01f504f40d4150e321809b0bbeee7441cedd6dac227
      run: |
        set -u
        curl -fsSL -o pbs.tgz "https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-$PYTHON_VERSION.$PYTHON_PATCH+$PBS_RELEASE-${{ matrix.job.arch }}-apple-darwin-install_only_stripped.tar.gz"
        echo "$PBS_SHA256__${{ matrix.job.arch }} *pbs.tgz" | shasum -a 256 -c -
        prefix=$(dirname "$(dirname "$DD_PYTHON3")")
        sudo mkdir -p "$prefix"
        sudo tar -xzf pbs.tgz -C "$prefix" --strip-components=1
        rm pbs.tgz

    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Download dependency constraints
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        name: resolution-constraints
        path: .constraints

    - name: Install management dependencies
      run: |
        ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
        ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt"

    - name: Cache builder root
      id: cache-builder-root
      uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
      with:
        path: ~/builder_root
        key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }}

    - name: Run the build
      env:
        # This sets the minimum macOS version compatible for all built artifacts
        MACOSX_DEPLOYMENT_TARGET: "12.0" # https://docs.datadoghq.com/agent/supported_platforms/?tab=macos
        CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
      run: |
        # If we hit the cache, we can skip the builder setup
        if [[ ${CACHE_HIT} == "true" ]]; then
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup --constraints .constraints/constraints.txt
        else
          mkdir ~/builder_root
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --constraints .constraints/constraints.txt
        fi

    - name: Upload artifacts
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: target-macos-${{ matrix.job.arch }}
        path: output


  publish:
    name: Publish artifacts and update lockfiles via PR
    if: needs.check-should-run.outputs.should_run_build == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.'))))
    needs:
    - build
    - build-macos
    - check-should-run
    runs-on: ubuntu-latest

    permissions:
      contents: read
      id-token: write
      pull-requests: read

    steps:
    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        ref: "${{ github.head_ref }}"

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download artifacts
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        path: targets
        pattern: target-*
        merge-multiple: true

    - name: Get credentials
      id: auth
      uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
      with:
        project_id: datadog-agent-int-build
        workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core

    - name: Upload wheels and create lock files
      run: python .builders/upload.py targets

    - name: Clean up repository
      run: |
        rm ${{ steps.auth.outputs.credentials_file_path }}
        rm -rf targets

    - name: Create token
      uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1
      id: token-generator
      with:
        app-id: ${{ vars.DD_AGENT_INTEGRATIONS_BOT_APP_ID }}
        private-key: ${{ secrets.DD_AGENT_INTEGRATIONS_BOT_PRIVATE_KEY }}
        repositories: integrations-core

    - name: Find triggering PR
      id: find-pr
      uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
      with:
        github-token: ${{ steps.token-generator.outputs.token }}
        script: |
          const { data: commit } = await github.rest.repos.getCommit({
            owner: context.repo.owner,
            repo: context.repo.repo,
            ref: context.sha,
          });
          const message = commit.commit.message;
          const match = message.match(/#(\d+)/);
          const prs = match ? [{ number: parseInt(match[1]), html_url: `https://github.com/${context.repo.owner}/${context.repo.repo}/pull/${match[1]}` }] : [];
          if (prs.length > 0) {
            const pr = prs[0];
            core.setOutput('pr_ref', `[#${pr.number}](${pr.html_url})`);
          } else {
            core.setOutput('pr_ref', '');
          }

    - name: Create pull request
      uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
      with:
        token: ${{ steps.token-generator.outputs.token }}
        title: Update dependency resolution
        commit-message: Update dependency resolution
        branch: bot/update-dependency-resolution
        branch-suffix: timestamp
        delete-branch: true
        labels: bot,qa/skip-qa,bot/resolve-build-deps
        body: |-
          ### Motivation

          Direct dependencies were updated in ${{ github.sha }}${{ steps.find-pr.outputs.pr_ref != '' && format(' (triggered by PR {0})', steps.find-pr.outputs.pr_ref) || '' }}.

          ### Additional Notes

          This PR was automatically generated by the following workflow:
          ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}