resolve-build-deps.yaml

name: Resolve Dependencies and Build Wheels

on:
  push:
    branches-ignore:
    - master
    - 7.*.*
    - gh-readonly-queue/**

# cancel-in-progress rules out the gate-vs-publish tree-divergence race: a new
# push to the same ref cancels the in-flight run, so the gate's hash and the
# tree the publish job commits the pin into are guaranteed to come from the
# same SHA. That is why upload.py can trust the resolution_hash passed in from
# the gate output rather than recomputing against the working tree.
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

env:
  PYTHONUNBUFFERED: "1"
  PYTHON_VERSION: "3.13"
  DIRECT_DEPENDENCY_FILE: agent_requirements.in
  # https://reproducible-builds.org/specs/source-date-epoch/
  SOURCE_DATE_EPOCH: "1580601600"

jobs:
  gate:
    name: Check content hashes
    runs-on: ubuntu-22.04
    permissions:
      contents: read
      pull-requests: read
      statuses: write
    outputs:
      needs_resolution: ${{ steps.hash-check.outputs.needs_resolution }}
      matrix_container: ${{ steps.hash-check.outputs.matrix_container }}
      matrix_macos: ${{ steps.hash-check.outputs.matrix_macos }}
      resolution_hash: ${{ steps.hash-check.outputs.resolution_hash }}
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Compute hashes
        id: hash-check
        run: |
          python .builders/inputs_hash.py status .builders/targets.json \
            >> "$GITHUB_OUTPUT"

      - name: Set dependency-wheel-promotion status
        uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8.0.0
        with:
          script: |
            const needsResolution = '${{ steps.hash-check.outputs.needs_resolution }}' === 'true';
            const prs = await github.paginate(github.rest.repos.listPullRequestsAssociatedWithCommit, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              commit_sha: context.sha,
            });

            const openPullRequests = prs.filter(pr => pr.state === 'open');
            let hasResolutionOutput = false;
            for (const pr of openPullRequests) {
              const files = await github.paginate(github.rest.pulls.listFiles, {
                owner: context.repo.owner,
                repo: context.repo.repo,
                pull_number: pr.number,
                per_page: 100,
              });
              if (files.some(file => file.filename.startsWith('.deps/'))) {
                hasResolutionOutput = true;
                break;
              }
            }

            const promotionRequired = needsResolution || hasResolutionOutput;
            const promoteTarget = openPullRequests[0]?.html_url ?? '<PR_URL>';
            await github.rest.repos.createCommitStatus({
              owner: context.repo.owner,
              repo: context.repo.repo,
              sha: context.sha,
              state: promotionRequired ? 'pending' : 'success',
              context: 'dependency-wheel-promotion',
              description: promotionRequired
                ? `Wheels must be promoted to stable before merge. Run: ddev dep promote ${promoteTarget}`
                : 'No dependency resolution needed; promotion not required.',
            });

  test:
    name: Run tests
    needs:
      - gate
    if: needs.gate.outputs.needs_resolution == 'true'
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        run: |
          pip install -r .builders/deps/host_dependencies.txt
          pip install -r .builders/test_dependencies.txt
      - name: Run tests
        run: |
          cd .builders
          pytest -vvv
      - name: Run mypy type checking
        run: |
          cd .builders
          python -m mypy --config-file pyproject.toml .

  # Pin transitive dependency versions before parallel builds to avoid race
  # conditions with new package releases on PyPI during the build window.
  pre-resolve:
    name: Pre-resolve dependency versions
    needs:
      - test
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

      - name: Determine resolution cutoff timestamp
        id: timestamp
        run: |
          # Use the checked-out commit timestamp for reproducibility on re-runs
          echo "cutoff=$(git log -1 --format=%cI)" >> "${GITHUB_OUTPUT}"

      - name: Install uv
        uses: astral-sh/setup-uv@37802adc94f370d6bfd71619e3f0bf239e1f3b78 # v7.6.0

      - name: Generate dependency constraints
        run: |
          # We only resolve against PyPI here. The custom index
          # (agent-int-packages.datadoghq.com) hosts direct dependencies that
          # are already pinned in agent_requirements.in, so they don't need
          # constraining. We only need to pin transitive deps, which all come
          # from PyPI.
          #
          # Note: rebasing a branch onto newer master (with no resolution input
          # changes) will not re-resolve against a newer cutoff, because the
          # content hash gate sees no change. The cutoff is per-commit, not
          # per-branch-tip. This is intentional: the cutoff prevents PyPI race
          # conditions within a build window, and the hash gate preserves that
          # window.
          uv pip compile ${{ env.DIRECT_DEPENDENCY_FILE }} \
            --exclude-newer "${{ steps.timestamp.outputs.cutoff }}" \
            --python-version ${{ env.PYTHON_VERSION }} \
            --output-file constraints.txt
          echo "Generated constraints:"
          cat constraints.txt

      - name: Upload constraints
        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
        with:
          name: resolution-constraints
          path: constraints.txt

  build:
    name: Target ${{ matrix.job.image }} on ${{ matrix.job.runner_os }}
    needs:
      - gate
      - pre-resolve
    runs-on: ${{ matrix.job.runner_os }}
    strategy:
      fail-fast: false
      matrix:
        job: ${{ fromJSON(needs.gate.outputs.matrix_container) }}

    permissions:
      packages: write

    env:
      OUT_DIR: output/${{ matrix.job.image }}
      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
      DOCKER: docker

    steps:
    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Persist target inputs hash
      run: echo "${{ matrix.job.hash }}" > /tmp/inputs_sha256

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download dependency constraints
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        name: resolution-constraints
        path: .constraints

    - name: Ensure Docker is running
      run: bash .github/actions/setup-test-target-scripts/src/ensure-docker.sh

    - name: Log in to GitHub Packages
      uses: docker/login-action@b45d80f862d83dbcd57f89517bcf500b2ab88fb2 # v4.0.0
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Build image and wheels
      if: matrix.job.rebuild == 'true'
      run: python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --constraints .constraints/constraints.txt

    - name: Pull image and build wheels
      if: matrix.job.rebuild == 'false'
      run: |
        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest --constraints .constraints/constraints.txt

    - name: Publish image
      if: matrix.job.rebuild == 'true'
      run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}

    - name: Save new image digest and inputs hash
      if: matrix.job.rebuild == 'true'
      run: |
        ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }} \
          | cut -d '@' -f 2 \
          > ${{ env.OUT_DIR }}/image_digest
        cp /tmp/inputs_sha256 ${{ env.OUT_DIR }}/inputs_sha256

    - name: Persist current image digest and inputs hash
      if: matrix.job.rebuild == 'false'
      run: |
        jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json > ${{ env.OUT_DIR }}/image_digest
        cp /tmp/inputs_sha256 ${{ env.OUT_DIR }}/inputs_sha256

    - name: Upload artifacts
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: target-${{ matrix.job.image }}
        path: output

  build-macos:
    name: Target ${{ matrix.job.image }} on ${{ matrix.job.runner_os }}
    needs:
      - gate
      - pre-resolve
    runs-on: ${{ matrix.job.runner_os }}
    strategy:
      fail-fast: false
      matrix:
        job: ${{ fromJSON(needs.gate.outputs.matrix_macos) }}
    env:
      TARGET_NAME: ${{ matrix.job.image }}
      OUT_DIR: output/${{ matrix.job.image }}
      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.13/bin/python"

    permissions:
      packages: write

    steps:
    - name: Set up environment
      run: |
        # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
        # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
        brew remove --force --ignore-dependencies $(brew list --formula)
        brew install coreutils

    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2

    - name: Set up Python
      # PBS pins live in .builders/images/macos/pbs.env so they enter inputs_hash per-target
      # digests (images/macos/**/*) and invalidate ~/builder_root cache on bump.
      # https://github.com/DataDog/integrations-core/pull/21692#pullrequestreview-3358660684
      run: |
        set -euo pipefail
        set -a
        # shellcheck disable=SC1091
        source .builders/images/macos/pbs.env
        set +a
        curl -fsSL -o pbs.tgz "https://github.com/astral-sh/python-build-standalone/releases/download/$PBS_RELEASE/cpython-$PYTHON_VERSION.$PYTHON_PATCH+$PBS_RELEASE-${{ matrix.job.arch }}-apple-darwin-install_only_stripped.tar.gz"
        echo "$PBS_SHA256__${{ matrix.job.arch }} *pbs.tgz" | shasum -a 256 -c -
        prefix=$(dirname "$(dirname "$DD_PYTHON3")")
        sudo mkdir -p "$prefix"
        sudo tar -xzf pbs.tgz -C "$prefix" --strip-components=1
        rm pbs.tgz

    - name: Download dependency constraints
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        name: resolution-constraints
        path: .constraints

    - name: Install management dependencies
      run: |
        ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
        ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt"

    - name: Cache builder root
      id: cache-builder-root
      uses: actions/cache@cdf6c1fa76f9f475f3d7449005a359c84ca0f306 # v5.0.3
      with:
        path: ~/builder_root
        # matrix.job.hash content-hashes per-target .builders/ inputs (including
        # images/macos/**/*, so pbs.env participates). Using it as the cache key
        # keeps the gate and Actions cache aligned.
        key: ${{ matrix.job.image }}-deps-builder-root-cache-${{ matrix.job.hash }}

    - name: Run the build
      env:
        # This sets the minimum macOS version compatible for all built artifacts
        MACOSX_DEPLOYMENT_TARGET: "12.0" # https://docs.datadoghq.com/agent/supported_platforms/?tab=macos
        CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
      run: |
        # If we hit the cache, we can skip the builder setup
        if [[ ${CACHE_HIT} == "true" ]]; then
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup --constraints .constraints/constraints.txt
        else
          mkdir ~/builder_root
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --constraints .constraints/constraints.txt
        fi

    - name: Save inputs hash into output
      run: echo "${{ matrix.job.hash }}" > ${{ env.OUT_DIR }}/inputs_sha256

    - name: Upload artifacts
      uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
      with:
        name: target-${{ matrix.job.image }}
        path: output


  publish:
    name: Publish artifacts and commit lockfiles to branch
    needs:
    - gate
    - build
    - build-macos
    runs-on: ubuntu-latest

    permissions:
      contents: write
      id-token: write
      pull-requests: read

    steps:
    - name: Create token
      uses: DataDog/dd-octo-sts-action@96a25462dbcb10ebf0bfd6e2ccc917d2ab235b9a # v1.0.4
      id: token-generator
      with:
        scope: DataDog/integrations-core
        policy: self.resolve-build-deps.push

    - name: Checkout code
      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      with:
        ref: ${{ github.ref_name }}
        token: ${{ steps.token-generator.outputs.token }}

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download artifacts
      uses: actions/download-artifact@70fc10c6e5e1ce46ad2ea6f2b72d43f7d47b13c3 # v8.0.0
      with:
        path: targets
        pattern: target-*
        merge-multiple: true

    - name: Get credentials
      id: auth
      uses: google-github-actions/auth@7c6bc770dae815cd3e89ee6cdf493a5fab2cc093 # v3.0.0
      with:
        project_id: datadog-agent-int-build
        workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core

    - name: Upload wheels and create lock files
      run: python .builders/upload.py targets --resolution-hash "${{ needs.gate.outputs.resolution_hash }}"

    - name: Clean up repository
      run: |
        rm ${{ steps.auth.outputs.credentials_file_path }}
        rm -rf targets

    - name: Commit updated lockfiles to branch
      run: |
        git config user.name "dd-agent-integrations-bot[bot]"
        git config user.email "dd-agent-integrations-bot[bot]@users.noreply.github.com"
        git add .deps/
        if git diff --cached --quiet; then
          echo "No lockfile changes to commit."
        else
          git commit -m "Update dependency resolution"
          git push
        fi