resolve-build-deps.yaml

name: Resolve Dependencies and Build Wheels

on:
  workflow_dispatch:
  pull_request:
    branches:
    - master
    - 7.*.*

  push:
    branches:
    - master
    - 7.*.*

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }}

defaults:
  run:
    shell: bash

env:
  PYTHONUNBUFFERED: "1"
  PYTHON_VERSION: "3.13"
  DIRECT_DEPENDENCY_FILE: agent_requirements.in
  # https://reproducible-builds.org/specs/source-date-epoch/
  SOURCE_DATE_EPOCH: "1580601600"

jobs: 
  # measure-disk-usage.yml depends on this workflow being triggered and completed,
  # so it can wait for the build to calculate dependency sizes.
  # The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected.

  check-dependency-changes:
    name: Check dependency changes
    runs-on: ubuntu-22.04
    permissions:
      actions: write
    outputs:
      dependency_changed: ${{ steps.dependency-check.outputs.dependency_changed }}
      builder_changed: ${{ steps.dependency-check.outputs.builder_changed }}
    steps:
      - name: Define diff commits
        id: set_sha
        run: |
          if [ "${{ github.event_name }}" == "pull_request" ]; then
            PREV_SHA=${{ github.event.pull_request.base.sha }}
            CURR_SHA=${{ github.event.pull_request.head.sha }}
          else
            PREV_SHA=${{ github.event.before }}
            CURR_SHA=${{ github.sha }}
          fi

          echo "prev_sha=$PREV_SHA" >> $GITHUB_OUTPUT
          echo "curr_sha=$CURR_SHA" >> $GITHUB_OUTPUT

          echo "Current SHA: $CURR_SHA"
          echo "Previous SHA: $PREV_SHA"

      - name: Get changed files
        id: changed-files
        run: |
          REPO="${{ github.repository }}"

          CHANGED_FILES=$(gh api "repos/$REPO/compare/${{ steps.set_sha.outputs.prev_sha }}...${{ steps.set_sha.outputs.curr_sha }}" --paginate | \
          jq -r 'select(.files != null) | .files | map(.filename) | join(" ")')

          echo "files_changed=$CHANGED_FILES" >> $GITHUB_OUTPUT
          echo "files_changed=$CHANGED_FILES"
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Check for dependency changes
        id: dependency-check
        run: |
          FILES_CHANGED="${{ steps.changed-files.outputs.files_changed }}"

          cat << EOF > dependency_files.txt
          agent_requirements\.in
          \.github/workflows/resolve-build-deps\.yaml
          \.builders/
          EOF

          cat <<EOF > builder_files.txt
          \.builders/
          EOF

          DEPENDENCY_CHANGED=$(
              echo "$FILES_CHANGED" | \
              grep -qf dependency_files.txt \
              && echo "true" || echo "false"
          ) 

          BUILDER_CHANGED=$(
              echo "$FILES_CHANGED" | \
              grep -qf builder_files.txt \
              && echo "true" || echo "false"
          )


          echo "dependency_changed=$DEPENDENCY_CHANGED" | tee -a $GITHUB_OUTPUT
          echo "builder_changed=$BUILDER_CHANGED" | tee -a $GITHUB_OUTPUT

        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

  test:
    name: Run tests
    needs:
      - check-dependency-changes
    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
    runs-on: ubuntu-22.04
    steps:
      - name: Checkout code
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Set up Python ${{ env.PYTHON_VERSION }} 
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: ${{ env.PYTHON_VERSION }}
      - name: Install dependencies
        run: |
          pip install -r .builders/deps/host_dependencies.txt
          pip install -r .builders/test_dependencies.txt
      - name: Run tests
        run: |
          cd .builders
          pytest -vvv

  build:
    name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }}
    needs:
      - check-dependency-changes
    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
        - os: ubuntu-22.04-arm
          image: linux-aarch64
        - os: ubuntu-22.04
          image: linux-x86_64
        - os: windows-2022
          image: windows-x86_64

    permissions:
      packages: write

    env:
      OUT_DIR: output/${{ matrix.job.image }}
      BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }}
      DOCKER: docker

    steps:
    - name: Checkout code
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: |
        pip install -r .builders/deps/host_dependencies.txt

    - name: Log in to GitHub Packages
      uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}

    - name: Build image and wheels
      if: needs.check-dependency-changes.outputs.builder_changed == 'true'
      run: |-
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3

    - name: Pull image and build wheels
      if: needs.check-dependency-changes.outputs.builder_changed == 'false'
      run: |-
        digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json)
        python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest

    - name: Publish image
      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
      run: ${DOCKER} push ${{ env.BUILDER_IMAGE }}

    - name: Save new image digest
      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true'
      run: >-
        ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }}
        | cut -d '@' -f 2
        > ${{ env.OUT_DIR }}/image_digest

    - name: Persist current image digest
      if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'false'
      run: >-
        jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json
        > ${{ env.OUT_DIR }}/image_digest

    - name: Upload artifacts
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
      with:
        name: target-${{ matrix.job.image }}
        path: output

  build-macos:
    name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }}
    needs:
      - check-dependency-changes
    if: needs.check-dependency-changes.outputs.dependency_changed == 'true'
    runs-on: ${{ matrix.job.os }}
    strategy:
      fail-fast: false
      matrix:
        job:
        - arch: x86_64
          os: macos-13
          macosx_version_min: "10.12"
        - arch: aarch64
          os: macos-13-xlarge # "macOS 13 Arm64" as per https://github.com/actions/runner-images/blob/main/README.md
          macosx_version_min: "11.0"
    env:
      TARGET_NAME: macos-${{ matrix.job.arch }}
      OUT_DIR: output/macos-${{ matrix.job.arch }}
      DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.13/bin/python"

    permissions:
      packages: write

    steps:
    - name: Set up environment
      run: |-
        # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that
        # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate`
        brew remove --force --ignore-dependencies $(brew list --formula)
        brew install coreutils

    - name: Set up Python
      env:
        # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel
        PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.7/python-3.13.7-macos11.pkg"
      run: |-
        curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg
        sudo installer -pkg python3.pkg -target /

    - name: Checkout code
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2

    - name: Install management dependencies
      run: |
        ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt
        ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt"

    - name: Cache builder root
      id: cache-builder-root
      uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3
      with:
        path: |
          ~/builder_root
        key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }}

    - name: Run the build
      env:
        # This sets the minimum macOS version compatible for all built artifacts
        MACOSX_DEPLOYMENT_TARGET: ${{ matrix.job.macosx_version_min }}
        CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }}
      run: |-
        # If we hit the cache, we can skip the builder setup
        if [[ ${CACHE_HIT} == "true" ]]; then
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup
        else
          mkdir ~/builder_root
          ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3
        fi

    - name: Upload artifacts
      uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
      with:
        name: target-macos-${{ matrix.job.arch }}
        path: output


  publish:
    name: Publish artifacts and update lockfiles via PR
    if: needs.check-dependency-changes.outputs.dependency_changed == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) 
    needs:
    - build
    - build-macos
    - check-dependency-changes
    runs-on: ubuntu-latest

    permissions:
      contents: read
      id-token: write

    steps:
    - name: Checkout code
      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      with:
        ref: "${{ github.head_ref }}"

    - name: Set up Python ${{ env.PYTHON_VERSION }}
      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
      with:
        python-version: ${{ env.PYTHON_VERSION }}

    - name: Install management dependencies
      run: pip install -r .builders/deps/host_dependencies.txt

    - name: Download artifacts
      uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
      with:
        path: targets
        pattern: target-*
        merge-multiple: true

    - name: Get credentials
      id: auth
      uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7
      with:
        project_id: datadog-agent-int-build
        workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core

    - name: Upload wheels
      run: python .builders/upload.py targets

    - name: Lock files
      run: python .builders/lock.py targets

    - name: Clean up repository
      run: |-
        rm ${{ steps.auth.outputs.credentials_file_path }}
        rm -rf targets

    - name: Create token
      uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0
      id: token-generator
      with:
        app-id: ${{ vars.DD_AGENT_INTEGRATIONS_BOT_APP_ID }}
        private-key: ${{ secrets.DD_AGENT_INTEGRATIONS_BOT_PRIVATE_KEY }}
        repositories: integrations-core

    - name: Create pull request
      uses: peter-evans/create-pull-request@4e1beaa7521e8b457b572c090b25bd3db56bf1c5 # v5.0.3
      with:
        token: ${{ steps.token-generator.outputs.token }}
        title: Update dependency resolution
        commit-message: Update dependency resolution
        branch: bot/update-dependency-resolution
        branch-suffix: timestamp
        delete-branch: true
        labels: bot,qa/skip-qa,bot/resolve-build-deps
        body: |-
          ### Motivation

          Direct dependencies were updated in ${{ github.sha }}.

          ### Additional Notes

          This PR was automatically generated by the following workflow:
          ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}