[pull] master from DataDog:master #621
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Resolve Dependencies and Build Wheels | |
| on: | |
| workflow_dispatch: | |
| pull_request: | |
| branches: | |
| - master | |
| - 7.*.* | |
| push: | |
| branches: | |
| - master | |
| - 7.*.* | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: ${{ github.event_name == 'pull_request' && true || false }} | |
| defaults: | |
| run: | |
| shell: bash | |
| env: | |
| PYTHONUNBUFFERED: "1" | |
| PYTHON_VERSION: "3.13" | |
| DIRECT_DEPENDENCY_FILE: agent_requirements.in | |
| # https://reproducible-builds.org/specs/source-date-epoch/ | |
| SOURCE_DATE_EPOCH: "1580601600" | |
| jobs: | |
| # measure-disk-usage.yml depends on this workflow being triggered and completed, | |
| # so it can wait for the build to calculate dependency sizes. | |
| # The 'on' setting ensures it runs, but this job cancels it if no dependency changes are detected. | |
| check-dependency-changes: | |
| name: Check dependency changes | |
| runs-on: ubuntu-22.04 | |
| permissions: | |
| actions: write | |
| outputs: | |
| dependency_changed: ${{ steps.dependency-check.outputs.dependency_changed }} | |
| builder_changed: ${{ steps.dependency-check.outputs.builder_changed }} | |
| steps: | |
| - name: Define diff commits | |
| id: set_sha | |
| run: | | |
| if [ "${{ github.event_name }}" == "pull_request" ]; then | |
| PREV_SHA=${{ github.event.pull_request.base.sha }} | |
| CURR_SHA=${{ github.event.pull_request.head.sha }} | |
| else | |
| PREV_SHA=${{ github.event.before }} | |
| CURR_SHA=${{ github.sha }} | |
| fi | |
| echo "prev_sha=$PREV_SHA" >> $GITHUB_OUTPUT | |
| echo "curr_sha=$CURR_SHA" >> $GITHUB_OUTPUT | |
| echo "Current SHA: $CURR_SHA" | |
| echo "Previous SHA: $PREV_SHA" | |
| - name: Get changed files | |
| id: changed-files | |
| run: | | |
| REPO="${{ github.repository }}" | |
| CHANGED_FILES=$(gh api "repos/$REPO/compare/${{ steps.set_sha.outputs.prev_sha }}...${{ steps.set_sha.outputs.curr_sha }}" --paginate | \ | |
| jq -r 'select(.files != null) | .files | map(.filename) | join(" ")') | |
| echo "files_changed=$CHANGED_FILES" >> $GITHUB_OUTPUT | |
| echo "files_changed=$CHANGED_FILES" | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Check for dependency changes | |
| id: dependency-check | |
| run: | | |
| FILES_CHANGED="${{ steps.changed-files.outputs.files_changed }}" | |
| cat << EOF > dependency_files.txt | |
| agent_requirements\.in | |
| \.github/workflows/resolve-build-deps\.yaml | |
| \.builders/ | |
| EOF | |
| cat <<EOF > builder_files.txt | |
| \.builders/ | |
| EOF | |
| DEPENDENCY_CHANGED=$( | |
| echo "$FILES_CHANGED" | \ | |
| grep -qf dependency_files.txt \ | |
| && echo "true" || echo "false" | |
| ) | |
| BUILDER_CHANGED=$( | |
| echo "$FILES_CHANGED" | \ | |
| grep -qf builder_files.txt \ | |
| && echo "true" || echo "false" | |
| ) | |
| echo "dependency_changed=$DEPENDENCY_CHANGED" | tee -a $GITHUB_OUTPUT | |
| echo "builder_changed=$BUILDER_CHANGED" | tee -a $GITHUB_OUTPUT | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| test: | |
| name: Run tests | |
| needs: | |
| - check-dependency-changes | |
| if: needs.check-dependency-changes.outputs.dependency_changed == 'true' | |
| runs-on: ubuntu-22.04 | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install dependencies | |
| run: | | |
| pip install -r .builders/deps/host_dependencies.txt | |
| pip install -r .builders/test_dependencies.txt | |
| - name: Run tests | |
| run: | | |
| cd .builders | |
| pytest -vvv | |
| build: | |
| name: Target ${{ matrix.job.image }} on ${{ matrix.job.os }} | |
| needs: | |
| - check-dependency-changes | |
| if: needs.check-dependency-changes.outputs.dependency_changed == 'true' | |
| runs-on: ${{ matrix.job.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: | |
| - os: ubuntu-22.04-arm | |
| image: linux-aarch64 | |
| - os: ubuntu-22.04 | |
| image: linux-x86_64 | |
| - os: windows-2022 | |
| image: windows-x86_64 | |
| permissions: | |
| packages: write | |
| env: | |
| OUT_DIR: output/${{ matrix.job.image }} | |
| BUILDER_IMAGE: ghcr.io/datadog/agent-int-builder:${{ matrix.job.image }} | |
| DOCKER: docker | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install management dependencies | |
| run: | | |
| pip install -r .builders/deps/host_dependencies.txt | |
| - name: Log in to GitHub Packages | |
| uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Build image and wheels | |
| if: needs.check-dependency-changes.outputs.builder_changed == 'true' | |
| run: |- | |
| python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 | |
| - name: Pull image and build wheels | |
| if: needs.check-dependency-changes.outputs.builder_changed == 'false' | |
| run: |- | |
| digest=$(jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json) | |
| python .builders/build.py ${{ matrix.job.image }} --python 3 ${{ env.OUT_DIR }}/py3 --digest $digest | |
| - name: Publish image | |
| if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true' | |
| run: ${DOCKER} push ${{ env.BUILDER_IMAGE }} | |
| - name: Save new image digest | |
| if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'true' | |
| run: >- | |
| ${DOCKER} inspect --format "{{index .RepoDigests 0}}" ${{ env.BUILDER_IMAGE }} | |
| | cut -d '@' -f 2 | |
| > ${{ env.OUT_DIR }}/image_digest | |
| - name: Persist current image digest | |
| if: github.event_name == 'push' && needs.check-dependency-changes.outputs.builder_changed == 'false' | |
| run: >- | |
| jq -r '.["${{ matrix.job.image }}"]' .deps/image_digests.json | |
| > ${{ env.OUT_DIR }}/image_digest | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: target-${{ matrix.job.image }} | |
| path: output | |
| build-macos: | |
| name: Target macOS/${{ matrix.job.arch }} on ${{ matrix.job.os }} | |
| needs: | |
| - check-dependency-changes | |
| if: needs.check-dependency-changes.outputs.dependency_changed == 'true' | |
| runs-on: ${{ matrix.job.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| job: | |
| - arch: x86_64 | |
| os: macos-13 | |
| macosx_version_min: "10.12" | |
| - arch: aarch64 | |
| os: macos-13-xlarge # "macOS 13 Arm64" as per https://github.com/actions/runner-images/blob/main/README.md | |
| macosx_version_min: "11.0" | |
| env: | |
| TARGET_NAME: macos-${{ matrix.job.arch }} | |
| OUT_DIR: output/macos-${{ matrix.job.arch }} | |
| DD_PYTHON3: "/Library/Frameworks/Python.framework/Versions/3.13/bin/python" | |
| permissions: | |
| packages: write | |
| steps: | |
| - name: Set up environment | |
| run: |- | |
| # We remove everything that comes pre-installed via Homebrew to avoid depending on or shipping stuff that | |
| # comes in the runner through Homebrew to better control what might get shipped in the wheels via `delocate` | |
| brew remove --force --ignore-dependencies $(brew list --formula) | |
| brew install coreutils | |
| - name: Set up Python | |
| env: | |
| # Despite the name, this is built for the macOS 11 SDK on arm64 and 10.9+ on intel | |
| PYTHON3_DOWNLOAD_URL: "https://www.python.org/ftp/python/3.13.7/python-3.13.7-macos11.pkg" | |
| run: |- | |
| curl "$PYTHON3_DOWNLOAD_URL" -o python3.pkg | |
| sudo installer -pkg python3.pkg -target / | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| - name: Install management dependencies | |
| run: | | |
| ${DD_PYTHON3} -m pip install -r .builders/deps/host_dependencies.txt | |
| ${DD_PYTHON3} -m pip install --no-warn-script-location -r ".builders/images/runner_dependencies.txt" | |
| - name: Cache builder root | |
| id: cache-builder-root | |
| uses: actions/cache@5a3ec84eff668545956fd18022155c47e93e2684 # v4.2.3 | |
| with: | |
| path: | | |
| ~/builder_root | |
| key: macos-${{ matrix.job.arch }}-deps-builder-root-cache-${{ hashFiles('./.builders/images/macos/*', './.builders/images/*', './.builders/deps/*', './.builders/build.py', './.github/workflows/resolve-build-deps.yml') }} | |
| - name: Run the build | |
| env: | |
| # This sets the minimum macOS version compatible for all built artifacts | |
| MACOSX_DEPLOYMENT_TARGET: ${{ matrix.job.macosx_version_min }} | |
| CACHE_HIT: ${{ steps.cache-builder-root.outputs.cache-hit }} | |
| run: |- | |
| # If we hit the cache, we can skip the builder setup | |
| if [[ ${CACHE_HIT} == "true" ]]; then | |
| ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 --skip-setup | |
| else | |
| mkdir ~/builder_root | |
| ${DD_PYTHON3} .builders/build.py ${{ env.TARGET_NAME }} --builder-root ~/builder_root --python 3 ${{ env.OUT_DIR }}/py3 | |
| fi | |
| - name: Upload artifacts | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 | |
| with: | |
| name: target-macos-${{ matrix.job.arch }} | |
| path: output | |
| publish: | |
| name: Publish artifacts and update lockfiles via PR | |
| if: needs.check-dependency-changes.outputs.dependency_changed == 'true' && (github.event_name == 'push' || (github.event_name == 'workflow_dispatch' && (github.ref_name == github.event.repository.default_branch || startsWith(github.ref_name, '7.')))) | |
| needs: | |
| - build | |
| - build-macos | |
| - check-dependency-changes | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: read | |
| id-token: write | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| ref: "${{ github.head_ref }}" | |
| - name: Set up Python ${{ env.PYTHON_VERSION }} | |
| uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install management dependencies | |
| run: pip install -r .builders/deps/host_dependencies.txt | |
| - name: Download artifacts | |
| uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 | |
| with: | |
| path: targets | |
| pattern: target-* | |
| merge-multiple: true | |
| - name: Get credentials | |
| id: auth | |
| uses: google-github-actions/auth@6fc4af4b145ae7821d527454aa9bd537d1f2dc5f # v2.1.7 | |
| with: | |
| project_id: datadog-agent-int-build | |
| workload_identity_provider: projects/574011472402/locations/global/workloadIdentityPools/github/providers/integrations-core | |
| - name: Upload wheels | |
| run: python .builders/upload.py targets | |
| - name: Lock files | |
| run: python .builders/lock.py targets | |
| - name: Clean up repository | |
| run: |- | |
| rm ${{ steps.auth.outputs.credentials_file_path }} | |
| rm -rf targets | |
| - name: Create token | |
| uses: actions/create-github-app-token@d72941d797fd3113feb6b93fd0dec494b13a2547 # v1.12.0 | |
| id: token-generator | |
| with: | |
| app-id: ${{ vars.DD_AGENT_INTEGRATIONS_BOT_APP_ID }} | |
| private-key: ${{ secrets.DD_AGENT_INTEGRATIONS_BOT_PRIVATE_KEY }} | |
| repositories: integrations-core | |
| - name: Create pull request | |
| uses: peter-evans/create-pull-request@4e1beaa7521e8b457b572c090b25bd3db56bf1c5 # v5.0.3 | |
| with: | |
| token: ${{ steps.token-generator.outputs.token }} | |
| title: Update dependency resolution | |
| commit-message: Update dependency resolution | |
| branch: bot/update-dependency-resolution | |
| branch-suffix: timestamp | |
| delete-branch: true | |
| labels: bot,qa/skip-qa,bot/resolve-build-deps | |
| body: |- | |
| ### Motivation | |
| Direct dependencies were updated in ${{ github.sha }}. | |
| ### Additional Notes | |
| This PR was automatically generated by the following workflow: | |
| ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} |