Update Dependencies From Metadata (Retrieve, Metadata, Compile, Test, Create PR) #1544
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Dependencies From Metadata (Retrieve, Metadata, Compile, Test, Create PR) | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '57 13 * * *' # daily at 13:57 UTC | |
| jobs: | |
| retrieve: | |
| name: Retrieve New Versions and Generate Metadata | |
| runs-on: ubuntu-latest | |
| outputs: | |
| metadata-filepath: ${{ steps.retrieve.outputs.metadata-filepath }} | |
| metadata-json: ${{ steps.retrieve.outputs.metadata-json }} | |
| # from-source-metadata-filepath is the path to a file containing a subset | |
| # of metadata-json entries for NON-compiled dependencies | |
| from-source-metadata-filepath: ${{ steps.retrieve.outputs.from-source-metadata-filepath }} | |
| # compilation-json is a subset of metadata-json entries which are missing | |
| # a `checksum` and `uri` | |
| compilation-json: ${{ steps.retrieve.outputs.compilation-json }} | |
| id: ${{ steps.retrieve.outputs.id }} | |
| length: ${{ steps.retrieve.outputs.length }} | |
| compilation-length: ${{ steps.retrieve.outputs.compilation-length }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Setup Go | |
| uses: actions/setup-go@v6 | |
| with: | |
| # hashFiles returns empty string if file does not exist | |
| go-version-file: ${{ hashFiles('dependency/retrieval/go.mod') != '' && 'dependency/retrieval/go.mod' || 'go.mod' }} | |
| - name: Run Retrieve | |
| id: retrieve | |
| working-directory: dependency | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| OUTPUT="/tmp/metadata.json" | |
| make retrieve \ | |
| buildpackTomlPath="${{ github.workspace }}/buildpack.toml" \ | |
| output="${OUTPUT}" | |
| id=$(jq -r .[0].id < "${OUTPUT}") | |
| content=$(jq -r < "${OUTPUT}") | |
| length=$(echo $content | jq -r '. | length') | |
| compilation=$(echo $content | jq -r 'map(select(.checksum == null and .uri == null))'?) | |
| complength=$(echo $compilation | jq -r '. | length') | |
| echo $content | jq -r 'map(select(.checksum != null and .uri != null))'? > "/tmp/from-source-metadata.json" | |
| echo "from-source-metadata-filepath=/tmp/from-source-metadata.json" >> "$GITHUB_OUTPUT" | |
| delimiter="$(uuidgen)" | |
| echo "metadata-filepath=${OUTPUT}" >> "$GITHUB_OUTPUT" | |
| printf "metadata-json<<%s\n%s\n%s\n" "${delimiter}" "${content}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |
| echo "id=$id" >> "$GITHUB_OUTPUT" | |
| echo "length=$length" >> "$GITHUB_OUTPUT" | |
| printf "compilation-json<<%s\n%s\n%s\n" "${delimiter}" "${compilation}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |
| echo "compilation-length=$complength" >> "$GITHUB_OUTPUT" | |
| - name: Upload `${{ steps.retrieve.outputs.metadata-filepath }}` | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: metadata.json | |
| path: ${{ steps.retrieve.outputs.metadata-filepath }} | |
| - name: Upload `${{ steps.retrieve.outputs.from-source-metadata-filepath }}` | |
| uses: actions/upload-artifact@v6 | |
| with: | |
| name: from-source-metadata.json | |
| path: ${{ steps.retrieve.outputs.from-source-metadata-filepath }} | |
| # Check if there is buildpack-provided compilation code and testing code | |
| # Optional compilation code expected at: <buildpack>/dependency/actions/compile/ | |
| # Optional testing code expected at: <buildpack>/dependency/test/ | |
| get-compile-and-test: | |
| name: Get Compilation and Testing Code | |
| outputs: | |
| should-compile: ${{ steps.compile-check.outputs.should-compile }} | |
| should-test: ${{ steps.test-check.outputs.should-test }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Has Compilation Action? | |
| id: compile-check | |
| run: | | |
| if test -d "dependency/actions/compile"; then | |
| echo "Compilation action provided" | |
| echo "should-compile=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Has Testing Action? | |
| id: test-check | |
| run: | | |
| if test -d "dependency/test"; then | |
| echo "Testing file provided" | |
| echo "should-test=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| test: | |
| name: Test Non-Compiled Dependency | |
| needs: | |
| - retrieve | |
| - get-compile-and-test | |
| strategy: | |
| matrix: | |
| includes: ${{ fromJSON(needs.retrieve.outputs.metadata-json) }} | |
| # Run job step if BOTH: | |
| # (1) needs.get-compile-and-test.outputs.should-test = TRUE -> if there is a dependency/test directory in the buildpack | |
| # (2) needs.get-compile-and-test.outputs.should-compile = FALSE -> if there is NOT a dependency/actions/compile directory in the buildpack | |
| # AND: | |
| # (3) there is at least one new version to test | |
| if: ${{ needs.retrieve.outputs.length > 0 && needs.get-compile-and-test.outputs.should-test == 'true' && needs.get-compile-and-test.outputs.should-compile == 'false' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v6 | |
| - name: Make Temporary Artifact Directory | |
| id: make-outputdir | |
| run: echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT" | |
| # Download the tarball for testing if: | |
| # (1) dependency testing code is present in the buildpack directory | |
| # (2) URI in metadata.json is available | |
| - name: Download upstream tarball (if not compiled) | |
| if: ${{ matrix.includes.uri != '' && needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| curl ${{ matrix.includes.uri }} \ | |
| --fail-with-body \ | |
| --show-error \ | |
| --silent \ | |
| --location \ | |
| --output ${{ steps.make-outputdir.outputs.outputdir }}/dependency.tgz | |
| # Test the dependency tarball if: | |
| # (1) dependency testing code is present in the buildpack directory | |
| - name: Test Upstream Dependency | |
| working-directory: dependency | |
| if: ${{ needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| run: | | |
| make test \ | |
| version="${{ matrix.includes.version }}" \ | |
| tarballPath="${{ steps.make-outputdir.outputs.outputdir }}/*.tgz" | |
| compile: | |
| name: Compile and Test Dependency | |
| needs: | |
| - retrieve | |
| - get-compile-and-test | |
| strategy: | |
| matrix: | |
| includes: ${{ fromJSON(needs.retrieve.outputs.compilation-json) }} | |
| # Run job step if: | |
| # (1) needs.get-compile-and-test.outputs.should-compile -> if there is a dependency/actions/compile directory in the buildpack | |
| # (2) OR needs.get-compile-and-test.outputs.should-test -> if there is a dependency/test directory in the buildpack | |
| # AND: | |
| # (3) there is at least one version to compile/test | |
| if: ${{ needs.retrieve.outputs.compilation-length > 0 && (needs.get-compile-and-test.outputs.should-compile == 'true' || needs.get-compile-and-test.outputs.should-test == 'true') }} | |
| uses: ./.github/workflows/compile-dependency.yml | |
| with: | |
| version: "${{ matrix.includes.version }}" | |
| target: "${{ matrix.includes.target }}" | |
| os: "${{ matrix.includes.os }}" | |
| arch: "${{ matrix.includes.arch }}" | |
| shouldCompile: ${{ matrix.includes.checksum == '' && matrix.includes.uri == '' }} | |
| shouldTest: ${{ matrix.includes.checksum == '' && matrix.includes.uri == '' && needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| uploadArtifactName: "${{ needs.retrieve.outputs.id }}-${{ matrix.includes.version }}-${{ matrix.includes.os != '' && matrix.includes.os || 'linux' }}-${{ matrix.includes.arch != '' && matrix.includes.arch || 'amd64' }}-${{ matrix.includes.target }}" |