ci: chart impit's own npm/PyPI release history on PR comments #1
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Compare impit versions | |
| on: | |
| schedule: | |
| # First of the month, offset from the client comparison so they don't compete for runners. | |
| - cron: '0 5 1 * *' | |
| workflow_dispatch: | |
| inputs: | |
| versions: | |
| description: Versions per ecosystem to compare | |
| default: '5' | |
| requests: | |
| description: Requests per run | |
| default: '2000' | |
| runs: | |
| description: Runs per version, the median is reported | |
| default: '11' | |
| # Exercise the harness whenever it changes, commenting the chart on the PR that changed it. | |
| pull_request: | |
| paths: | |
| - benchmarks/** | |
| - .github/workflows/version-benchmark.yaml | |
| permissions: | |
| contents: read | |
| concurrency: | |
| # Runs on different PRs shouldn't block each other, but a new push to the same PR | |
| # makes its own previous run's comment stale, so that one is cancelled outright. | |
| group: version-benchmark-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| VERSIONS: ${{ inputs.versions || '5' }} | |
| REQUESTS: ${{ inputs.requests || '2000' }} | |
| RUNS: ${{ inputs.runs || '11' }} | |
| jobs: | |
| benchmark: | |
| name: Measure | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| steps: | |
| # No credentials in this job: it installs and runs several unpinned releases | |
| # of impit, so it must have nothing worth stealing. | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }} | |
| persist-credentials: false | |
| - name: Setup Node.js | |
| uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7 | |
| with: | |
| node-version: 24 | |
| - name: Setup uv | |
| uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1 | |
| - name: Set up a Python venv with matplotlib | |
| run: | | |
| uv venv --seed --python 3.12 benchmarks/python/.venv | |
| uv pip install --python benchmarks/python/.venv/bin/python matplotlib | |
| - name: Benchmark the npm releases | |
| run: node benchmarks/node/bench-versions.mjs --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS" | |
| - name: Benchmark the PyPI releases | |
| run: benchmarks/python/.venv/bin/python benchmarks/python/bench_versions.py --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS" | |
| - name: Render the chart | |
| run: benchmarks/python/.venv/bin/python benchmarks/chart-versions.py | |
| - name: Upload the chart and raw measurements | |
| uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 | |
| if: always() | |
| with: | |
| name: version-chart | |
| path: | | |
| benchmarks/version-chart.png | |
| benchmarks/results-node-versions.json | |
| benchmarks/results-python-versions.json | |
| # Separate runner, so the token never shares an environment with the unpinned | |
| # releases installed above. Only meaningful on a pull request: there is nowhere | |
| # to comment on a schedule or a manual run. | |
| comment: | |
| name: Comment on the pull request | |
| needs: benchmark | |
| if: github.event_name == 'pull_request' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 | |
| with: | |
| ref: master | |
| token: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| - name: Download the chart | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 | |
| with: | |
| name: version-chart | |
| path: downloaded | |
| - name: Publish the chart and comment on the pull request | |
| env: | |
| GH_TOKEN: ${{ secrets.APIFY_SERVICE_ACCOUNT_GITHUB_TOKEN }} | |
| PR_NUMBER: ${{ github.event.pull_request.number }} | |
| run: | | |
| test -f downloaded/version-chart.png || { echo 'the artifact did not unpack where expected'; exit 1; } | |
| # The chart is published to its own branch instead of the pull request, | |
| # so a raw.githubusercontent.com URL can be embedded in a comment - | |
| # GitHub strips data: URIs from comment markdown, and the pull request | |
| # itself should not carry a generated PNG as one of its files. | |
| git config user.name 'apify-service-account' | |
| git config user.email 'apify-service-account@users.noreply.github.com' | |
| git fetch origin benchmarks/version-chart-assets || true | |
| if git show-ref --verify --quiet refs/remotes/origin/benchmarks/version-chart-assets; then | |
| git checkout -B benchmarks/version-chart-assets origin/benchmarks/version-chart-assets | |
| else | |
| git checkout --orphan benchmarks/version-chart-assets | |
| git rm -rf . > /dev/null | |
| fi | |
| filename="pr-${PR_NUMBER}.png" | |
| mv downloaded/version-chart.png "$filename" | |
| git add "$filename" | |
| git commit -m "chore: publish version benchmark chart for #${PR_NUMBER}" | |
| git push origin benchmarks/version-chart-assets | |
| url="https://raw.githubusercontent.com/${GITHUB_REPOSITORY}/benchmarks/version-chart-assets/${filename}?run=${GITHUB_RUN_ID}" | |
| { | |
| echo '### impit version comparison' | |
| echo | |
| echo "Median throughput of the last $VERSIONS npm and PyPI releases of impit." | |
| echo | |
| echo "" | |
| } > comment.md | |
| gh pr comment "$PR_NUMBER" --body-file comment.md --edit-last --create-if-none \ | |
| || gh pr comment "$PR_NUMBER" --body-file comment.md |