impit/.github/workflows/version-benchmark.yaml at 6640681d5baf321103830112e6baf558c8191259 · apify/impit · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
name: Compare impit versions

on:
  schedule:
    # First of the month, offset from the client comparison so they don't compete for runners.
    - cron: '0 5 1 * *'
  workflow_dispatch:
    inputs:
      versions:
        description: Versions per ecosystem to compare
        default: '5'
      requests:
        description: Requests per run
        default: '2000'
      runs:
        description: Runs per version, the median is reported
        default: '11'
  # Exercise the harness whenever it changes. version-benchmark-comment.yaml posts the
  # resulting chart as a pull request comment - see that file for why it's a separate
  # workflow rather than a second job here.
  pull_request:
    paths:
      - benchmarks/**
      - .github/workflows/version-benchmark.yaml

permissions:
  contents: read

concurrency:
  # Runs on different PRs shouldn't block each other, but a new push to the same PR
  # makes its own previous run's comment stale, so that one is cancelled outright.
  group: version-benchmark-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  VERSIONS: ${{ inputs.versions || '5' }}
  REQUESTS: ${{ inputs.requests || '2000' }}
  RUNS: ${{ inputs.runs || '11' }}

jobs:
  benchmark:
    name: Measure
    runs-on: ubuntu-latest
    timeout-minutes: 60
    steps:
      # No credentials in this job: it installs and runs several unpinned releases
      # of impit, so it must have nothing worth stealing.
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          ref: ${{ github.event_name == 'pull_request' && github.sha || 'master' }}
          persist-credentials: false

      - name: Setup Node.js
        uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
        with:
          node-version: 24

      - name: Setup uv
        uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1

      - name: Set up a Python venv with matplotlib
        run: |
          uv venv --seed --python 3.12 benchmarks/python/.venv
          uv pip install --python benchmarks/python/.venv/bin/python matplotlib

      - name: Benchmark the npm releases
        run: node benchmarks/node/bench-versions.mjs --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS"

      - name: Benchmark the PyPI releases
        run: benchmarks/python/.venv/bin/python benchmarks/python/bench_versions.py --versions "$VERSIONS" --requests "$REQUESTS" --runs "$RUNS"

      - name: Render the chart
        run: benchmarks/python/.venv/bin/python benchmarks/chart-versions.py

      - name: Upload the chart and raw measurements
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        if: always()
        with:
          name: version-chart
          path: |
            benchmarks/version-chart.png
            benchmarks/results-node-versions.json
            benchmarks/results-python-versions.json