Benchmark PR #7
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Benchmark PR | |
| on: | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| - reopened | |
| - ready_for_review | |
| workflow_dispatch: | |
| inputs: | |
| versions: | |
| description: 'Comma/space-separated Javalin versions (overrides config/pr-versions.txt)' | |
| required: false | |
| type: string | |
| iterations: | |
| description: 'JMH warmup and measurement iterations' | |
| required: false | |
| default: '10' | |
| type: string | |
| iterationTimeMs: | |
| description: 'JMH warmup and measurement time in milliseconds' | |
| required: false | |
| default: '1000' | |
| type: string | |
| forks: | |
| description: 'JMH forks' | |
| required: false | |
| default: '2' | |
| type: string | |
| threads: | |
| description: 'JMH worker threads' | |
| required: false | |
| default: '4' | |
| type: string | |
| sourceRepository: | |
| description: 'Optional source repository for snapshot metadata' | |
| required: false | |
| type: string | |
| sourceSha: | |
| description: 'Optional source commit SHA for snapshot metadata' | |
| required: false | |
| type: string | |
| sourceRef: | |
| description: 'Optional source ref for snapshot metadata' | |
| required: false | |
| type: string | |
| sourcePrNumber: | |
| description: 'Optional source PR number for snapshot metadata' | |
| required: false | |
| type: string | |
| triggerRepository: | |
| description: 'Optional triggering repository' | |
| required: false | |
| type: string | |
| triggerPrNumber: | |
| description: 'Optional triggering PR number' | |
| required: false | |
| type: string | |
| triggerPrUrl: | |
| description: 'Optional triggering PR URL' | |
| required: false | |
| type: string | |
| permissions: | |
| contents: write | |
| pages: write | |
| id-token: write | |
| concurrency: | |
| group: benchmark-pr-${{ github.event.pull_request.number || github.event.inputs.sourcePrNumber || github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| benchmark: | |
| runs-on: ubuntu-latest | |
| env: | |
| INPUT_VERSIONS: ${{ github.event.inputs.versions }} | |
| INPUT_ITERATIONS: ${{ github.event.inputs.iterations }} | |
| INPUT_ITERATION_TIME_MS: ${{ github.event.inputs.iterationTimeMs }} | |
| INPUT_FORKS: ${{ github.event.inputs.forks }} | |
| INPUT_THREADS: ${{ github.event.inputs.threads }} | |
| INPUT_SOURCE_REPOSITORY: ${{ github.event.inputs.sourceRepository }} | |
| INPUT_SOURCE_SHA: ${{ github.event.inputs.sourceSha }} | |
| INPUT_SOURCE_REF: ${{ github.event.inputs.sourceRef }} | |
| INPUT_SOURCE_PR_NUMBER: ${{ github.event.inputs.sourcePrNumber }} | |
| INPUT_TRIGGER_REPOSITORY: ${{ github.event.inputs.triggerRepository }} | |
| INPUT_TRIGGER_PR_NUMBER: ${{ github.event.inputs.triggerPrNumber }} | |
| INPUT_TRIGGER_PR_URL: ${{ github.event.inputs.triggerPrUrl }} | |
| GH_EVENT_PR_NUMBER: ${{ github.event.pull_request.number || '' }} | |
| PR_OR_RUN_ID: ${{ github.event.pull_request.number || github.event.inputs.sourcePrNumber || github.run_id }} | |
| GITHUB_REPOSITORY: ${{ github.repository }} | |
| GITHUB_WORKFLOW: ${{ github.workflow }} | |
| GITHUB_RUN_NUMBER: ${{ github.run_number }} | |
| GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} | |
| GITHUB_SHA: ${{ github.sha }} | |
| GITHUB_REF_NAME: ${{ github.ref_name }} | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PERFORMANCE_TESTS_TRIGGER_TOKEN: ${{ secrets.PERFORMANCE_TESTS_TRIGGER_TOKEN }} | |
| BENCHMARK_DATA_BRANCH: ${{ vars.BENCHMARK_DATA_BRANCH || 'benchmark-data' }} | |
| PR_PREVIEW_BASE_URL: ${{ vars.PR_PREVIEW_PAGES_BASE_URL || 'https://javalin.github.io/javalin-performance-tests-testing/pr-previews' }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Set up Java | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: temurin | |
| java-version: '17' | |
| cache: gradle | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.x' | |
| - name: Install Javalin source snapshot to Maven local (optional) | |
| id: source_snapshot | |
| if: ${{ env.INPUT_SOURCE_REPOSITORY != '' && env.INPUT_SOURCE_SHA != '' }} | |
| run: | | |
| set -euo pipefail | |
| source_dir="$(mktemp -d)" | |
| git clone "https://github.com/${INPUT_SOURCE_REPOSITORY}.git" "${source_dir}" | |
| pushd "${source_dir}" >/dev/null | |
| checkout_target="${INPUT_SOURCE_REF:-${INPUT_SOURCE_SHA}}" | |
| git checkout "${checkout_target}" | |
| chmod +x ./mvnw | |
| snapshot_version="$(./mvnw -q -DforceStdout help:evaluate -Dexpression=project.version | tail -n 1 | tr -d '\r')" | |
| if [ -z "${snapshot_version}" ]; then | |
| echo "Could not resolve snapshot project.version." >&2 | |
| exit 1 | |
| fi | |
| echo "snapshot_version=${snapshot_version}" >> "$GITHUB_OUTPUT" | |
| ./mvnw -DRunningOnCi=true -DskipTests=true clean install --file pom.xml --batch-mode | |
| popd >/dev/null | |
| - name: Run PR benchmark suite | |
| run: | | |
| set -euo pipefail | |
| RUN_ID="pr-${PR_OR_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| RUN_TIMESTAMP_UTC="$(date -u +%Y-%m-%dT%H:%M:%SZ)" | |
| VERSIONS_JSON="$(python3 scripts/resolve_versions.py --raw "${INPUT_VERSIONS:-}" --config config/pr-versions.txt)" | |
| SNAPSHOT_VERSION="${{ steps.source_snapshot.outputs.snapshot_version }}" | |
| if [ -n "${SNAPSHOT_VERSION:-}" ]; then | |
| VERSIONS_JSON="$(python3 -c 'import json, sys; versions = json.loads(sys.argv[1]); snapshot = sys.argv[2]; versions.append(snapshot) if snapshot and snapshot not in versions else None; print(json.dumps(versions))' "${VERSIONS_JSON}" "${SNAPSHOT_VERSION}")" | |
| fi | |
| ITERATIONS="${INPUT_ITERATIONS:-10}" | |
| ITERATION_TIME_MS="${INPUT_ITERATION_TIME_MS:-1000}" | |
| FORKS="${INPUT_FORKS:-2}" | |
| THREADS="${INPUT_THREADS:-4}" | |
| BENCHMARK_REPOSITORY="${INPUT_SOURCE_REPOSITORY:-${GITHUB_REPOSITORY}}" | |
| BENCHMARK_GIT_SHA="${INPUT_SOURCE_SHA:-${GITHUB_SHA}}" | |
| BENCHMARK_GIT_REF="${INPUT_SOURCE_REF:-${GITHUB_REF_NAME}}" | |
| mkdir -p current-run/results | |
| python3 scripts/collect_runner_info.py current-run/runner-info.json | |
| python3 scripts/write_run_metadata.py \ | |
| --output current-run/run-metadata.json \ | |
| --run-id "${RUN_ID}" \ | |
| --run-timestamp-utc "${RUN_TIMESTAMP_UTC}" \ | |
| --versions-json "${VERSIONS_JSON}" \ | |
| --iterations "${ITERATIONS}" \ | |
| --iteration-time-ms "${ITERATION_TIME_MS}" \ | |
| --forks "${FORKS}" \ | |
| --threads "${THREADS}" \ | |
| --repository "${BENCHMARK_REPOSITORY}" \ | |
| --workflow "${GITHUB_WORKFLOW}" \ | |
| --run-number "${GITHUB_RUN_NUMBER}" \ | |
| --run-attempt "${GITHUB_RUN_ATTEMPT}" \ | |
| --git-sha "${BENCHMARK_GIT_SHA}" \ | |
| --git-ref "${BENCHMARK_GIT_REF}" \ | |
| --source-repository "${INPUT_SOURCE_REPOSITORY:-}" \ | |
| --source-sha "${INPUT_SOURCE_SHA:-}" \ | |
| --source-ref "${INPUT_SOURCE_REF:-}" \ | |
| --source-pr-number "${INPUT_SOURCE_PR_NUMBER:-}" \ | |
| --trigger-repository "${INPUT_TRIGGER_REPOSITORY:-}" \ | |
| --trigger-pr-number "${INPUT_TRIGGER_PR_NUMBER:-}" \ | |
| --trigger-pr-url "${INPUT_TRIGGER_PR_URL:-}" | |
| python3 scripts/json_to_lines.py "${VERSIONS_JSON}" > /tmp/versions.txt | |
| while IFS= read -r version; do | |
| echo "Running PR benchmark for Javalin ${version}" | |
| ./gradlew --no-daemon clean benchmark \ | |
| -PjavalinVersion="${version}" \ | |
| -Piterations="${ITERATIONS}" \ | |
| -PiterationTime="${ITERATION_TIME_MS}" \ | |
| -Pthreads="${THREADS}" \ | |
| -Pforks="${FORKS}" \ | |
| -PresultFormat="json" | |
| cp "results/${version}.json" "current-run/results/${version}.json" | |
| done < /tmp/versions.txt | |
| - name: Build trend report including history branch | |
| run: | | |
| set -euo pipefail | |
| mkdir -p pr-history/runs | |
| repo_url="https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" | |
| if git ls-remote --exit-code --heads "$repo_url" "${BENCHMARK_DATA_BRANCH}" >/dev/null 2>&1; then | |
| git clone --depth 1 --branch "${BENCHMARK_DATA_BRANCH}" "$repo_url" benchmark-history | |
| else | |
| mkdir benchmark-history | |
| pushd benchmark-history >/dev/null | |
| git init | |
| git checkout -b "${BENCHMARK_DATA_BRANCH}" | |
| git remote add origin "$repo_url" | |
| popd >/dev/null | |
| fi | |
| if [ -d benchmark-history/runs ]; then | |
| cp -R benchmark-history/runs/. pr-history/runs/ | |
| fi | |
| RUN_ID="$(python3 -c 'import json; print(json.load(open("current-run/run-metadata.json"))["runId"])')" | |
| mkdir -p "pr-history/runs/${RUN_ID}" | |
| cp -R current-run/. "pr-history/runs/${RUN_ID}/" | |
| python3 scripts/generate_pages.py \ | |
| --history-root pr-history/runs \ | |
| --output-dir pr-site \ | |
| --repository "${GITHUB_REPOSITORY}" | |
| - name: Resolve PR preview target | |
| id: preview_target | |
| run: | | |
| set -euo pipefail | |
| pr_number="${INPUT_SOURCE_PR_NUMBER:-${INPUT_TRIGGER_PR_NUMBER:-${GH_EVENT_PR_NUMBER:-}}}" | |
| run_id="$(python3 -c 'import json; print(json.load(open("current-run/run-metadata.json"))["runId"])')" | |
| if [ -z "${pr_number}" ]; then | |
| echo "No PR number resolved; skipping PR preview publishing." | |
| echo "has_pr_number=false" >> "$GITHUB_OUTPUT" | |
| exit 0 | |
| fi | |
| preview_root="pr-${pr_number}" | |
| preview_run_dir="${preview_root}/${run_id}" | |
| preview_latest_dir="${preview_root}/latest" | |
| base_url="${PR_PREVIEW_BASE_URL%/}" | |
| echo "has_pr_number=true" >> "$GITHUB_OUTPUT" | |
| echo "pr_number=${pr_number}" >> "$GITHUB_OUTPUT" | |
| echo "preview_root=${preview_root}" >> "$GITHUB_OUTPUT" | |
| echo "preview_run_dir=${preview_run_dir}" >> "$GITHUB_OUTPUT" | |
| echo "preview_latest_dir=${preview_latest_dir}" >> "$GITHUB_OUTPUT" | |
| echo "preview_run_url=${base_url}/${preview_run_dir}/" >> "$GITHUB_OUTPUT" | |
| echo "preview_latest_url=${base_url}/${preview_latest_dir}/" >> "$GITHUB_OUTPUT" | |
| - name: Publish PR preview content to benchmark-data branch | |
| if: ${{ steps.preview_target.outputs.has_pr_number == 'true' }} | |
| run: | | |
| set -euo pipefail | |
| preview_run_path="pr-previews/${{ steps.preview_target.outputs.preview_run_dir }}" | |
| preview_latest_path="pr-previews/${{ steps.preview_target.outputs.preview_latest_dir }}" | |
| preview_root_path="pr-previews/${{ steps.preview_target.outputs.preview_root }}" | |
| rm -rf "benchmark-history/${preview_run_path}" "benchmark-history/${preview_latest_path}" | |
| mkdir -p "benchmark-history/${preview_run_path}" "benchmark-history/${preview_latest_path}" | |
| cp -R pr-site/. "benchmark-history/${preview_run_path}/" | |
| cp -R pr-site/. "benchmark-history/${preview_latest_path}/" | |
| pushd benchmark-history >/dev/null | |
| git config user.name "github-actions[bot]" | |
| git config user.email "41898282+github-actions[bot]@users.noreply.github.com" | |
| git add "${preview_root_path}" | |
| if git diff --cached --quiet; then | |
| echo "No preview page changes to commit." | |
| else | |
| git commit -m "Update PR preview ${preview_root_path}" | |
| git push origin HEAD:"${BENCHMARK_DATA_BRANCH}" | |
| fi | |
| popd >/dev/null | |
| - name: Build deployable site (main + PR previews) | |
| run: | | |
| set -euo pipefail | |
| mkdir -p benchmark-history/runs | |
| python3 scripts/generate_pages.py \ | |
| --history-root benchmark-history/runs \ | |
| --output-dir site \ | |
| --repository "${GITHUB_REPOSITORY}" | |
| if [ -d benchmark-history/pr-previews ]; then | |
| mkdir -p site/pr-previews | |
| cp -R benchmark-history/pr-previews/. site/pr-previews/ | |
| fi | |
| - name: Configure Pages | |
| uses: actions/configure-pages@v5 | |
| - name: Upload Pages artifact | |
| uses: actions/upload-pages-artifact@v3 | |
| with: | |
| path: site | |
| - name: Deploy to GitHub Pages | |
| id: deploy | |
| uses: actions/deploy-pages@v4 | |
| - name: Add PR preview links to job summary | |
| if: ${{ steps.preview_target.outputs.has_pr_number == 'true' }} | |
| run: | | |
| { | |
| echo "### PR Preview Links" | |
| echo "- Latest: ${{ steps.preview_target.outputs.preview_latest_url }}" | |
| echo "- This run: ${{ steps.preview_target.outputs.preview_run_url }}" | |
| } >> "$GITHUB_STEP_SUMMARY" | |
| - name: Add summary to job page | |
| run: | | |
| python3 scripts/print_summary_markdown.py pr-site/summary.json --limit 60 >> "$GITHUB_STEP_SUMMARY" | |
| - name: Upload PR benchmark artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pr-benchmark-${{ github.run_id }} | |
| path: | | |
| current-run | |
| pr-site | |
| if-no-files-found: error | |
| - name: Comment benchmark success on triggering PR | |
| if: success() && env.INPUT_TRIGGER_PR_NUMBER != '' | |
| uses: actions/github-script@v8 | |
| with: | |
| github-token: ${{ secrets.PERFORMANCE_TESTS_TRIGGER_TOKEN }} | |
| script: | | |
| const [owner, repo] = process.env.INPUT_TRIGGER_REPOSITORY.split("/"); | |
| const prNumber = parseInt(process.env.INPUT_TRIGGER_PR_NUMBER, 10); | |
| const runUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| const previewUrl = `${{ steps.preview_target.outputs.preview_latest_url }}`; | |
| const previewLine = previewUrl ? `- [View PR preview page](${previewUrl})\n` : ""; | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: prNumber, | |
| body: | |
| "Performance benchmark completed successfully.\n\n" + | |
| `- [View benchmark workflow run](${runUrl})\n` + | |
| previewLine, | |
| }); | |
| - name: Comment benchmark failure on triggering PR | |
| if: failure() && env.INPUT_TRIGGER_PR_NUMBER != '' | |
| uses: actions/github-script@v8 | |
| with: | |
| github-token: ${{ secrets.PERFORMANCE_TESTS_TRIGGER_TOKEN }} | |
| script: | | |
| const [owner, repo] = process.env.INPUT_TRIGGER_REPOSITORY.split("/"); | |
| const prNumber = parseInt(process.env.INPUT_TRIGGER_PR_NUMBER, 10); | |
| const runUrl = `https://github.com/${process.env.GITHUB_REPOSITORY}/actions/runs/${process.env.GITHUB_RUN_ID}`; | |
| await github.rest.issues.createComment({ | |
| owner, | |
| repo, | |
| issue_number: prNumber, | |
| body: | |
| "Performance benchmark failed.\n\n" + | |
| `- [View workflow run](${runUrl})`, | |
| }); |