chore(ci): update zizmor workflow checker #903
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run performance regression benchmarks and return parsed results to associated pull-request. | |
| name: benchmark_perf_regression | |
| on: | |
| issue_comment: | |
| types: [ created ] | |
| pull_request: | |
| types: [ labeled ] | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| SLACK_CHANNEL: ${{ secrets.SLACK_CHANNEL }} | |
| SLACK_ICON: https://pbs.twimg.com/profile_images/1274014582265298945/OjBKP9kn_400x400.png | |
| SLACK_USERNAME: ${{ secrets.BOT_USERNAME }} | |
| SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} | |
| permissions: { } | |
| jobs: | |
| verify-triggering-actor: | |
| name: benchmark_perf_regression/verify-actor | |
| if: (github.event_name == 'pull_request' && | |
| (contains(github.event.label.name, 'bench-perfs-cpu') || | |
| contains(github.event.label.name, 'bench-perfs-gpu'))) || | |
| (github.event.issue.pull_request && startsWith(github.event.comment.body, '/bench')) | |
| uses: ./.github/workflows/verify_triggering_actor.yml | |
| secrets: | |
| ALLOWED_TEAM: ${{ secrets.RELEASE_TEAM }} | |
| READ_ORG_TOKEN: ${{ secrets.READ_ORG_TOKEN }} | |
| prepare-benchmarks: | |
| name: benchmark_perf_regression/prepare-benchmarks | |
| needs: verify-triggering-actor | |
| runs-on: ubuntu-latest | |
| outputs: | |
| commands: ${{ steps.set_commands.outputs.commands }} | |
| slab-backend: ${{ steps.set_slab_details.outputs.backend }} | |
| slab-profile: ${{ steps.set_slab_details.outputs.profile }} | |
| hardware-name: ${{ steps.get_hardware_name.outputs.name }} | |
| tfhe-backend: ${{ steps.set_regression_details.outputs.tfhe-backend }} | |
| selected-regression-profile: ${{ steps.set_regression_details.outputs.selected-profile }} | |
| custom-env: ${{ steps.get_custom_env.outputs.custom_env }} | |
| permissions: | |
| # Needed to write a comment in a pull-request | |
| pull-requests: write | |
| steps: | |
| - name: Checkout tfhe-rs repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| with: | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Acknowledge issue comment | |
| if: github.event_name == 'issue_comment' | |
| uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 | |
| with: | |
| comment-id: ${{ github.event.comment.id }} | |
| reactions: '+1' | |
| - name: Display workflow run URL | |
| if: github.event_name == 'issue_comment' | |
| uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body: | | |
| User triggered performance regression benchmark. | |
| Workflow run URL: ${{ env.ACTION_RUN_URL }} | |
| - name: Generate CPU benchmarks command from label | |
| if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-cpu')) | |
| run: | | |
| echo "DEFAULT_BENCH_OPTIONS=--backend cpu" >> "${GITHUB_ENV}" | |
| - name: Generate GPU benchmarks command from label | |
| if: (github.event_name == 'pull_request' && contains(github.event.label.name, 'bench-perfs-gpu')) | |
| run: | | |
| echo "DEFAULT_BENCH_OPTIONS=--backend gpu" >> "${GITHUB_ENV}" | |
| # TODO add support for HPU backend | |
| - name: Install Python requirements | |
| run: | | |
| python3 -m pip install -r ci/perf_regression/requirements.txt | |
| - name: Generate cargo commands and env from label | |
| if: github.event_name == 'pull_request' | |
| run: | | |
| python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "/bench ${DEFAULT_BENCH_OPTIONS}" | |
| echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}" | |
| - name: Dump issue comment into file # To avoid possible code-injection | |
| if: github.event_name == 'issue_comment' | |
| run: | | |
| echo "${COMMENT_BODY}" >> dumped_comment.txt | |
| env: | |
| COMMENT_BODY: ${{ github.event.comment.body }} | |
| - name: Generate cargo commands and env | |
| if: github.event_name == 'issue_comment' | |
| run: | | |
| python3 ci/perf_regression/perf_regression.py parse_profile --issue-comment "$(cat dumped_comment.txt)" | |
| echo "COMMANDS=$(cat ci/perf_regression/perf_regression_generated_commands.json)" >> "${GITHUB_ENV}" | |
| - name: Set commands output | |
| id: set_commands | |
| run: | # zizmor: ignore[template-injection] this env variable is safe | |
| echo "commands=${{ toJSON(env.COMMANDS) }}" >> "${GITHUB_OUTPUT}" | |
| - name: Set Slab details outputs | |
| id: set_slab_details | |
| run: | | |
| echo "backend=$(cat ci/perf_regression/perf_regression_slab_backend_config.txt)" >> "${GITHUB_OUTPUT}" | |
| echo "profile=$(cat ci/perf_regression/perf_regression_slab_profile_config.txt)" >> "${GITHUB_OUTPUT}" | |
| - name: Get hardware name | |
| id: get_hardware_name | |
| run: | # zizmor: ignore[template-injection] these interpolations are safe | |
| HARDWARE_NAME=$(python3 ci/hardware_finder.py "${{ steps.set_slab_details.outputs.backend }}" "${{ steps.set_slab_details.outputs.profile }}"); | |
| echo "name=${HARDWARE_NAME}" >> "${GITHUB_OUTPUT}" | |
| - name: Set regression details outputs | |
| id: set_regression_details | |
| run: | | |
| echo "tfhe-backend=$(cat ci/perf_regression/perf_regression_tfhe_rs_backend_config.txt)" >> "${GITHUB_OUTPUT}" | |
| echo "selected-profile=$(cat ci/perf_regression/perf_regression_selected_profile_config.txt)" >> "${GITHUB_OUTPUT}" | |
| - name: Get custom env vars | |
| id: get_custom_env | |
| run: | | |
| echo "custom_env=$(cat ci/perf_regression/perf_regression_custom_env.sh)" >> "${GITHUB_OUTPUT}" | |
| setup-instance: | |
| name: benchmark_perf_regression/setup-instance | |
| needs: prepare-benchmarks | |
| runs-on: ubuntu-latest | |
| outputs: | |
| runner-name: ${{ steps.start-instance.outputs.label }} | |
| steps: | |
| - name: Start instance | |
| id: start-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: ${{ needs.prepare-benchmarks.outputs.slab-backend }} | |
| profile: ${{ needs.prepare-benchmarks.outputs.slab-profile }} | |
| install-cuda-dependencies-if-required: | |
| name: benchmark_perf_regression/install-cuda-dependencies-if-required | |
| needs: [ prepare-benchmarks, setup-instance ] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| strategy: | |
| matrix: | |
| # explicit include-based build matrix, of known valid options | |
| include: | |
| - cuda: "12.8" | |
| gcc: 11 | |
| steps: | |
| - name: Checkout tfhe-rs repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| with: | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Setup Hyperstack dependencies | |
| if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack' | |
| uses: ./.github/actions/gpu_setup | |
| with: | |
| cuda-version: ${{ matrix.cuda }} | |
| gcc-version: ${{ matrix.gcc }} | |
| regression-benchmarks: | |
| name: benchmark_perf_regression/regression-benchmarks | |
| needs: [ prepare-benchmarks, setup-instance, install-cuda-dependencies-if-required ] | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| concurrency: | |
| group: ${{ github.workflow_ref }}_${{ needs.prepare-benchmarks.outputs.slab-backend }}_${{ needs.prepare-benchmarks.outputs.slab-profile }} | |
| cancel-in-progress: true | |
| timeout-minutes: 720 # 12 hours | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 1 | |
| matrix: | |
| command: ${{ fromJson(needs.prepare-benchmarks.outputs.commands) }} | |
| steps: | |
| - name: Checkout tfhe-rs repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| with: | |
| fetch-depth: 0 # Needed to get commit hash | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Get benchmark details | |
| run: | | |
| COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${SHA}"); | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=${COMMIT_DATE}"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| env: | |
| SHA: ${{ github.sha }} | |
| - name: Export custom env variables | |
| run: | # zizmor: ignore[template-injection] this env variable is safe | |
| { | |
| ${{ needs.prepare-benchmarks.outputs.custom-env }} | |
| } >> "$GITHUB_ENV" | |
| # Re-export environment variables as dependencies setup perform this task in the previous job. | |
| # Local env variables are cleaned at the end of each job. | |
| - name: Export CUDA variables | |
| if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack' | |
| shell: bash | |
| run: | | |
| echo "CUDA_PATH=$CUDA_PATH" >> "${GITHUB_ENV}" | |
| echo "PATH=$PATH:$CUDA_PATH/bin" >> "${GITHUB_PATH}" | |
| echo "LD_LIBRARY_PATH=$CUDA_PATH/lib64:$LD_LIBRARY_PATH" >> "${GITHUB_ENV}" | |
| echo "CUDA_MODULE_LOADER=EAGER" >> "${GITHUB_ENV}" | |
| env: | |
| CUDA_PATH: /usr/local/cuda-12.8 | |
| - name: Export gcc and g++ variables | |
| if: needs.prepare-benchmarks.outputs.slab-backend == 'hyperstack' | |
| shell: bash | |
| run: | | |
| { | |
| echo "CC=/usr/bin/gcc-${GCC_VERSION}"; | |
| echo "CXX=/usr/bin/g++-${GCC_VERSION}"; | |
| echo "CUDAHOSTCXX=/usr/bin/g++-${GCC_VERSION}"; | |
| } >> "${GITHUB_ENV}" | |
| env: | |
| GCC_VERSION: 11 | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@b3b07ba8b418998c39fb20f53e8b695cdcc8de1b # zizmor: ignore[stale-action-refs] this action doesn't create releases | |
| with: | |
| toolchain: nightly | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Run regression benchmarks | |
| run: | | |
| make BENCH_CUSTOM_COMMAND="${BENCH_COMMAND}" bench_custom | |
| env: | |
| BENCH_COMMAND: ${{ matrix.command }} | |
| - name: Parse results | |
| run: | | |
| python3 ./ci/benchmark_parser.py target/criterion "${RESULTS_FILENAME}" \ | |
| --database tfhe_rs \ | |
| --hardware "${HARDWARE_NAME}" \ | |
| --backend "${TFHE_BACKEND}" \ | |
| --project-version "${COMMIT_HASH}" \ | |
| --branch "${REF_NAME}" \ | |
| --commit-date "${COMMIT_DATE}" \ | |
| --bench-date "${BENCH_DATE}" \ | |
| --walk-subdirs \ | |
| --name-suffix regression \ | |
| --bench-type "${BENCH_TYPE}" | |
| echo "RESULTS_FILE_SHA=$(sha256sum "${RESULTS_FILENAME}" | cut -d " " -f1)" >> "${GITHUB_ENV}" | |
| env: | |
| HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }} | |
| TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }} | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| BENCH_TYPE: ${{ env.__TFHE_RS_BENCH_TYPE }} | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@330a01c490aca151604b8cf639adc76d48f6c5d4 | |
| with: | |
| name: ${{ github.sha }}_regression_${{ env.RESULTS_FILE_SHA }} # RESULT_FILE_SHA is needed to avoid collision between matrix.command runs | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ | |
| --slab-url "${SLAB_URL}" | |
| env: | |
| JOB_SECRET: ${{ secrets.JOB_SECRET }} | |
| SLAB_URL: ${{ secrets.SLAB_URL }} | |
| check-regressions: | |
| name: benchmark_perf_regression/check-regressions | |
| needs: [ prepare-benchmarks, regression-benchmarks ] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| # Needed to write a comment in a pull-request | |
| pull-requests: write | |
| # Needed to set up Python dependencies | |
| contents: read | |
| env: | |
| REF_NAME: ${{ github.head_ref || github.ref_name }} | |
| steps: | |
| - name: Checkout tfhe-rs repo | |
| uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 | |
| with: | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Install recent Python | |
| uses: actions/setup-python@e797f83bcb11b83ae66e0230d6156d7c80228e7c # v6.0.0 | |
| with: | |
| python-version: '3.12' | |
| - name: Fetch data | |
| run: | | |
| python3 -m pip install -r ci/data_extractor/requirements.txt | |
| python3 ci/data_extractor/src/data_extractor.py regression_data \ | |
| --generate-regression-json \ | |
| --regression-profiles ci/regression.toml \ | |
| --regression-selected-profile "${REGRESSION_PROFILE}" \ | |
| --backend "${TFHE_BACKEND}" \ | |
| --hardware "${HARDWARE_NAME}" \ | |
| --branch "${REF_NAME}" \ | |
| --time-span-days 60 | |
| env: | |
| REGRESSION_PROFILE: ${{ needs.prepare-benchmarks.outputs.selected-regression-profile }} | |
| TFHE_BACKEND: ${{ needs.prepare-benchmarks.outputs.tfhe-backend }} | |
| HARDWARE_NAME: ${{ needs.prepare-benchmarks.outputs.hardware-name }} | |
| DATA_EXTRACTOR_DATABASE_HOST: ${{ secrets.DATABASE_HOST }} | |
| DATA_EXTRACTOR_DATABASE_USER: ${{ secrets.DATABASE_USER }} | |
| DATA_EXTRACTOR_DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD }} | |
| - name: Generate regression report | |
| run: | | |
| python3 -m pip install -r ci/perf_regression/requirements.txt | |
| python3 ci/perf_regression/perf_regression.py check_regression \ | |
| --results-file regression_data.json \ | |
| --generate-report | |
| - name: Write report in pull-request | |
| uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 | |
| with: | |
| issue-number: ${{ github.event.pull_request.number || github.event.issue.number }} | |
| body-path: ci/perf_regression/regression_report.md | |
| comment-on-failure: | |
| name: benchmark_perf_regression/comment-on-failure | |
| needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ] | |
| runs-on: ubuntu-latest | |
| if: ${{ failure() && github.event_name == 'issue_comment' }} | |
| continue-on-error: true | |
| permissions: | |
| # Needed to write a comment in a pull-request | |
| pull-requests: write | |
| steps: | |
| - name: Write failure message | |
| uses: peter-evans/create-or-update-comment@e8674b075228eee787fea43ef493e45ece1004c9 # v5.0.0 | |
| with: | |
| issue-number: ${{ github.event.issue.number }} | |
| body: | | |
| :x: Performance regression benchmark failed ([workflow run](${{ env.ACTION_RUN_URL }})) | |
| slack-notify: | |
| name: benchmark_perf_regression/slack-notify | |
| needs: [ prepare-benchmarks, setup-instance, regression-benchmarks, check-regressions ] | |
| runs-on: ubuntu-latest | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| steps: | |
| - name: Send message | |
| uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 | |
| env: | |
| SLACK_COLOR: failure | |
| SLACK_MESSAGE: "Performance regression benchmarks failed. (${{ env.ACTION_RUN_URL }})" | |
| teardown-instance: | |
| name: benchmark_perf_regression/teardown-instance | |
| if: ${{ always() && needs.setup-instance.result == 'success' }} | |
| needs: [ setup-instance, regression-benchmarks ] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Stop instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} | |
| - name: Slack Notification | |
| if: ${{ failure() }} | |
| continue-on-error: true | |
| uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 | |
| env: | |
| SLACK_COLOR: ${{ job.status }} | |
| SLACK_MESSAGE: "Instance teardown (regression-benchmarks) finished with status: ${{ job.status }}. (${{ env.ACTION_RUN_URL }})" |