coprocessor-benchmarks-cpu #143
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Run fhevm coprocessor benchmarks on a CPU instance and return parsed results to Slab. | |
| name: coprocessor-benchmarks-cpu | |
| permissions: {} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| benchmarks: | |
| description: "Benchmark set" | |
| required: true | |
| type: choice | |
| options: | |
| - "erc20" | |
| - "dex" | |
| - "synthetics" | |
| - "all" | |
| batch_size: | |
| description: "Batch sizes (# FHE operations executed per batch)" | |
| required: true | |
| type: string | |
| default: "5000" | |
| scheduling_policy: | |
| description: "Scheduling policy" | |
| required: true | |
| type: choice | |
| options: | |
| - "MAX_PARALLELISM" | |
| - "MAX_LOCALITY" | |
| - "LOOP" | |
| - "FINE_GRAIN" | |
| benchmark_type: | |
| description: "Benchmark type" | |
| required: false | |
| type: choice | |
| options: | |
| - "ALL" | |
| - "THROUGHPUT" | |
| - "LATENCY" | |
| default: "ALL" | |
| env: | |
| CARGO_TERM_COLOR: always | |
| RESULTS_FILENAME: parsed_benchmark_results_${{ github.sha }}.json | |
| ACTION_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| RUST_BACKTRACE: "full" | |
| RUST_MIN_STACK: "8388608" | |
| CHECKOUT_TOKEN: ${{ secrets.REPO_CHECKOUT_TOKEN || secrets.GITHUB_TOKEN }} | |
| jobs: | |
| setup-instance: | |
| name: coprocessor-benchmarks-cpu/setup-instance | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' # Required to checkout repository code | |
| outputs: | |
| runner-name: ${{ steps.start-remote-instance.outputs.label }} | |
| steps: | |
| - name: Start remote instance | |
| id: start-remote-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: start | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| backend: aws | |
| profile: bench | |
| benchmarks-cpu: | |
| name: coprocessor-benchmarks-cpu/benchmarks-cpu (bpr) | |
| needs: setup-instance | |
| runs-on: ${{ needs.setup-instance.outputs.runner-name }} | |
| continue-on-error: true | |
| timeout-minutes: 720 # 12 hours | |
| permissions: | |
| contents: 'read' # Required to checkout repository code | |
| packages: 'read' # Required to read GitHub packages/container registry | |
| strategy: | |
| fail-fast: false | |
| # explicit include-based build matrix, of known valid options | |
| matrix: | |
| include: | |
| - os: ubuntu-22.04 | |
| cuda: "12.2" | |
| gcc: 11 | |
| steps: | |
| - name: Install git LFS | |
| run: | | |
| # Wait for apt locks to be released (e.g., unattended-upgrades may hold the lock on fresh instances) | |
| while sudo fuser /var/lib/dpkg/lock-frontend >/dev/null 2>&1; do | |
| echo "Waiting for apt lock to be released..." | |
| sleep 5 | |
| done | |
| sudo apt-get update | |
| sudo apt-get install -y git-lfs | |
| git lfs install | |
| - name: Checkout fhevm-backend | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 | |
| with: | |
| persist-credentials: 'false' | |
| fetch-depth: 0 | |
| lfs: true | |
| - name: Checkout LFS objects | |
| run: git lfs checkout | |
| - name: Get benchmark details | |
| run: | | |
| { | |
| echo "BENCH_DATE=$(date --iso-8601=seconds)"; | |
| echo "COMMIT_DATE=$(git --no-pager show -s --format=%cd --date=iso8601-strict "${GITHUB_SHA}")"; | |
| echo "COMMIT_HASH=$(git describe --tags --dirty)"; | |
| } >> "${GITHUB_ENV}" | |
| - name: Install rust | |
| uses: dtolnay/rust-toolchain@a54c7afa936fefeb4456b2dd8068152669aa8203 | |
| with: | |
| toolchain: nightly | |
| - name: Install cargo dependencies | |
| run: | | |
| sudo systemctl stop docker | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get remove -y docker docker-engine docker.io containerd runc | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get purge -y docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-compose | |
| sudo rm -rf /etc/bash_completion.d/docker /usr/local/bin/docker-compose /etc/bash_completion.d/docker-compose | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get update | |
| DEBIAN_FRONTEND=noninteractive sudo apt-get install -y protobuf-compiler cmake \ | |
| pkg-config libssl-dev \ | |
| libclang-dev docker-compose-v2 \ | |
| docker.io acl | |
| sudo systemctl start docker | |
| cargo +stable install sqlx-cli --version 0.7.2 --no-default-features --features postgres --locked | |
| - name: Install foundry | |
| uses: foundry-rs/foundry-toolchain@de808b1eea699e761c404bda44ba8f21aba30b2c | |
| - name: Cache cargo | |
| uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 | |
| with: | |
| path: | | |
| ~/.cargo/registry | |
| ~/.cargo/git | |
| target | |
| key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} | |
| restore-keys: ${{ runner.os }}-cargo- | |
| - name: Login to GitHub Container Registry | |
| uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| registry: ghcr.io | |
| username: ${{ github.actor }} | |
| password: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Login to Chainguard Registry | |
| uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 | |
| with: | |
| registry: cgr.dev | |
| username: ${{ secrets.CGR_USERNAME }} | |
| password: ${{ secrets.CGR_PASSWORD }} | |
| - name: Init database | |
| run: make init_db | |
| working-directory: coprocessor/fhevm-engine/tfhe-worker | |
| - name: Use Node.js | |
| uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 | |
| with: | |
| node-version: 20.x | |
| - name: Start localstack | |
| run: | | |
| docker run --rm -d -p 4566:4566 --name localstack localstack/localstack:latest | |
| - name: Run benchmarks on CPU | |
| run: | | |
| DATABASE_URL=postgresql://postgres:postgres@localhost:5432/coprocessor TXN_SENDER_TEST_GLOBAL_LOCALSTACK=1 BENCHMARK_BATCH_SIZE="${BATCH_SIZE}" BENCHMARK_TYPE="${BENCHMARK_TYPE}" FHEVM_DF_SCHEDULE="${SCHEDULING_POLICY}" make -e "benchmark_${BENCHMARKS}_cpu" | |
| working-directory: coprocessor/fhevm-engine/tfhe-worker | |
| env: | |
| BENCHMARK_TYPE: ${{ inputs.benchmark_type }} | |
| BATCH_SIZE: ${{ inputs.batch_size }} | |
| SCHEDULING_POLICY: ${{ inputs.scheduling_policy }} | |
| BENCHMARKS: ${{ inputs.benchmarks }} | |
| - name: Parse results | |
| run: | | |
| python3 ./ci/benchmark_parser.py coprocessor/fhevm-engine/target/criterion "${RESULTS_FILENAME}" \ | |
| --database coprocessor \ | |
| --hardware "hpc7a.96xlarge" \ | |
| --backend cpu \ | |
| --project-version "${COMMIT_HASH}" \ | |
| --branch "${GH_REF_NAME}" \ | |
| --commit-date "${COMMIT_DATE}" \ | |
| --bench-date "${BENCH_DATE}" \ | |
| --walk-subdirs \ | |
| --name-suffix "operation_batch_size_${BATCH_SIZE}-schedule_${SCHEDULING_POLICY}" | |
| env: | |
| RESULTS_FILENAME: ${{ env.RESULTS_FILENAME }} | |
| COMMIT_HASH: ${{ env.COMMIT_HASH }} | |
| GH_REF_NAME: ${{ github.ref_name }} | |
| COMMIT_DATE: ${{ env.COMMIT_DATE }} | |
| BENCH_DATE: ${{ env.BENCH_DATE }} | |
| BATCH_SIZE: ${{ inputs.batch_size }} | |
| SCHEDULING_POLICY: ${{ inputs.scheduling_policy }} | |
| - name: Upload parsed results artifact | |
| uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 | |
| with: | |
| name: ${{ github.sha }}_${{ inputs.benchmarks }}_cpu | |
| path: ${{ env.RESULTS_FILENAME }} | |
| - name: Checkout Slab repo | |
| uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 | |
| with: | |
| repository: zama-ai/slab | |
| path: slab | |
| persist-credentials: 'false' | |
| token: ${{ secrets.REPO_CHECKOUT_TOKEN }} | |
| - name: Send data to Slab | |
| shell: bash | |
| run: | | |
| python3 slab/scripts/data_sender.py "${RESULTS_FILENAME}" "${JOB_SECRET}" \ | |
| --slab-url "${SLAB_URL}" | |
| env: | |
| JOB_SECRET: ${{ secrets.JOB_SECRET }} | |
| RESULTS_FILENAME: ${{ env.RESULTS_FILENAME }} | |
| SLAB_URL: ${{ secrets.SLAB_URL }} | |
| teardown-instance: | |
| name: coprocessor-benchmarks-cpu/teardown | |
| if: ${{ always() && needs.setup-instance.result == 'success' }} | |
| needs: [ setup-instance, benchmarks-cpu ] | |
| runs-on: ubuntu-latest | |
| permissions: | |
| contents: 'read' # Required to checkout repository code | |
| steps: | |
| - name: Stop remote instance | |
| id: stop-instance | |
| uses: zama-ai/slab-github-runner@79939325c3c429837c10d6041e4fd8589d328bac | |
| with: | |
| mode: stop | |
| github-token: ${{ secrets.SLAB_ACTION_TOKEN }} | |
| slab-url: ${{ secrets.SLAB_BASE_URL }} | |
| job-secret: ${{ secrets.JOB_SECRET }} | |
| label: ${{ needs.setup-instance.outputs.runner-name }} |