ci: verify Firewood setup #5
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Firewood Comparison Benchmark | ||
|
Check failure on line 1 in .github/workflows/firewood-ab-reexecution-bench.yml
|
||
| # | ||
| # Compares performance between two Firewood versions by running C-Chain re-execution benchmarks. | ||
| # Supports both tagged releases (uses pre-built binaries when available) and source builds. | ||
| # Each version runs with its specified AvalancheGo commit for compatibility. | ||
| name: Firewood AB Reexecution Benchmark | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| current-state-dir-src: | ||
| description: 'Contains the VM database and chain state needed as a starting point for re-execution' | ||
| type: choice | ||
| required: false | ||
| default: 'cchain-current-state-firewood-100' | ||
| options: | ||
| - cchain-current-state-firewood-100/ | ||
| - cchain-current-state-firewood-1m/ | ||
| - cchain-current-state-firewood-10m/ | ||
| - cchain-current-state-firewood-20m/ | ||
| - cchain-current-state-firewood-30m/ | ||
| - cchain-current-state-firewood-33m/ | ||
| - cchain-current-state-firewood-40m/ | ||
| - cchain-current-state-firewood-50m/ | ||
| runner: | ||
| description: 'Select self-hosted runner' | ||
| type: choice | ||
| required: true | ||
| options: | ||
| - avalanche-avalanchego-runner | ||
| - avalanche-avalanchego-runner-2ti | ||
| firewood-baseline-version: | ||
| description: 'Baseline Firewood version (tag like ffi/v0.0.12 or commit hash)' | ||
| required: true | ||
| avalanchego-baseline-version: | ||
| description: 'AvalancheGo version (tag like v1.13.0 or commit hash)' | ||
| required: true | ||
| default: 'master' | ||
| firewood-candidate-version: | ||
| description: 'Candidate Firewood version (tag or commit hash)' | ||
| required: true | ||
| avalanchego-candidate-version: | ||
| description: 'AvalancheGo version (tag like v1.13.0 or commit hash)' | ||
| required: true | ||
| start-block: | ||
| description: 'Start block for benchmark' | ||
| required: false | ||
| default: '101' | ||
| end-block: | ||
| description: 'End block for benchmark' | ||
| required: false | ||
| default: '200' | ||
| env: | ||
| RUST_VERSION: '1.88' | ||
| jobs: | ||
| baseline-benchmark: | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ghcr.io/actions/actions-runner:2.325.0 | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
| name: "Baseline: ${{ inputs.firewood-baseline-version }}" | ||
| steps: | ||
| - name: Install dependencies | ||
| shell: bash | ||
| run: | | ||
| if ! command -v xz &> /dev/null; then | ||
| sudo apt-get update | ||
| sudo apt-get install -y xz-utils | ||
| fi | ||
| - name: Checkout AvalancheGo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ avalanchego-baseline-version }} | ||
| - name: Install nix | ||
| uses: ./.github/actions/install-nix | ||
| - name: Setup Go | ||
| uses: ./.github/actions/setup-go-for-project | ||
| - name: Setup Firewood | ||
| uses: ./.github/actions/setup-firewood | ||
| with: | ||
| firewood-version: ${{ inputs.firewood-baseline-version }} | ||
| rust-version: ${{ env.RUST_VERSION }} | ||
| - name: Configure AvalancheGo to work with Firewood | ||
| run: | | ||
| # Configure go.mod to use local FFI | ||
| go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi | ||
| go mod download | ||
| # Stash changes for benchmark without this compare-results fails | ||
| git add go.mod go.sum || true | ||
| git stash push -m "temp stash for baseline benchmark" || echo "Nothing to stash" | ||
| - name: Run Baseline Benchmark | ||
| uses: ./.github/actions/c-chain-reexecution-benchmark | ||
| with: | ||
| config: firewood | ||
| current-state-dir-src: ${{ inputs.current-state-dir-src }} | ||
| start-block: ${{ inputs.start-block }} | ||
| end-block: ${{ inputs.end-block }} | ||
| prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} | ||
| prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} | ||
| push-github-action-benchmark: false | ||
| aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} | ||
| aws-region: us-east-2 | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| push-post-state: '' | ||
| runner_name: '${{ inputs.runner }}' | ||
| - name: Restore go.mod changes | ||
| shell: nix develop --command bash -x {0} | ||
| run: | | ||
| git stash pop || echo "Nothing to restore" | ||
| - name: Upload baseline results | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: baseline-benchmark-results | ||
| path: | | ||
| output.txt | ||
| benchmark-results/ | ||
| retention-days: 7 | ||
| candidate-benchmark: | ||
| runs-on: ${{ inputs.runner }} | ||
| container: | ||
| image: ghcr.io/actions/actions-runner:2.325.0 | ||
| permissions: | ||
| id-token: write | ||
| contents: write | ||
| name: "Candidate: ${{ inputs.firewood-candidate-version }}" | ||
| steps: | ||
| - name: Install dependencies | ||
| shell: bash | ||
| run: | | ||
| if ! command -v xz &> /dev/null; then | ||
| sudo apt-get update | ||
| sudo apt-get install -y xz-utils | ||
| fi | ||
| - name: Checkout AvalancheGo | ||
| uses: actions/checkout@v4 | ||
| with: | ||
| ref: ${{ inputs.avalanchego-candidate-version }} | ||
| - name: Install nix | ||
| uses: ./.github/actions/install-nix | ||
| - name: Setup Go | ||
| uses: ./.github/actions/setup-go-for-project | ||
| - name: Setup Firewood | ||
| uses: ./.github/actions/setup-firewood | ||
| with: | ||
| firewood-version: ${{ inputs.firewood-candidate-version }} | ||
| rust-version: ${{ env.RUST_VERSION }} | ||
| - name: Configure AvalancheGo to work with Firewood | ||
| shell: nix develop --command bash -x {0} | ||
| run: | | ||
| # Configure go.mod to use local FFI | ||
| go mod edit -replace github.com/ava-labs/firewood-go-ethhash/ffi=./ffi | ||
| go mod download | ||
| # Stash changes for benchmark | ||
| git add go.mod go.sum || true | ||
| git stash push -m "temp stash for candidate benchmark" || echo "Nothing to stash" | ||
| - name: Run Candidate Benchmark | ||
| uses: ./.github/actions/c-chain-reexecution-benchmark | ||
| with: | ||
| config: firewood | ||
| current-state-dir-src: ${{ inputs.current-state-dir-src }} | ||
| start-block: ${{ inputs.start-block }} | ||
| end-block: ${{ inputs.end-block }} | ||
| prometheus-username: ${{ secrets.PROMETHEUS_ID || '' }} | ||
| prometheus-password: ${{ secrets.PROMETHEUS_PASSWORD || '' }} | ||
| push-github-action-benchmark: false | ||
| aws-role: ${{ secrets.AWS_S3_READ_ONLY_ROLE }} | ||
| aws-region: us-east-2 | ||
| github-token: ${{ secrets.GITHUB_TOKEN }} | ||
| push-post-state: '' | ||
| runner_name: '${{ inputs.runner }}' | ||
| - name: Restore go.mod changes | ||
| shell: nix develop --command bash -x {0} | ||
| run: | | ||
| git stash pop || echo "Nothing to restore" | ||
| - name: Upload candidate results | ||
| uses: actions/upload-artifact@v4 | ||
| with: | ||
| name: candidate-benchmark-results | ||
| path: | | ||
| output.txt | ||
| benchmark-results/ | ||
| retention-days: 7 | ||
| compare-results: | ||
| needs: [baseline-benchmark, candidate-benchmark] | ||
| runs-on: ubuntu-latest | ||
| if: always() | ||
| steps: | ||
| - name: Download baseline results | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: baseline-benchmark-results | ||
| path: ./baseline-results | ||
| continue-on-error: true | ||
| - name: Download candidate results | ||
| uses: actions/download-artifact@v4 | ||
| with: | ||
| name: candidate-benchmark-results | ||
| path: ./candidate-results | ||
| continue-on-error: true | ||
| - name: Compare Results | ||
| shell: bash | ||
| run: | | ||
| # Function to extract mgas/s value from benchmark output | ||
| extract_mgas() { | ||
| local file="$1" | ||
| if [ -f "$file" ]; then | ||
| # Extract decimal number followed by " mgas/s" | ||
| grep -oE '[0-9]+(\.[0-9]+)? mgas/s' "$file" | head -1 | grep -oE '[0-9]+(\.[0-9]+)?' || echo "" | ||
| else | ||
| echo "" | ||
| fi | ||
| } | ||
| # Function to check if a number is valid | ||
| is_valid_number() { | ||
| [[ $1 =~ ^[0-9]+(\.[0-9]+)?$ ]] && [ -n "$1" ] | ||
| } | ||
| # Extract performance metrics | ||
| BASELINE_MGAS=$(extract_mgas "./baseline-results/output.txt") | ||
| CANDIDATE_MGAS=$(extract_mgas "./candidate-results/output.txt") | ||
| # Start building the summary | ||
| echo "# Firewood Performance Comparison" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Baseline:** \`${{ inputs.firewood-baseline-version }}\`" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Candidate:** \`${{ inputs.firewood-candidate-version }}\`" >> $GITHUB_STEP_SUMMARY | ||
| echo "**Blocks:** ${{ inputs.start-block }} - ${{ inputs.end-block }}" >> $GITHUB_STEP_SUMMARY | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| # Display results with better error handling | ||
| if is_valid_number "$BASELINE_MGAS"; then | ||
| echo "- **Baseline (\`${{ inputs.firewood-baseline-version }}\`):** ${BASELINE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY | ||
| BASELINE_SUCCESS=true | ||
| else | ||
| echo "- **Baseline (\`${{ inputs.firewood-baseline-version }}\`):** Failed to parse results" >> $GITHUB_STEP_SUMMARY | ||
| BASELINE_SUCCESS=false | ||
| fi | ||
| if is_valid_number "$CANDIDATE_MGAS"; then | ||
| echo "- **Candidate (\`${{ inputs.firewood-candidate-version }}\`):** ${CANDIDATE_MGAS} mgas/s" >> $GITHUB_STEP_SUMMARY | ||
| CANDIDATE_SUCCESS=true | ||
| else | ||
| echo "- **Candidate (\`${{ inputs.firewood-candidate-version }}\`):** Failed to parse results" >> $GITHUB_STEP_SUMMARY | ||
| CANDIDATE_SUCCESS=false | ||
| fi | ||
| # Calculate percentage change if both values are valid | ||
| if [ "$BASELINE_SUCCESS" = true ] && [ "$CANDIDATE_SUCCESS" = true ]; then | ||
| CHANGE=$(awk "BEGIN {printf \"%.2f\", ($CANDIDATE_MGAS - $BASELINE_MGAS) / $BASELINE_MGAS * 100}") | ||
| CHANGE_ABS=$(awk "BEGIN {printf \"%.2f\", ($CHANGE < 0) ? -$CHANGE : $CHANGE}") | ||
| # Determine performance indicator | ||
| if (( $(awk "BEGIN {print ($CHANGE > 5)}") )); then | ||
| PERF_INDICATOR="**Significant Improvement**" | ||
| elif (( $(awk "BEGIN {print ($CHANGE > 1)}") )); then | ||
| PERF_INDICATOR="**Improvement**" | ||
| elif (( $(awk "BEGIN {print ($CHANGE < -5)}") )); then | ||
| PERF_INDICATOR="**Significant Regression**" | ||
| elif (( $(awk "BEGIN {print ($CHANGE < -1)}") )); then | ||
| PERF_INDICATOR="**Regression**" | ||
| else | ||
| PERF_INDICATOR="**No significant change**" | ||
| fi | ||
| if (( $(awk "BEGIN {print ($CHANGE > 0)}") )); then | ||
| echo "- **Performance Change:** +${CHANGE}% ${PERF_INDICATOR}" >> $GITHUB_STEP_SUMMARY | ||
| elif (( $(awk "BEGIN {print ($CHANGE < 0)}") )); then | ||
| echo "- **Performance Change:** ${CHANGE}% ${PERF_INDICATOR}" >> $GITHUB_STEP_SUMMARY | ||
| else | ||
| echo "- **Performance Change:** No change ${PERF_INDICATOR}" >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "## Raw Results" >> $GITHUB_STEP_SUMMARY | ||
| echo "### Baseline (${{ inputs.firewood-baseline-version }})" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat ./baseline-results/output.txt >> $GITHUB_STEP_SUMMARY 2>/dev/null || echo "No baseline output found" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| echo "### Candidate (${{ inputs.firewood-candidate-version }})" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat ./candidate-results/output.txt >> $GITHUB_STEP_SUMMARY 2>/dev/null || echo "No candidate output found" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| else | ||
| echo "" >> $GITHUB_STEP_SUMMARY | ||
| echo "## Raw Results" >> $GITHUB_STEP_SUMMARY | ||
| if [ -f "./baseline-results/output.txt" ]; then | ||
| echo "### Baseline (${{ inputs.firewood-baseline-version }})" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat ./baseline-results/output.txt >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| if [ -f "./candidate-results/output.txt" ]; then | ||
| echo "### Candidate (${{ inputs.firewood-candidate-version }})" >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| cat ./candidate-results/output.txt >> $GITHUB_STEP_SUMMARY | ||
| echo '```' >> $GITHUB_STEP_SUMMARY | ||
| fi | ||
| fi | ||
