bench-all-runtimes.yml

name: Bench all runtimes

on:
  # schedule:
    # - cron: '0 1 * * 0' # weekly on Sunday night 01:00 UTC
  workflow_dispatch:
    inputs:
      draft:
        type: boolean
        default: false
        description: "Whether to create a draft PR"

permissions: # allow the action to create a PR
  contents: write
  issues: write
  pull-requests: write
  actions: read

jobs:
  preflight:
    uses: ./.github/workflows/reusable-preflight.yml

  runtime-matrix:
    runs-on: ubuntu-latest
    needs: [preflight]
    timeout-minutes: 30
    outputs:
      runtime: ${{ steps.runtime.outputs.runtime }}
      branch: ${{ steps.branch.outputs.branch }}
      date: ${{ steps.branch.outputs.date }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    name: Extract runtimes from matrix
    steps:
      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          ref: master

      - name: Extract runtimes
        id: runtime
        run: |
          RUNTIMES=$(jq '[.[] | select(.package != null)]' .github/workflows/runtimes-matrix.json)

          RUNTIMES=$(echo $RUNTIMES | jq -c .)
          echo "runtime=$RUNTIMES"
          echo "runtime=$RUNTIMES" >> $GITHUB_OUTPUT

      - name: Create branch
        id: branch
        run: |
          DATE=$(date +'%Y-%m-%d-%s')
          BRANCH="update-weights-weekly-$DATE"
          # Fixes "detected dubious ownership" error in the ci
          git config --global --add safe.directory $GITHUB_WORKSPACE

          git checkout -b $BRANCH
          git push --set-upstream origin $BRANCH

          echo "date=$DATE" >> $GITHUB_OUTPUT
          echo "branch=$BRANCH" >> $GITHUB_OUTPUT

  run-frame-omni-bencher:
    needs: [preflight, runtime-matrix]
    runs-on: ${{ needs.preflight.outputs.RUNNER_WEIGHTS }}
    # 24 hours per runtime. 
    # Max it takes 14hr for westend to recalculate, but due to limited runners,
    # sometimes it can take longer.
    timeout-minutes: 1440
    strategy:
      fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures
      matrix:
        runtime: ${{ fromJSON(needs.runtime-matrix.outputs.runtime) }}
    container:
      image: ${{ needs.preflight.outputs.IMAGE }}
    env:
      PACKAGE_NAME: ${{ matrix.runtime.package }}
      FLAGS: ${{ matrix.runtime.bench_flags }}
      RUST_LOG: "frame_omni_bencher=info,polkadot_sdk_frame=info"
    steps:
    
      - name: Checkout
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          fetch-depth: 0
          ref: ${{ needs.runtime-matrix.outputs.branch }} # checkout always from the initially created branch to avoid conflicts

      - name: script
        id: required
        run: |
          git --version
          # Fixes "detected dubious ownership" error in the ci
          git config --global --add safe.directory $GITHUB_WORKSPACE
          git remote -v
          python3 -m pip install -r .github/scripts/generate-prdoc.requirements.txt
          python3 .github/scripts/cmd/cmd.py bench --runtime ${{ matrix.runtime.name }}
          git add .
          git status

          if [ -f /tmp/cmd/command_output.log ]; then
            CMD_OUTPUT=$(cat /tmp/cmd/command_output.log)
            # export to summary to display in the PR
            echo "$CMD_OUTPUT" >> $GITHUB_STEP_SUMMARY
            # should be multiline, otherwise it captures the first line only
            echo 'cmd_output<<EOF' >> $GITHUB_OUTPUT
            echo "$CMD_OUTPUT" >> $GITHUB_OUTPUT
            echo 'EOF' >> $GITHUB_OUTPUT
          fi

          # Create patch that includes both modifications and new files
          git add -A
          git diff --staged > diff-${{ matrix.runtime.name }}.patch -U0
          git reset
      
      - name: Upload diff
        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
        with:
          name: diff-${{ matrix.runtime.name }}
          path: diff-${{ matrix.runtime.name }}.patch

  apply-diff-commit:
    runs-on: ubuntu-latest
    needs: [runtime-matrix, run-frame-omni-bencher]
    steps:
      - name: Checkout
        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
        with:
          fetch-depth: 0
          ref: ${{ needs.runtime-matrix.outputs.branch }}

      - name: Download all artifacts
        uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
        with:
          path: patches      

      # needs to be able to trigger CI
      - uses: actions/create-github-app-token@29824e69f54612133e76f7eaac726eef6c875baf # v2.2.1
        id: generate_token
        with:
          app-id: ${{ secrets.CMD_BOT_APP_ID }}
          private-key: ${{ secrets.CMD_BOT_APP_KEY }}

      - name: Apply diff and create PR
        env:
          GH_TOKEN: ${{ steps.generate_token.outputs.token }}
          BRANCH: ${{ needs.runtime-matrix.outputs.branch }}
          DATE: ${{ needs.runtime-matrix.outputs.date }}
        run: |
          git --version          
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"

          git status
          
          # Apply all patches
          for file in patches/diff-*/diff-*.patch; do
            if [ -f "$file" ] && [ -s "$file" ]; then
              echo "Applying $file"
              # using --3way and --ours for conflicts resolution. Requires git 2.47+
              git apply "$file" --unidiff-zero --allow-empty --3way --ours || echo "Failed to apply $file"
            else
              echo "Skipping empty or non-existent patch file: $file"
            fi
          done

          rm -rf patches

          # Get release tags from 1 and 3 months ago
          ONE_MONTH_AGO=$(date -d "1 month ago" +%Y-%m-%d)
          THREE_MONTHS_AGO=$(date -d "3 months ago" +%Y-%m-%d)
          
          # Get tags with their dates
          ONE_MONTH_INFO=$(git for-each-ref --sort=-creatordate --format '%(refname:short)|%(creatordate:iso-strict-local)' 'refs/tags/polkadot-v*' | awk -v date="$ONE_MONTH_AGO" -F'|' '$2 <= date {print $0; exit}')
          THREE_MONTHS_INFO=$(git for-each-ref --sort=-creatordate --format '%(refname:short)|%(creatordate:iso-strict-local)' 'refs/tags/polkadot-v*' | awk -v date="$THREE_MONTHS_AGO" -F'|' '$2 <= date {print $0; exit}')
          
          # Split into tag and date
          ONE_MONTH_TAG=$(echo "$ONE_MONTH_INFO" | cut -d'|' -f1)
          ONE_MONTH_DATE=$(echo "$ONE_MONTH_INFO" | cut -d'|' -f2 | cut -d'T' -f1)
          THREE_MONTHS_TAG=$(echo "$THREE_MONTHS_INFO" | cut -d'|' -f1)
          THREE_MONTHS_DATE=$(echo "$THREE_MONTHS_INFO" | cut -d'|' -f2 | cut -d'T' -f1)

          # Base URL for Subweight comparisons
          BASE_URL="https://weights.tasty.limo/compare?repo=polkadot-sdk&threshold=5&path_pattern=.%2F**%2Fweights%2F**%2F*.rs%2C.%2F**%2Fweights.rs&method=asymptotic&ignore_errors=true&unit=time"
          
          # Generate comparison links
          MASTER_LINK="${BASE_URL}&old=master&new=${BRANCH}"
          ONE_MONTH_LINK="${BASE_URL}&old=${ONE_MONTH_TAG}&new=${BRANCH}"
          THREE_MONTHS_LINK="${BASE_URL}&old=${THREE_MONTHS_TAG}&new=${BRANCH}"
          
          # Create PR body with all links in a temporary file
          cat > /tmp/pr_body.md << EOF
          Auto-update of all weights for ${DATE}.

          Subweight results:
          - [now vs master](${MASTER_LINK})
          - [now vs ${ONE_MONTH_TAG} (${ONE_MONTH_DATE})](${ONE_MONTH_LINK})
          - [now vs ${THREE_MONTHS_TAG} (${THREE_MONTHS_DATE})](${THREE_MONTHS_LINK})
          EOF
          
          git add .
          git commit -m "Update all weights weekly for $DATE"
          git push --set-upstream origin "$BRANCH"

          MAYBE_DRAFT=${{ inputs.draft && '--draft' || '' }}
          
          PR_TITLE="Auto-update of all weights for $DATE"
          gh pr create \
            --title "$PR_TITLE" \
            --head "$BRANCH" \
            --base "master" \
            --reviewer paritytech/ci \
            --reviewer paritytech/release-engineering \
            $MAYBE_DRAFT \
            --label "R0-no-crate-publish-required" \
            --body "$(cat /tmp/pr_body.md)"