benchmark.yml

name: Benchmark

on:
  pull_request_target:
    branches:
      - master
      - develop
    types:
      - opened
      - synchronize
      - reopened
  workflow_dispatch:
    inputs:
      clickhouse:
        description: "ClickHouse version"
        required: true
        default: "latest"
      java:
        description: "Java version"
        required: true
        default: "8"
      baseline:
        description: "Baseline to compare"
        required: true
        default: "0.2.6"
      driver:
        description: "Driver version"
        required: true
        default: "0.3.2-SNAPSHOT"
      options:
        description: "Benchmark options"
        required: true
        default: "-prof gc 'Query|Insertion'"
      pr:
        description: "Pull request#"
        required: false

jobs:
  benchmark-pull-request:
    runs-on: ubuntu-latest
    name: Benchmark pull request
    if: github.event_name == 'pull_request_target'
    steps:
      - name: Check out repository
        uses: actions/checkout@v2
      - name: Set up JDK
        uses: actions/setup-java@v1
        with:
          java-version: 8
      - name: Extract version
        id: version
        run: |
          echo ::set-output name=value::$(grep '<revision>' pom.xml | sed -e 's|.*>\(.*\)<.*|\1|')
      - name: Update version and group id
        run: |
          find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ steps.version.outputs.value }}|g' \
            -e 's|^\(    <version>\).*\(</version>\)$|\1${{ steps.version.outputs.value }}\2|' \
            -e 's|${parent.groupId}|com.clickhouse|g' '{}' \;
          find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
      - name: Build project
        run: |
          mvn --batch-mode --update-snapshots -q -DskipTests install
          cd clickhouse-benchmark
          java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf text \
            -p client=clickhouse-http-jdbc1 -p client=clickhouse-grpc-jdbc -p type=object Basic
          echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
          cat jmh-result.text >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          cd -
      - name: Comment PR
        uses: actions/github-script@v3
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const report = process.env.BENCHMARK_REPORT;
            const { issue: { number: issue_number }, repo: { owner, repo } } = context;
            if (report) {
              github.issues.createComment({issue_number, owner, repo, body: '```\n' + report + '\n```'});
            }

  benchmark-on-demand:
    runs-on: ubuntu-latest
    name: Benchmark on demand
    if: github.event_name == 'workflow_dispatch'
    steps:
      - name: Check out repository
        uses: actions/checkout@v2
      - name: Check out PR
        run: |
          git fetch --no-tags --prune --progress --no-recurse-submodules --depth=1 \
            origin pull/${{ github.event.inputs.pr }}/merge:merged-pr && git checkout merged-pr
        if: github.event.inputs.pr != ''
      - name: Set up JDK ${{ github.event.inputs.java }}
        uses: actions/setup-java@v1
        with:
          java-version: ${{ github.event.inputs.java }}
        continue-on-error: true
      - name: Update version and group id
        run: |
          find . -type f -name "pom.xml" -exec sed -i -e 's|${revision}|${{ github.event.inputs.driver }}|g' \
            -e 's|^\(    <version>\).*\(</version>\)$|\1${{ github.event.inputs.driver }}\2|' \
            -e 's|${parent.groupId}|com.clickhouse|g' -e 's|${project.parent.groupId}|com.clickhouse|g' '{}' \;
          find . -type f -name "simplelogger.*" -exec rm -fv '{}' \;
        continue-on-error: true
      - name: Install driver as needed
        run: mvn --batch-mode --update-snapshots -q -DskipTests install
        if: endsWith(github.event.inputs.driver, '-SNAPSHOT')
        continue-on-error: true
      - name: Build project
        run: |
          mvn --batch-mode --update-snapshots -DskipTests -pl clickhouse-benchmark -am package
          cd clickhouse-benchmark
          java -DclickhouseVersion="21.8" -jar target/benchmarks.jar -rf json ${{ github.event.inputs.options }} > output.txt
          echo "BENCHMARK_REPORT<<EOF" >> $GITHUB_ENV
          tail -n +$(grep -n '^REMEMBER:' output.txt | tail -1 | awk -F: '{print $1+6}') output.txt | head -n -2 | grep -v ':·' >> $GITHUB_ENV
          echo "EOF" >> $GITHUB_ENV
          cd -
        id: benchmark
        continue-on-error: true
      - name: Record benchmark results
        run: |
          mv -fv clickhouse-benchmark/jmh-result.json \
            clickhouse-benchmark/docs/results/v${{ github.event.inputs.driver }}.json
        if: github.event.inputs.pr == ''
        continue-on-error: true
      - name: Record pull request benchmark results
        run: |
          mv -fv clickhouse-benchmark/jmh-result.json \
            clickhouse-benchmark/docs/results/pull-request_${{github.event.inputs.pr}}.json
        if: github.event.inputs.pr != ''
        continue-on-error: true
      - name: Commit benchmark result
        uses: zhicwu/add-and-commit@v7
        with:
          add: 'clickhouse-benchmark/docs/results/*.json'
          author_name: zhicwu
          author_email: 4270380+zhicwu@users.noreply.github.com
          branch: develop
          message: 'Record benchmark results'
          push: true
        continue-on-error: true
      - name: Get commit hash
        run: |
          echo ::set-output name=hash::$(git log --pretty=format:'%H' -n 1)
        id: commit
        continue-on-error: true
      - name: Comment PR
        uses: actions/github-script@v3
        if: github.event.inputs.pr != ''
        env:
          PR_NO: ${{ github.event.inputs.pr }}
          COMMIT_HASH: ${{ steps.commit.outputs.hash }}
          COMPARE_TO: ${{ github.event.inputs.baseline }}
          CLICKHOUSE_VRESION: ${{ github.event.inputs.clickhouse }}
          DRIVER_VRESION: ${{ github.event.inputs.driver }}
          JAVA_VERSION: ${{ github.event.inputs.java }}
          PREV_STEP_RESULT: '${{ steps.benchmark.outcome }}'
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          script: |
            const issue_number = process.env.PR_NO;
            const { repo: { owner, repo }  } = context;
            const result = process.env.PREV_STEP_RESULT;
            const compareUrl = `http://jmh.morethan.io/?sources=https://raw.githubusercontent.com/${owner}/${repo}/develop/clickhouse-benchmark/docs/results/v${process.env.COMPARE_TO}.json,https://raw.githubusercontent.com/${owner}/${repo}/${process.env.COMMIT_HASH || 'develop'}/clickhouse-benchmark/docs/results/${issue_number ? 'pull-request_' + issue_number : 'v' + process.env.DRIVER_VERSION}.json`;
            // const benchmarkUrl = `http://jmh.morethan.io/?source=https://raw.githubusercontent.com/${owner}/${repo}/${process.env.COMMIT_HASH || 'develop'}/clickhouse-benchmark/docs/results/${issue_number ? 'pull-request_' + issue_number : 'v' + process.env.DRIVER_VERSION}.json`;
            const buildUrl = `https://github.com/${owner}/${repo}/actions/runs/${context.runId}`;
            const flag = result === 'success'
              ? ':green_circle:'
              : (result === 'failure' ? ':red_circle:' : ':yellow_circle:');
            let msg = `${flag} [benchmark](${compareUrl}) using JDK [${process.env.JAVA_VERSION}] and ClickHouse [${process.env.CLICKHOUSE_VRESION}]: [${result}](${buildUrl})`;
            if (result === 'success') {
              msg = msg + '<details>\n<summary>Expand to see details...</summary>\n\n```\n'
                + process.env.BENCHMARK_REPORT || '' + '\n```\n</details>';
            }
            github.issues.createComment({ issue_number, owner, repo, body: msg });