diff --git a/.github/workflows/benchmark-competitive.yml b/.github/workflows/benchmark-competitive.yml new file mode 100644 index 0000000..892b6f8 --- /dev/null +++ b/.github/workflows/benchmark-competitive.yml @@ -0,0 +1,152 @@ +name: Competitive benchmark + +on: + pull_request: + paths: + - 'src/**' + - 'pyproject.toml' + - '.github/workflows/benchmark-competitive.yml' + workflow_dispatch: + +permissions: + contents: read + +jobs: + krippendorff-alpha: + name: TurkishEvalKit vs krippendorff + runs-on: ubuntu-24.04 + timeout-minutes: 15 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + + - name: Install benchmark environment + shell: bash + run: | + set -euo pipefail + python3 -m venv /tmp/tek-bench-venv + /tmp/tek-bench-venv/bin/pip install --quiet --upgrade pip + /tmp/tek-bench-venv/bin/pip install --quiet -e . 'krippendorff==0.8.2' + + - name: Run correctness-gated algorithm benchmark + shell: bash + run: | + set -euo pipefail + mkdir -p benchmark-results + cat > /tmp/benchmark_reliability.py <<'PY' + import json + import platform + import statistics + import time + + import krippendorff + import numpy as np + from turkishevalkit.reliability import _krippendorff_alpha + + UNIT_COUNT = 10_000 + WARMUPS = 3 + RUNS = 20 + + # Deterministic 3-rater nominal dataset with controlled disagreement. + units = [] + for i in range(UNIT_COUNT): + base = i % 3 + row = [base, base, base] + if i % 5 == 0: + row[2] = (base + 1) % 3 + if i % 17 == 0: + row[1] = (base + 2) % 3 + units.append(row) + + external_matrix = np.asarray(units, dtype=float).T + + def tek(): + estimate = _krippendorff_alpha(units, scale='nominal') + assert estimate.applicable and estimate.value is not None + return estimate.value + + def reference(): + return float(krippendorff.alpha( + reliability_data=external_matrix, + level_of_measurement='nominal', + )) + + tek_value = tek() + reference_value = reference() + if round(reference_value, 4) != tek_value: + raise SystemExit( + f'correctness mismatch: TurkishEvalKit={tek_value}, reference={reference_value}' + ) + + for _ in range(WARMUPS): + tek(); reference() + + tek_times = [] + reference_times = [] + # Counterbalanced order to reduce systematic first/second-call effects. + for i in range(RUNS): + order = (('tek', tek), ('reference', reference)) if i % 2 == 0 else (('reference', reference), ('tek', tek)) + for name, fn in order: + start = time.perf_counter_ns() + fn() + elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000 + (tek_times if name == 'tek' else reference_times).append(elapsed_ms) + + result = { + 'workload': 'Krippendorff nominal alpha on 10,000 units x 3 raters', + 'correctness': { + 'TurkishEvalKit_rounded_4dp': tek_value, + 'krippendorff_0.8.2_raw': reference_value, + }, + 'runs': RUNS, + 'warmups': WARMUPS, + 'TurkishEvalKit_ms': { + 'median': statistics.median(tek_times), + 'mean': statistics.mean(tek_times), + 'stdev': statistics.stdev(tek_times), + 'min': min(tek_times), + 'max': max(tek_times), + }, + 'krippendorff_0.8.2_ms': { + 'median': statistics.median(reference_times), + 'mean': statistics.mean(reference_times), + 'stdev': statistics.stdev(reference_times), + 'min': min(reference_times), + 'max': max(reference_times), + }, + 'python': platform.python_version(), + 'platform': platform.platform(), + } + ratio = result['TurkishEvalKit_ms']['median'] / result['krippendorff_0.8.2_ms']['median'] + result['median_ratio_TurkishEvalKit_over_reference'] = ratio + print(json.dumps(result, indent=2, sort_keys=True)) + with open('benchmark-results/krippendorff-alpha.json', 'w') as f: + json.dump(result, f, indent=2, sort_keys=True) + PY + /tmp/tek-bench-venv/bin/python /tmp/benchmark_reliability.py | tee benchmark-results/krippendorff-alpha.txt + { + echo "runner=ubuntu-24.04" + echo "kernel=$(uname -srmo)" + echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,\"\",$2); print $2; exit}')" + echo "turkishevalkit=$(/tmp/tek-bench-venv/bin/python -c 'import importlib.metadata; print(importlib.metadata.version(\"turkishevalkit\"))')" + echo "krippendorff=$(/tmp/tek-bench-venv/bin/python -c 'import importlib.metadata; print(importlib.metadata.version(\"krippendorff\"))')" + } > benchmark-results/environment.txt + + - name: Publish summary + if: always() + run: | + if [[ -f benchmark-results/krippendorff-alpha.txt ]]; then + echo '## Krippendorff nominal alpha' >> "$GITHUB_STEP_SUMMARY" + echo '```json' >> "$GITHUB_STEP_SUMMARY" + cat benchmark-results/krippendorff-alpha.txt >> "$GITHUB_STEP_SUMMARY" + echo '```' >> "$GITHUB_STEP_SUMMARY" + fi + + - name: Upload raw evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: turkishevalkit-competitive-benchmark-${{ github.sha }} + path: benchmark-results/ + if-no-files-found: error + retention-days: 90