bench: compare Krippendorff alpha with reference package

.github/workflows/benchmark-competitive.yml

@@ -0,0 +1,152 @@
+name: Competitive benchmark
+
+on:
+  pull_request:
+    paths:
+      - 'src/**'
+      - 'pyproject.toml'
+      - '.github/workflows/benchmark-competitive.yml'
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  krippendorff-alpha:
+    name: TurkishEvalKit vs krippendorff
+    runs-on: ubuntu-24.04
+    timeout-minutes: 15
+    steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
+
+      - name: Install benchmark environment
+        shell: bash
+        run: |
+          set -euo pipefail
+          python3 -m venv /tmp/tek-bench-venv
+          /tmp/tek-bench-venv/bin/pip install --quiet --upgrade pip
+          /tmp/tek-bench-venv/bin/pip install --quiet -e . 'krippendorff==0.8.2'
+
+      - name: Run correctness-gated algorithm benchmark
+        shell: bash
+        run: |
+          set -euo pipefail
+          mkdir -p benchmark-results
+          cat > /tmp/benchmark_reliability.py <<'PY'
+          import json
+          import platform
+          import statistics
+          import time
+
+          import krippendorff
+          import numpy as np
+          from turkishevalkit.reliability import _krippendorff_alpha
+
+          UNIT_COUNT = 10_000
+          WARMUPS = 3
+          RUNS = 20
+
+          # Deterministic 3-rater nominal dataset with controlled disagreement.
+          units = []
+          for i in range(UNIT_COUNT):
+              base = i % 3
+              row = [base, base, base]
+              if i % 5 == 0:
+                  row[2] = (base + 1) % 3
+              if i % 17 == 0:
+                  row[1] = (base + 2) % 3
+              units.append(row)
+
+          external_matrix = np.asarray(units, dtype=float).T
+
+          def tek():
+              estimate = _krippendorff_alpha(units, scale='nominal')
+              assert estimate.applicable and estimate.value is not None
+              return estimate.value
+
+          def reference():
+              return float(krippendorff.alpha(
+                  reliability_data=external_matrix,
+                  level_of_measurement='nominal',
+              ))
+
+          tek_value = tek()
+          reference_value = reference()
+          if round(reference_value, 4) != tek_value:
+              raise SystemExit(
+                  f'correctness mismatch: TurkishEvalKit={tek_value}, reference={reference_value}'
+              )
+
+          for _ in range(WARMUPS):
+              tek(); reference()
+
+          tek_times = []
+          reference_times = []
+          # Counterbalanced order to reduce systematic first/second-call effects.
+          for i in range(RUNS):
+              order = (('tek', tek), ('reference', reference)) if i % 2 == 0 else (('reference', reference), ('tek', tek))
+              for name, fn in order:
+                  start = time.perf_counter_ns()
+                  fn()
+                  elapsed_ms = (time.perf_counter_ns() - start) / 1_000_000
+                  (tek_times if name == 'tek' else reference_times).append(elapsed_ms)
+
+          result = {
+              'workload': 'Krippendorff nominal alpha on 10,000 units x 3 raters',
+              'correctness': {
+                  'TurkishEvalKit_rounded_4dp': tek_value,
+                  'krippendorff_0.8.2_raw': reference_value,
+              },
+              'runs': RUNS,
+              'warmups': WARMUPS,
+              'TurkishEvalKit_ms': {
+                  'median': statistics.median(tek_times),
+                  'mean': statistics.mean(tek_times),
+                  'stdev': statistics.stdev(tek_times),
+                  'min': min(tek_times),
+                  'max': max(tek_times),
+              },
+              'krippendorff_0.8.2_ms': {
+                  'median': statistics.median(reference_times),
+                  'mean': statistics.mean(reference_times),
+                  'stdev': statistics.stdev(reference_times),
+                  'min': min(reference_times),
+                  'max': max(reference_times),
+              },
+              'python': platform.python_version(),
+              'platform': platform.platform(),
+          }
+          ratio = result['TurkishEvalKit_ms']['median'] / result['krippendorff_0.8.2_ms']['median']
+          result['median_ratio_TurkishEvalKit_over_reference'] = ratio
+          print(json.dumps(result, indent=2, sort_keys=True))
+          with open('benchmark-results/krippendorff-alpha.json', 'w') as f:
+              json.dump(result, f, indent=2, sort_keys=True)
+          PY
+          /tmp/tek-bench-venv/bin/python /tmp/benchmark_reliability.py | tee benchmark-results/krippendorff-alpha.txt
+          {
+            echo "runner=ubuntu-24.04"
+            echo "kernel=$(uname -srmo)"
+            echo "cpu=$(lscpu | awk -F: '/Model name/{gsub(/^[ \t]+/,\"\",$2); print $2; exit}')"
+            echo "turkishevalkit=$(/tmp/tek-bench-venv/bin/python -c 'import importlib.metadata; print(importlib.metadata.version(\"turkishevalkit\"))')"
+            echo "krippendorff=$(/tmp/tek-bench-venv/bin/python -c 'import importlib.metadata; print(importlib.metadata.version(\"krippendorff\"))')"
+          } > benchmark-results/environment.txt
+
+      - name: Publish summary
+        if: always()
+        run: |
+          if [[ -f benchmark-results/krippendorff-alpha.txt ]]; then
+            echo '## Krippendorff nominal alpha' >> "$GITHUB_STEP_SUMMARY"
+            echo '```json' >> "$GITHUB_STEP_SUMMARY"
+            cat benchmark-results/krippendorff-alpha.txt >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Upload raw evidence
+        if: always()
+        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
+        with:
+          name: turkishevalkit-competitive-benchmark-${{ github.sha }}
+          path: benchmark-results/
+          if-no-files-found: error
+          retention-days: 90