copium/.github/workflows/eval.yml at main · iKislay/copium · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
name: Evaluation Suite

on:
  schedule:
    - cron: '0 6 * * 1'  # Weekly on Monday 6am UTC
  workflow_dispatch:       # Manual trigger
  pull_request:
    paths:
      - 'copium/transforms/**'
      - 'copium/evals/**'
      - 'copium/compress.py'

jobs:
  # Fast smoke test on PRs touching compression code (~$0.05, ~2 min)
  smoke-test:
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Cache pip
        uses: actions/cache@v5
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-eval-${{ hashFiles('pyproject.toml') }}
          restore-keys: ${{ runner.os }}-pip-eval-

      # `pip install -e .` invokes maturin (declared in pyproject.toml's
      # build-system) which calls cargo to compile the Rust extension.
      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@1.96.0

      - name: Cache cargo registry + build
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target"

      - name: Install dependencies (builds Rust extension via maturin)
        run: |
          pip install -e ".[all]"
          python -c "from copium._core import SmartCrusher; print('copium._core OK:', SmartCrusher)"

      - name: Run CCR round-trip (zero cost)
        run: |
          python -c "
          from copium.evals.runners.compression_only import CompressionOnlyRunner
          runner = CompressionOnlyRunner()
          cases = runner.generate_ccr_test_cases(n=50)
          result = runner.evaluate_ccr_lossless(cases)
          print(f'CCR Round-trip: {result.passed_cases}/{result.total_cases} passed')
          assert result.passed, f'CCR failures: {result.errors}'
          "

      - name: Run tool schema compaction integrity eval (zero cost)
        run: |
          python -c "
          from copium.evals.runners.compression_only import CompressionOnlyRunner
          runner = CompressionOnlyRunner()
          result = runner.evaluate_tool_schema_compaction()
          print(f'Tool schema compaction: {result.passed_cases}/{result.total_cases} passed, {result.total_tokens_saved} annotation tokens stripped')
          assert result.passed, f'Schema compaction failures: {result.errors}'
          "

      # OPENAI_API_KEY is intentionally not set in the public OSS repo
      # (the secret list is empty). The CCR round-trip step above is the
      # mandatory gate; this step only runs when an operator has wired
      # OPENAI_API_KEY as a repo secret (e.g. on a downstream fork). When
      # missing, emit a loud GitHub `::warning::` annotation so the skip
      # is visible in the run summary — never a silent pass.
      - name: Run built-in tool output eval (skipped when OPENAI_API_KEY unset)
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
        run: |
          if [ -z "${OPENAI_API_KEY}" ]; then
            echo "::warning title=Smoke eval skipped::OPENAI_API_KEY is not configured for this repo; only the CCR round-trip gate ran. Wire the secret to enable the live OpenAI eval."
            exit 0
          fi
          python -m copium.evals quick -n 8 --provider openai --model gpt-4o-mini

  # Full Tier 1 suite, weekly or manual (~$3-5, ~30-45 min)
  weekly-suite:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    timeout-minutes: 90
    steps:
      - uses: actions/checkout@v6
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"

      - name: Cache pip
        uses: actions/cache@v5
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-eval-${{ hashFiles('pyproject.toml') }}
          restore-keys: ${{ runner.os }}-pip-eval-

      - name: Install Rust toolchain
        uses: dtolnay/rust-toolchain@1.96.0

      - name: Cache cargo registry + build
        uses: Swatinem/rust-cache@v2
        with:
          workspaces: ". -> target"

      - name: Install dependencies (builds Rust extension via maturin)
        run: |
          pip install -e ".[all]"
          python -c "from copium._core import SmartCrusher; print('copium._core OK')"
      - name: Run Tier 1 evaluation suite
        run: |
          if [ -z "${OPENAI_API_KEY}" ]; then
            echo "::warning title=Weekly eval skipped::OPENAI_API_KEY is not configured for this repo; skipping the live Tier 1 suite."
            mkdir -p eval_results
            printf '%s\n\n%s\n' \
              '# Weekly Evaluation Skipped' \
              'OPENAI_API_KEY is not configured for this repository, so the live Tier 1 evaluation suite was skipped.' \
              > eval_results/skipped.md
            exit 0
          fi
          python -m copium.evals suite --tier 1 --ci -o eval_results/
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
      - name: Upload results
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: eval-results-${{ github.run_number }}
          path: eval_results/