release: v9.2.0 — analyzeOutgoingImpact, symbolExists, Cartographer 3.0.0 #298
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: NFR Tests | |
| on: | |
| pull_request: | |
| branches: [main] | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| permissions: | |
| contents: read | |
| pull-requests: write | |
| jobs: | |
| nfr-head: | |
| name: NFR (PR Head) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| - name: Set up Go | |
| uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 | |
| with: | |
| go-version-file: 'go.mod' | |
| cache: true | |
| - name: Download dependencies | |
| run: go mod download | |
| - name: Run NFR Tests | |
| run: | | |
| set +e | |
| go test -v -run TestNFRScenarios ./internal/mcp/... 2>&1 | tee nfr-output.txt | |
| echo "exit_code=${PIPESTATUS[0]}" >> $GITHUB_OUTPUT | |
| set -e | |
| # Always succeed - comparison job decides pass/fail | |
| exit 0 | |
| - name: Upload head results | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: nfr-head | |
| path: nfr-output.txt | |
| retention-days: 7 | |
| nfr-base: | |
| name: NFR (Base Branch) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 | |
| with: | |
| ref: ${{ github.event.pull_request.base.sha }} | |
| - name: Set up Go | |
| uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6 | |
| with: | |
| go-version-file: 'go.mod' | |
| cache: true | |
| - name: Download dependencies | |
| run: go mod download | |
| - name: Run NFR Tests | |
| run: | | |
| set +e | |
| go test -v -run TestNFRScenarios ./internal/mcp/... 2>&1 | tee nfr-output.txt | |
| set -e | |
| # Always succeed - comparison job decides pass/fail | |
| exit 0 | |
| - name: Upload base results | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: nfr-base | |
| path: nfr-output.txt | |
| retention-days: 7 | |
| nfr-compare: | |
| name: NFR Compare | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| needs: [nfr-head, nfr-base] | |
| if: always() | |
| steps: | |
| - name: Download head results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: nfr-head | |
| path: head/ | |
| - name: Download base results | |
| uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 | |
| with: | |
| name: nfr-base | |
| path: base/ | |
| - name: Compare NFR results | |
| id: compare | |
| run: | | |
| cat > compare_nfr.py << 'PYEOF' | |
| import re | |
| import sys | |
| def parse_nfr(path): | |
| """Parse NFR output into {tool_tier: {actual, timing}}""" | |
| results = {} | |
| pattern = r'(\w+)_(\w+): (\d+) bytes \(baseline: (\d+), max: (\d+)\) \[([^\]]+)\]' | |
| try: | |
| with open(path, "r") as f: | |
| content = f.read() | |
| for match in re.finditer(pattern, content): | |
| tool, tier, actual, baseline, max_allowed, timing = match.groups() | |
| key = f"{tool}_{tier}" | |
| results[key] = { | |
| "tool": tool, "tier": tier, | |
| "actual": int(actual), | |
| "static_baseline": int(baseline), | |
| "max": int(max_allowed), | |
| "timing": timing.strip() | |
| } | |
| except FileNotFoundError: | |
| pass | |
| return results | |
| def fmt_time(t): | |
| t = t.strip() | |
| if "ms" in t and "\u00b5s" not in t: | |
| return t | |
| if "\u00b5s" in t: | |
| val = float(t.replace("\u00b5s", "")) | |
| if val >= 1000: | |
| return f"{val/1000:.2f}ms" | |
| return f"{val:.0f}\u00b5s" | |
| return t | |
| head = parse_nfr("head/nfr-output.txt") | |
| base = parse_nfr("base/nfr-output.txt") | |
| if not head: | |
| print("No NFR metrics found in PR head output") | |
| sys.exit(0) | |
| rows = [] | |
| for key, h in sorted(head.items()): | |
| b = base.get(key) | |
| if b: | |
| # Dynamic comparison against base branch | |
| baseline_val = b["actual"] | |
| diff_pct = ((h["actual"] - baseline_val) / baseline_val) * 100 if baseline_val > 0 else 0 | |
| else: | |
| # New scenario not in base - compare against static baseline | |
| baseline_val = h["static_baseline"] | |
| diff_pct = ((h["actual"] - baseline_val) / baseline_val) * 100 if baseline_val > 0 else 0 | |
| budget_pct = (h["actual"] / h["max"]) * 100 if h["max"] > 0 else 0 | |
| if diff_pct >= 10: | |
| verdict = "FAIL" | |
| elif diff_pct >= 5: | |
| verdict = "WARN" | |
| elif diff_pct <= -5: | |
| verdict = "GOOD" | |
| else: | |
| verdict = "OK" | |
| rows.append({ | |
| "tool": h["tool"], "tier": h["tier"], | |
| "actual": h["actual"], | |
| "baseline": baseline_val, | |
| "base_in_branch": b is not None, | |
| "max": h["max"], | |
| "diff_pct": diff_pct, | |
| "budget_pct": budget_pct, | |
| "timing": fmt_time(h["timing"]), | |
| "verdict": verdict | |
| }) | |
| if rows: | |
| fails = [r for r in rows if r["verdict"] == "FAIL"] | |
| warns = [r for r in rows if r["verdict"] == "WARN"] | |
| improved = [r for r in rows if r["diff_pct"] < -0.05] | |
| unchanged = [r for r in rows if abs(r["diff_pct"]) <= 0.05] | |
| worsened = [r for r in rows if r["diff_pct"] > 0.05 and r["verdict"] not in ("FAIL", "WARN")] | |
| indicators = [] | |
| if fails: | |
| indicators.append(f"\u274c {len(fails)} failed") | |
| if warns: | |
| indicators.append(f"\u26a0\ufe0f {len(warns)} regressed") | |
| if improved: | |
| indicators.append(f"\u2b06\ufe0f {len(improved)} improved") | |
| if unchanged: | |
| indicators.append(f"\u2705 {len(unchanged)} unchanged") | |
| if worsened: | |
| indicators.append(f"\U0001f7e1 {len(worsened)} slightly worse") | |
| print(f"## NFR Tests {' \u00b7 '.join(indicators)}\n") | |
| print(f"Comparing PR against **{('${{ github.event.pull_request.base.ref }}' or 'base')}** branch (dynamic baseline).\n") | |
| if fails + warns: | |
| reg_val = f"{len(fails)} failed, {len(warns)} regressed" if fails else f"{len(warns)} regressed" | |
| print(f"**Regressions:** {reg_val}\n") | |
| else: | |
| print(f"**Regressions:** 0 \u2705\n") | |
| print("**Thresholds:** WARN \u2265 +5% \u2022 FAIL \u2265 +10%\n") | |
| # Diff block preview | |
| preview_bad = sorted([r for r in rows if r["diff_pct"] >= 5], key=lambda r: -r["diff_pct"])[:3] | |
| preview_good = sorted([r for r in rows if r["diff_pct"] < -2], key=lambda r: r["diff_pct"])[:3] | |
| if preview_bad or preview_good: | |
| all_preview = preview_bad + preview_good | |
| max_name = max(len(f"{r['tool']}/{r['tier']}") for r in all_preview) | |
| print("```diff") | |
| for r in preview_bad: | |
| name = f"{r['tool']}/{r['tier']}" | |
| print(f"- {r['verdict']:4} {name:<{max_name}} {r['diff_pct']:+6.1f}% {r['actual']:>9,}B {r['timing']}") | |
| for r in preview_good: | |
| name = f"{r['tool']}/{r['tier']}" | |
| print(f"+ SAVE {name:<{max_name}} {r['diff_pct']:+6.1f}% {r['actual']:>9,}B {r['timing']}") | |
| print("```\n") | |
| # Full breakdown | |
| print("<details><summary>All scenarios</summary>\n") | |
| print("| | Scenario | Change | Actual (B) | Base (B) | Time |") | |
| print("|:---:|---|---:|---:|---:|---:|") | |
| for r in sorted(rows, key=lambda x: -x["diff_pct"]): | |
| d = r["diff_pct"] | |
| if d > 2: | |
| icon = "\U0001f534" # red: significant regression | |
| elif d > 0.05: | |
| icon = "\U0001f7e1" # yellow: moderate regression | |
| elif d < -2: | |
| icon = "\U0001f7e2" # green: significant improvement | |
| elif d < -0.05: | |
| icon = "\U0001f535" # blue: moderate improvement | |
| else: | |
| icon = "\u26aa" # grey: unchanged | |
| src = "" if r["base_in_branch"] else " *" | |
| print(f"| {icon} | {r['tool']} / {r['tier']}{src} | {r['diff_pct']:+.1f}% | {r['actual']:,} | {r['baseline']:,} | {r['timing']} |") | |
| print("\n*\\* = new scenario, compared against static baseline*") | |
| print("\n</details>") | |
| # Set exit code | |
| if fails: | |
| sys.exit(1) | |
| else: | |
| print("No NFR metrics found in output") | |
| PYEOF | |
| if ! python3 compare_nfr.py > nfr-table.md 2>&1; then | |
| COMPARE_EXIT=$? | |
| cat nfr-table.md | |
| echo "has_failures=true" >> $GITHUB_OUTPUT | |
| else | |
| cat nfr-table.md | |
| echo "has_failures=false" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Job summary | |
| if: always() | |
| run: | | |
| if [ -f nfr-table.md ]; then | |
| cat nfr-table.md >> $GITHUB_STEP_SUMMARY | |
| fi | |
| - name: Comment on PR | |
| if: always() && github.event_name == 'pull_request' | |
| uses: actions/github-script@ed597411d8f924073f98dfc5c65a23a2325f34cd # v8 | |
| with: | |
| script: | | |
| const fs = require('fs'); | |
| let body = 'No NFR metrics available'; | |
| try { | |
| body = fs.readFileSync('nfr-table.md', 'utf8'); | |
| } catch (e) {} | |
| const { data: comments } = await github.rest.issues.listComments({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| }); | |
| const botComment = comments.find(c => | |
| c.user.type === 'Bot' && c.body.includes('NFR Tests') | |
| ); | |
| if (botComment) { | |
| await github.rest.issues.updateComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| comment_id: botComment.id, | |
| body: body | |
| }); | |
| } else { | |
| await github.rest.issues.createComment({ | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| issue_number: context.issue.number, | |
| body: body | |
| }); | |
| } | |
| - name: Upload NFR results | |
| if: always() | |
| uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 | |
| with: | |
| name: nfr-results | |
| path: | | |
| head/nfr-output.txt | |
| base/nfr-output.txt | |
| nfr-table.md | |
| retention-days: 7 | |
| - name: Fail on regressions | |
| if: always() && steps.compare.outputs.has_failures == 'true' | |
| run: exit 1 |