Skip to content
Quality Gate

deep-bug-investigator: run synthesis on opus, pin track subagents to sonnet #360

Sign in to view logs

deep-bug-investigator: run synthesis on opus, pin track subagents to sonnet

deep-bug-investigator: run synthesis on opus, pin track subagents to sonnet #360

Workflow file for this run

.github/workflows/lint.yml at 9bfbb0b
name: Quality Gate
on:
push:
branches: [main]
# Validate stacked PRs too; GitHub evaluates this workflow from the PR base.
pull_request:
jobs:
lint:
name: Markdown + YAML + JSON
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "20"
- name: Install dependencies
run: npm ci
- name: Lint Markdown
run: npm run lint
- name: Lint YAML
uses: ibiqlik/action-yamllint@2576378a8e339169678f9939646ee3ee325e845c # v3
with:
file_or_dir: .
config_file: .yamllint.yml
strict: false
- name: Validate JSON
run: |
for file in $(find . -name "*.json" -not -path "./node_modules/*" -not -path "./.claude/*"); do
echo "Validating $file"
python3 -m json.tool "$file" > /dev/null
done
python:
name: Python Lint + Security
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install dependencies
run: pip install pyyaml ruff pip-audit
- name: Ruff lint (Python)
run: ruff check lab/ scripts/ --select E,F,W --ignore E501,E402
- name: Ruff format check
run: ruff format --check lab/ scripts/ || true
- name: Pip audit (security)
run: pip-audit -r requirements.txt || true
shell:
name: Shell Lint
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: ShellCheck
run: |
sudo apt-get install -y shellcheck
find plugins/elixir-phoenix/hooks/scripts -name "*.sh" -exec shellcheck {} + || true
find lab/autoresearch/scripts -name "*.sh" -exec shellcheck {} + || true
shellcheck lab/eval/run_eval.sh || true
security:
name: Dependency Audit
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Node.js
uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7
with:
node-version: "20"
- name: Install dependencies
run: npm ci
- name: npm audit
run: npm audit --audit-level=high
test:
name: Python Tests
runs-on: ubuntu-latest
needs: python
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install dependencies
run: pip install -r requirements.txt
- name: Run pytest
run: python3 -m pytest lab/eval/tests/ scripts/tests/ -v --tb=short
- name: Check Amp generated skills
run: python3 -m scripts.build_amp_skills --check
- name: Check Codex generated skills
run: python3 -m scripts.build_codex_skills --check
- name: Check Pi generated skills
run: python3 -m scripts.build_pi_skills --check
- name: Check OpenCode generated skills
run: python3 -m scripts.build_opencode_skills --check
- name: Check generated target golden snapshots
run: python3 -m scripts.generated_target_snapshots --check
eval:
name: Skill & Agent Eval
runs-on: ubuntu-latest
needs: [lint, python, test]
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
- name: Setup Python
uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
with:
python-version: "3.12"
- name: Install PyYAML
run: pip install pyyaml
- name: Score all skills (8 dimensions)
run: |
python3 -m lab.eval.scorer --all 2>/dev/null | python3 -c "
import json, sys
d = json.load(sys.stdin)
perfect = sum(1 for v in d.values() if v['composite'] >= 0.999)
avg = sum(v['composite'] for v in d.values()) / len(d)
fixable_failures = {}
for name, data in d.items():
if data['composite'] < 0.95:
all_behavioral = all(
dim == 'behavioral'
for dim, dd in data['dimensions'].items()
for a in dd['assertions'] if not a['passed']
)
if not all_behavioral:
fixable_failures[name] = round(data['composite'], 3)
print(f'{len(d)} skills | {perfect} perfect | avg {avg:.3f}')
if fixable_failures:
print(f'FIXABLE FAILURES: {fixable_failures}')
sys.exit(1)
print('All skills pass (behavioral edge cases expected)')
"
- name: Score all agents (5 dimensions)
run: |
python3 -m lab.eval.agent_scorer --all 2>&1 | tee /dev/stderr | tail -1 | grep -q "perfect"