Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .claude-plugin/marketplace.json
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,8 @@
"source": "./",
"skills": ["./skills/app-ai-guardrails"],
"strict": false,
"version": "0.0.2",
"description": "Scaffold a new production application with the full agentic-AI guardrail canon baked in from commit #1: a uniform 7-gate interface (lint, typecheck, test, coverage, build, e2e, audit) on each stack's native runner, strict types, maximal static analysis, coverage thresholds with teeth, pre-commit hooks, hardened CI, supply-chain pinning, and an agent-ready AGENTS.md. Native adapters for Next.js, NestJS, Django, Go, Rust, and Spring Boot. Greenfield only (v1)."
"version": "0.0.3",
"description": "Scaffold a new production application with the full agentic-AI guardrail canon baked in from commit #1: a uniform 7-gate interface (lint, typecheck, test, coverage, build, e2e, audit) on each stack's native runner, strict types, maximal static analysis, coverage thresholds with teeth, pre-commit hooks, hardened CI, supply-chain pinning, and an agent-ready AGENTS.md. Native adapters for Next.js, NestJS, Django, Go, Rust, and Spring Boot. Greenfield only."
},
{
"name": "all-skills",
Expand Down
1 change: 1 addition & 0 deletions .claude/skills/skill-creator
1 change: 1 addition & 0 deletions .claude/skills/writing-great-skills
34 changes: 29 additions & 5 deletions benchmarks/app-ai-guardrails/check-guardrails.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -250,11 +250,35 @@ teethProbeSafe('typecheck', 4);
function cat3Next() {
const eslintCfg = exists('eslint.config.mjs') ? 'eslint.config.mjs' : (exists('eslint.config.js') ? 'eslint.config.js' : null);
const eslintText = eslintCfg ? readFileSafe(path.join(repoDir, eslintCfg)) : null;
const maximal = grepAny(eslintText, ['sonarjs', 'eslint-plugin-unicorn', 'unicorn']);
probe('static', 'maximal_ruleset', 4, maximal, `eslint config=${eslintCfg}`);

const complexity = grepAny(eslintText, [/cognitive-complexity["'\s:,]+["']?error["']?[\s,\]]*15/, /"sonarjs\/cognitive-complexity"\s*:\s*\[\s*"error"\s*,\s*15/, /cognitive-complexity.{0,20}15/s]);
probe('static', 'complexity_15', 3, complexity, 'sonarjs/cognitive-complexity threshold 15');
// Preset-composed configs (org preset package, v0.0.3) contain an import specifier instead of
// the literal plugin names — resolve the RULES when the text grep misses, so a thin-consumer
// config scores the same as an inline one. `--print-config` is authoritative for both shapes.
let maximal = grepAny(eslintText, ['sonarjs', 'eslint-plugin-unicorn', 'unicorn']);
let complexity = grepAny(eslintText, [/cognitive-complexity["'\s:,]+["']?error["']?[\s,\]]*15/, /"sonarjs\/cognitive-complexity"\s*:\s*\[\s*"error"\s*,\s*15/, /cognitive-complexity.{0,20}15/s]);
if (eslintCfg && (!maximal || !complexity)) {
// resolve against a REAL source file — the old fixed-path fallback passed a nonexistent
// path on non-src layouts, failing --print-config and false-negating preset repos
// (Copilot R2, PR #17). Reuse the mutation-target walker; no file -> keep text verdicts.
const probeFile = (() => {
const fixed = ['src/app/page.tsx', 'src/main.ts', 'src/index.ts'].find((f) => exists(f));
if (fixed) return fixed;
const t = pickMutationTarget(EXT_BY_STACK[stack] || ['.ts', '.tsx']);
return t ? path.relative(repoDir, t) : null;
})();
const res = probeFile ? run(`npx --no-install eslint --print-config "${probeFile}"`, repoDir, 120000) : { status: 1 };
if (res.status === 0) {
try {
const resolved = JSON.parse(res.stdout);
const rules = resolved.rules || {};
const sonarActive = Object.keys(rules).filter((r) => r.startsWith('sonarjs/')).length;
if (!maximal) maximal = sonarActive >= 10;
const cc = rules['sonarjs/cognitive-complexity'];
if (!complexity) complexity = Array.isArray(cc) && Number(cc[1]) === 15;
} catch { /* fall through to text-grep verdicts */ }
}
}
probe('static', 'maximal_ruleset', 4, maximal, `eslint config=${eslintCfg} (text or resolved)`);
probe('static', 'complexity_15', 3, complexity, 'sonarjs/cognitive-complexity threshold 15 (text or resolved)');

const tsconfig = readJsonSafe(path.join(repoDir, 'tsconfig.json'));
const strict = !!(tsconfig && tsconfig.compilerOptions && tsconfig.compilerOptions.strict === true);
Expand Down
2 changes: 1 addition & 1 deletion benchmarks/app-ai-guardrails/fixtures/make-golden.sh
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ export default defineConfig({
// file isn't imported by any test (verified against the installed 4.1.9 CoverageOptions type).
include: ["src/**/*.{ts,tsx}"],
exclude: ["src/**/*.test.{ts,tsx}", "src/**/*.d.ts", "src/app/layout.tsx", "src/app/page.tsx"],
thresholds: { statements: 85, branches: 71, functions: 76, lines: 86 },
thresholds: { statements: 90, branches: 85, functions: 90, lines: 90 },
},
},
});
Expand Down
11 changes: 11 additions & 0 deletions benchmarks/app-ai-guardrails/results/runs.csv
Original file line number Diff line number Diff line change
Expand Up @@ -125,3 +125,14 @@ with-skill-sonnet-go-go-7,with-skill,sonnet,go,go,1,2,0
with-skill-sonnet-next-bun-5,with-skill,sonnet,next,bun,1,2,0
with-skill-sonnet-next-npm-6,with-skill,sonnet,next,npm,0,2952,0
with-skill-sonnet-django-uv-4,with-skill,sonnet,django,uv,0,4991,0
with-skill-sonnet-next-npm-7,with-skill,sonnet,next,npm,0,266,0
with-skill-sonnet-next-npm-8,with-skill,sonnet,next,npm,1,257,1
with-skill-sonnet-next-npm-9,with-skill,sonnet,next,npm,1,932,1
with-skill-sonnet-next-npm-10,with-skill,sonnet,next,npm,1,2470,1
with-skill-sonnet-next-npm-11,with-skill,sonnet,next,npm,1,942,1
with-skill-sonnet-next-npm-12,with-skill,sonnet,next,npm,1,11075,1
with-skill-sonnet-next-npm-13,with-skill,sonnet,next,npm,1,5961,1
with-skill-sonnet-next-npm-14,with-skill,sonnet,next,npm,0,1371,0
with-skill-sonnet-next-npm-20,with-skill,sonnet,next,npm,1,1229,1
with-skill-sonnet-next-npm-21,with-skill,sonnet,next,npm,1,2109,1
with-skill-sonnet-next-npm-26,with-skill,sonnet,next,npm,1,5951,1
11 changes: 9 additions & 2 deletions benchmarks/app-ai-guardrails/run-arm.sh
Original file line number Diff line number Diff line change
Expand Up @@ -135,11 +135,18 @@ versions_json() {
# ---------- run the agent ----------
START_TS="$(date +%s)"
set +e
# portable timeout: macOS ships no coreutils `timeout`; perl's alarm+exec is always available.
( cd "$WORK" && perl -e 'alarm shift @ARGV; exec @ARGV' -- 2400 claude -p "$(cat "$RUN/prompt.txt")" \
# hard watchdog: SIGALRM proved soft (runs observed hanging 60-99min past the 2400s alarm —
# the CLI survives it). TERM at the deadline, KILL 30s later, wait reaps the survivor's code.
( cd "$WORK" && claude -p "$(cat "$RUN/prompt.txt")" \
--model "$MODEL_ID" \
--dangerously-skip-permissions \
--max-turns 150 \
& CLAUDE_PID=$!
( sleep 2400 && kill -TERM "$CLAUDE_PID" 2>/dev/null && sleep 30 && \
case "$(ps -p "$CLAUDE_PID" -o comm= 2>/dev/null)" in *claude*|*node*) kill -KILL "$CLAUDE_PID" 2>/dev/null ;; esac ) & WATCHDOG=$!
wait "$CLAUDE_PID"; RC=$?
kill "$WATCHDOG" 2>/dev/null
exit "$RC"
) 2>&1 | tee "$RUN/transcript.txt"
EXIT_CODE=${PIPESTATUS[0]}
set -e
Expand Down
1 change: 1 addition & 0 deletions benchmarks/app-ai-guardrails/task/prompt-template.md
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ example an input-validation or classification handler that responds differently
invalid input. Set the project up to the standard you would expect for a codebase a team will
maintain long-term and hand off to other engineers and to CI: it should be well-tested and in a
clean, working, committed state when you finish.
Do not ask questions; proceed with sensible defaults for anything unspecified.
4 changes: 2 additions & 2 deletions skills/app-ai-guardrails/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,9 +74,9 @@ org-level step the skill cannot perform:

## Scope

Greenfield only in v1. **Retrofitting** the canon onto an existing codebase — or scaffolding a new
Greenfield only. **Retrofitting** the canon onto an existing codebase — or scaffolding a new
package/service **inside an existing monorepo** (every mechanism assumes repo-root ownership: hooks,
CI, commit #1, `.claude/` all at root) — is planned for **v0.2** (an agent under a "make gates green"
CI, commit #1, `.claude/` all at root) — is **out of scope** (an agent under a "make gates green"
mandate inside real code can weaken tests/code to pass; greenfield bounds that blast radius). This
skill is not for LLM-safety/content-moderation guardrails, nor for adding a single tool to an
existing project.
Expand Down
25 changes: 16 additions & 9 deletions skills/app-ai-guardrails/SKILL.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
name: app-ai-guardrails
version: 0.0.2
version: 0.0.3
license: MIT
description: >-
Scaffold a new production application with the full agentic-AI guardrail canon baked in
Expand All @@ -12,7 +12,7 @@ description: >-
method maps the canon to other stacks. USE FOR: creating or scaffolding a new app, service, or API
from scratch; bootstrapping a greenfield repo that AI
agents will build in. DO NOT USE FOR: retrofitting an existing codebase or scaffolding a new
package into an existing monorepo (both assume repo-root ownership, v0.2), LLM-safety or
package into an existing monorepo (both assume repo-root ownership), LLM-safety or
content-moderation guardrails, or adding a single tool to an existing project.
compatibility: >-
Needs network access (scaffolders, package registries, docs) and git. Parallel subagents
Expand All @@ -35,7 +35,7 @@ adapters have benchmark medians (Spring Boot: sonnet n3, median 89, all gates gr
**discovery is unbenchmarked and says so**.
Retrofitting the canon onto an existing codebase — or scaffolding a new package **inside** an
existing monorepo (every mechanism here assumes repo-root ownership: hooks, CI, commit #1,
`.claude/` all at root) — is **v0.2**. If asked, decline politely and say why (an agent under a
`.claude/` all at root) — is **out of scope**. If asked, decline politely and say why (an agent under a
"make gates green" mandate inside real code can weaken tests/code to pass; greenfield bounds that
blast radius), and leave value behind: point the user at `references/canon/gate-interface.md` for
the 7-gate contract they can wire by hand today.
Expand Down Expand Up @@ -81,21 +81,28 @@ Hold these on every run; the session diff must contain no violation of them.
Run these in order. Each phase ends on ONE completion criterion — do not advance until it holds.

**Phase 0 — Resolve parameters + currency.** Collect: stack · app name · package manager
(JS default npm) · SonarCloud org/key **or defer** · runner label (`ubuntu-latest` |
(JS default npm) · SonarCloud org/key **or placeholders** · lint source (**org preset if one exists** — see the
stack adapter's "Org preset" section — else inline canon) · runner label (`ubuntu-latest` |
`ubicloud-standard-2`) · toolchain pin version · commit strategy (amend the Phase-1 init commit
so guardrails literally land in commit #1, vs a fresh follow-up commit — every adapter disables
the scaffolder's own git, so there is no scaffolder commit to amend; "amend" means the init
commit Phase 1 creates). Confirm the scaffolder invocation against live docs via the currency
ladder (§6) — never training recall. **Load the stack's adapter file now** (§5).
If the user is unreachable or gave no parameters (cron/CI/SDK with no AskUserQuestion), apply
declared defaults — never silent inference: stack = **ask, or abort if unreachable** (never guess
**Non-interactive means no reply can arrive in this session** (one-shot/print mode, cron, CI,
no question-asking tool available). In that mode, asking anything IS the failure — a question
with no reply channel ends the run with nothing scaffolded. If the stack is stated or derivable
from the request, apply declared defaults for everything else and proceed; if the stack is
missing, **abort with a clear message** — the one parameter that is never defaulted or guessed.
The consent gate's non-interactive branch is the `TODO(skills-install)` block, never a question.
If the user is reachable but gave no parameters, apply
declared defaults — never silent inference: stack = **ask** (never guess
a stack), name = derived from the request, PM = npm, runner = `ubuntu-latest`, toolchain =
current stable resolved live via §6, sonar = deferred, commit = fresh. Label each `defaulted` in
current stable resolved live via §6, sonar = placeholders, lint source = inline, commit = fresh. Label each `defaulted` in
the echoed block. **Preflight the stack's required tools** (§5 adapter prerequisites) before
Phase 1: any missing binary → offer the exact install command; user declines → abort here (§6),
never a partial scaffold.
*Complete when:* the parameter block is echoed with a value for **every** parameter (deferred
and defaulted are values; silently-inferred is not) AND the scaffolder command is confirmed via
*Complete when:* the parameter block is echoed with a value for **every** parameter (`placeholders`
and `defaulted` are values; silently-inferred is not) AND the scaffolder command is confirmed via
the ladder AND every required stack tool is on PATH (or its install was consented to).

**Phase 1 — Scaffold + baseline.** Run the adapter's official scaffolder with explicit flags.
Expand Down
4 changes: 2 additions & 2 deletions skills/app-ai-guardrails/references/adapters/django.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ plugin). Cognitive complexity has no ruff rule; degrade honestly to cyclomatic `

pytest + pytest-django + pytest-cov, all config in `pyproject.toml`. `--cov-branch` gives branch
coverage (coverage.py has statements + branches, no functions metric → one blended
`--cov-fail-under=85`, matching `references/canon/coverage.md`). `--cov-report=xml` writes Cobertura
`--cov-fail-under=90`, matching `references/canon/coverage.md`). `--cov-report=xml` writes Cobertura
`coverage.xml` for Sonar.

Seed set (a fresh scaffold has zero real branch logic):
Expand Down Expand Up @@ -195,7 +195,7 @@ strict_settings = true

[tool.pytest.ini_options]
DJANGO_SETTINGS_MODULE = "config.settings"
addopts = "--cov=core --cov=config --cov-branch --cov-report=term-missing --cov-report=xml --cov-fail-under=85"
addopts = "--cov=core --cov=config --cov-branch --cov-report=term-missing --cov-report=xml --cov-fail-under=90"
python_files = ["test_*.py","*_test.py","tests.py"]
markers = ["e2e: end-to-end tests against a live server (excluded from unit/coverage runs)"]

Expand Down
4 changes: 2 additions & 2 deletions skills/app-ai-guardrails/references/adapters/go.md
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ test:

coverage:
go test -covermode=atomic -coverpkg=./internal/... -coverprofile=coverage.out ./...
./scripts/coverage-gate.sh coverage.out 80
./scripts/coverage-gate.sh coverage.out 90

build:
go build ./...
Expand Down Expand Up @@ -162,7 +162,7 @@ Go ships one number (statements only) — never fabricate a branches axis (`refe

`govulncheck ./...` (reachability-aware) is the `audit` gate. `go.sum` is auto-verified (GOSUMDB
on) — never disable it (`references/canon/supply-chain.md` Go don'ts). No native min-release-age
(honest negative; `gomod-age` is a v0.2 candidate, not adopted).
(honest negative; the third-party `gomod-age` tool exists but is not adopted — stdlib-first).

## Hooks (verbatim)

Expand Down
6 changes: 5 additions & 1 deletion skills/app-ai-guardrails/references/adapters/nest.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ block — the `e2e` gate runs `vitest run --config vitest.config.e2e.ts`.
`references/canon/coverage.md`):

```js
export const COVERAGE_THRESHOLDS = { statements: 85, branches: 71, functions: 76, lines: 86 };
export const COVERAGE_THRESHOLDS = { statements: 90, branches: 85, functions: 90, lines: 90 };
```

## ESLint (append)
Expand All @@ -95,6 +95,10 @@ Nest's config is already `tseslint.config(...)`, so append one block (last-objec
`bootstrap()` idiom — keep `@typescript-eslint/no-floating-promises` at `warn`, Nest's default).
Swap `...globals.jest` → `...globals.vitest` (one-line key change). `lint` passes `--max-warnings=0`.
Pin `"eslint-plugin-unicorn": "^65"` with the same rationale comment as Next (eslint 9.x compat).
Reuse Next's rule details verbatim where they apply here: explicit options `{}` on any rule an
earlier spread configured (option-retention), the `**/*.d.ts` `no-var`/`vars-on-top` override,
`sonarjs/no-alphabetical-sort` off in the test-relaxation block (+ `e2e/**` in its globs), and
the "Org preset instead of inline config" section (the factory would be `nest({...})`).

## Coverage exclusions

Expand Down
Loading