audit_rubric.yaml

# Audits rubric — the primary lens for auditing a target gym.
#
# Every audit is a QUALITY QUESTION the auditor asks: "does this gym have
# an enforcement mechanism for X?". The baseline_impl field points at the
# canonical implementation in gym-cua-anthropic — that's what a scanner
# looks for (by function name / docstring / regex) when checking a NEW gym.
#
# For each audit the audit produces:
#   - presence  : is a check for this audit implemented in the target?
#   - coverage  : rough fraction of tasks the check runs against
#   - lives_in  : file:function reference in the target
#   - status    : green / yellow / red / unknown

schema_version: 1
categories:
- id: placeholder_interpolation
  name: Placeholder & interpolation hygiene
  description: 'Guards against template tokens, placeholders, and unresolved refs

    surviving from authoring or hydration into the agent-visible surface.

    '
  audits:
  - id: no_surviving_placeholders
    name: No `LLM_MUST_FILL` placeholders survive hydration
    what: Every `LLM_MUST_FILL_THIS_PLACEHOLDER` token has been replaced by hydration.
    why: A surviving placeholder is a broken task — the agent sees a template string instead of real content,
      and grading fires on a literal `LLM_MUST_FILL…` value.
    baseline_impl: task-designer/stages/reward_linter.py::_check_surviving_placeholders
  - id: no_world_tokens_in_agent_fields
    name: No `$world${…}` template tokens in agent-visible fields
    what: Template tokens don't appear in prompts, wiki bodies, issue text, or any surface the agent reads.
    why: A leaked token means the agent sees raw `$world${repo_alpha}` instead of "alpha" — visible bug
      that also breaks grading.
    baseline_impl: task-designer/stages/reward_linter.py::_check_world_template_in_agent_fields
  - id: no_double_interpolation
    name: No double-interpolation patterns
    what: No `${$world${…}}`, no double braces, no nested substitution attempts.
    why: Always an authoring mistake — the inner substitution never fires and the outer receives a literal
      string.
    baseline_impl: task-designer/stages/reward_linter.py::_check_double_interpolation
- id: reward_shape_validity
  name: Reward declaration shape validity
  description: 'Structural checks on the reward YAML itself — recognized keys, correct

    operator shapes, kind ∈ registered set, no deprecated syntax.

    '
  audits:
  - id: only_known_top_level_keys
    name: Only recognized top-level keys on reward declarations
    what: Reward declarations don't carry keys the compiler doesn't know about.
    why: Unknown keys are silently dropped; the author's intent never reaches the harness.
    baseline_impl: task-designer/stages/reward_linter.py::_check_unknown_compiler_keys
  - id: every_reward_has_check_block
    name: Every reward has a `check:` block
    what: Every reward declares what it does via a `check:` sub-dict (v3 rewards).
    why: Rewards without checks are unscoreable.
    baseline_impl: task-designer/stages/evidence_linter.py::_check_reward_missing_check
  - id: v3_dispatch_shape_correct
    name: v3 rewards have the correct nested shape
    what: v3 rewards carry `check:` sub-dict + `evidence_strict:` flag in the correct positions.
    why: Wrong shape → v3 dispatch fails silently and the reward routes to legacy handling.
    baseline_impl: task-designer/stages/evidence_linter.py::_check_v3_dispatch_shapes
  - id: assertion_key_shape
    name: Assertion dicts use recognized keys only
    what: '`operator`, `path`, `expected`, `name` — no typos.'
    why: Typos silently drop the assertion; the reward compiles but doesn't test what the author intended.
    baseline_impl: task-designer/stages/reward_linter.py::_check_assertion_keys
  - id: numeric_match_needs_path
    name: '`NUMERIC_MATCH` operators carry a `path:` field'
    what: Numeric comparators must have a value path to compare against.
    why: Without `path`, the comparator passes vacuously.
    baseline_impl: task-designer/stages/reward_linter.py::_check_numeric_match_needs_path
  - id: no_deprecated_expect_on
    name: No deprecated `expect:` / `on:` shapes
    what: Rewards use the current shape, not legacy `expect:` / `on:` shorthand.
    why: Deprecated keys parse but no longer route to the intended compiler; behaviour silently changes.
    baseline_impl: task-designer/stages/expect_deprecated_linter.py::lint_expect_deprecated
  - id: reward_kind_in_registry
    name: Reward kind is in the registered closed set (runtime)
    what: Every reward `kind` matches a registered compiler.
    why: Unknown kinds raise at compile time — no silent fallback.
    baseline_impl: gym_github/reward_compiler/registry.py::@register
- id: reward_pairing_contradictions
  name: Reward naming, pairing & contradictions
  description: 'Guards against duplicated names, broken gates, inverted polarity, sibling

    collisions, and rewards that would grade contradictory outcomes.

    '
  audits:
  - id: no_duplicate_reward_names
    name: No duplicate reward names
    what: No two rewards share a `name:`.
    why: Duplicate names collide in scoring dispatch; one silently overwrites the other.
    baseline_impl: task-designer/stages/reward_linter.py::_check_duplicate_reward_names
    also_in:
    - task-designer/stages/task_contradictions_linter.py::_check_duplicate_names
  - id: gated_by_resolves
    name: '`gated_by:` references resolve'
    what: Every gate points at a real reward name.
    why: A stale gate silently disables the gated reward.
    baseline_impl: task-designer/stages/reward_linter.py::_check_gated_by
  - id: reward_polarity_correct
    name: '`_absent` / `_not_*` reward polarity flags are correct'
    what: Negative-polarity rewards have their polarity flag set consistently with the name.
    why: Polarity inversion silently flips scoring — pass becomes fail.
    baseline_impl: task-designer/stages/reward_linter.py::_check_name_assertion_polarity
  - id: guard_has_positive_twin
    name: Every `_absent` guard has a positive-content twin
    what: For every "must NOT contain X" there's a "must contain Y" somewhere.
    why: A guard alone gates on nothing — the author probably forgot the positive.
    baseline_impl: task-designer/stages/reward_linter.py::_check_missing_positive_content
  - id: no_indistinguishable_siblings
    name: Sibling rewards on the same entity aren't near-duplicates
    what: Two rewards targeting the same entity differ in a meaningful way.
    why: Near-duplicate siblings score the same action twice; agent gets double credit.
    baseline_impl: task-designer/stages/reward_linter.py::_check_indistinguishable_siblings
  - id: no_direct_conflicts_between_rewards
    name: No cross-reward conflicts on the same entity
    what: No two rewards assert directly-conflicting states (e.g. "closed" AND "not closed") on the same
      entity.
    why: Both fire; agent gets double credit for a single action.
    baseline_impl: task-designer/stages/reward_pair_linter.py::_check_direct_conflicts
  - id: no_contains_excludes_contradiction
    name: 'No `body_contains: X` alongside `body_excludes: X` on the same reward'
    what: A single reward doesn't set contradictory text-match conditions.
    why: Impossible predicate — the reward is unsolvable, silently.
    baseline_impl: task-designer/stages/task_contradictions_linter.py::_check_contains_excludes_contradictions
  - id: no_author_filter_contradiction
    name: Author filters don't contradict
    what: A reward doesn't say both `author == agent` and `author != agent`.
    why: Unsolvable predicate, silently.
    baseline_impl: task-designer/stages/task_contradictions_linter.py::_check_agent_author_contradictions
  - id: no_empty_on_list
    name: No reward has an empty `on:` list
    what: Rewards target at least one entity.
    why: Empty targets = the reward grades nothing but still counts toward the total, distorting the score.
    baseline_impl: task-designer/stages/task_contradictions_linter.py::_check_empty_on
- id: scope_and_guard_typing
  name: Scope inference & guard typing
  description: 'Rewards inherit an inferred scope; the inference must match author

    intent, and guards must reference correctly-typed positives.

    '
  audits:
  - id: accounted_by_type_correct
    name: '`accounted_by:` on guard rewards references correctly-typed positives'
    what: Guards name positive-twin rewards of the matching type (e.g. a `no_extra_agent_issues` guard's
      `accounted_by` must name an `agent_created_issue` reward).
    why: Wrong type → the guard over-fires (blocks legitimate positives) or under-fires (lets distractors
      through).
    baseline_impl: task-designer/stages/accounted_by_type_linter.py::lint_accounted_by_types
- id: structural_quality_signals
  name: Structural quality signals
  description: 'Whole-file / whole-corpus signals — thin content, artificial-looking

    files, degenerate structure.

    '
  audits:
  - id: source_files_not_thin
    name: Seeded source files aren't stub-thin placeholders
    what: Files carry realistic content, not `# TODO` one-liners.
    why: Thin files look artificial to the agent and are trivially distinguishable from real code.
    baseline_impl: task-designer/stages/reward_linter.py::_check_thin_source_files
meta_audits:
- id: suppression_discipline
  name: Suppression discipline
  description: 'When a lint rule is intentionally suppressed, what enforcement exists

    to ensure it''s reviewed, rationalized, and doesn''t age into

    load-bearing magic?

    '
  audits:
  - id: lint_ignore_parser_exists
    name: A `# lint-ignore:` directive parser exists
    what: 'Comments of the form `# lint-ignore: CATEGORY` are extracted and applied to findings before
      returning them.'
    why: Without a real parser, suppressions become copy-paste convention with no code contract.
    baseline_impl: task-designer/stages/reward_linter.py::_parse_lint_ignores
  - id: scoped_suppression_supported
    name: 'Scoped suppression syntax honored (`# lint-ignore: X @ path`)'
    what: The parser respects `@ path` scoping so a suppression can target one location, not the whole
      file.
    why: Global suppression is too broad; scoped is the discipline for one-off intentional defects.
    baseline_impl: task-designer/stages/reward_linter.py::_finding_ignored
- id: linter_infrastructure
  name: Linter infrastructure & wiring
  description: 'The plumbing that ensures the linters actually run, produce findings in

    a consistent shape, and block ship at the right severity.

    '
  audits:
  - id: linter_binary_wired
    name: A linter binary is wired into the pipeline
    what: The linter is invoked automatically at hydration + pre-ship, not just runnable ad-hoc.
    why: A linter that must be manually invoked won't be — regressions ship.
    baseline_impl: task-designer/run.py::_lint_fix_loop
  - id: standalone_cli
    name: A standalone CLI exposes the linter for CI / manual use
    what: '`scripts/lint_tasks.py` (or equivalent) runs the linter over a given YAML with the same rules.'
    why: Ad-hoc verification, PR checks, and manual review all need the same entry point.
    baseline_impl: scripts/lint_tasks.py
  - id: severity_classification
    name: Findings are classified by severity
    what: Every finding carries `CRITICAL / HIGH / MEDIUM / LOW`.
    why: Blocking policy depends on severity; without it, everything is either critical or ignored.
    baseline_impl: task-designer/stages/reward_linter.py::LintFinding
  - id: blocking_categories_defined
    name: Blocking categories are documented and enforced
    what: A closed set of categories (COVERAGE_*, BROKEN_IMAGE_URL, etc.) is documented AND blocks ship.
    why: Without a defined blocking set, subjective judgment decides what ships.
    baseline_impl: task-designer/run.py + CLAUDE.md