Add Hermes configuration audit skill

pyproject.toml

@@ -10,6 +10,7 @@ dev = ["pyrefly>=1.1.1", "ruff>=0.16.0"]
 [tool.pyrefly]
 search_path = [".", "skills/multilingual-caption-video/scripts"]
 ignore-missing-imports = ["faster_whisper", "yt_dlp"]
+project_excludes = ["tests/auditing-hermes-config/fixtures/config-defaults.py"]
 
 [tool.ruff]
 line-length = 80

skills/auditing-hermes-config/SKILL.md

@@ -0,0 +1,34 @@
+---
+name: auditing-hermes-config
+description: Audits exact-revision Hermes application options, native NixOS module options, and one host's evaluated configuration.
+compatibility: Requires Node.js 20+, Python 3, GitHub CLI authentication, and normally Nix.
+---
+
+# Auditing Hermes Config
+
+Run the one read-only command:
+
+```sh
+node scripts/hermes-config-audit.mjs audit --target-repo /path/to/nixos-config --host HOST
+```
+
+It gets owner, repository, and exact SHA from the target's `flake.lock`,
+checks `gh auth status`, enumerates that SHA's tree once, resolves bounded
+Hermes-specific source roles by semantic signatures, and reads only selected
+files through `gh api`,
+and performs exactly one `nix eval`. It writes mode-0600 JSON and Markdown to
+`$XDG_STATE_HOME/hermes-config-audit` (or `~/.local/state/...`) and prints both
+paths. `--output-dir` may select another directory outside both repositories.
+GitHub reads follow the [`gh api` manual](https://cli.github.com/manual/gh_api).
+
+Use `--source /git/repository` offline; the repository need only contain the
+pinned commit because reads use `git show SHA:path`. `--no-nix` is an explicitly
+incomplete fallback. `--latest` keeps the pinned inventories authoritative and
+adds an explicit comparison with the current GitHub HEAD, so it still requires
+authenticated `gh` even when pinned reads use `--source`.
+
+The target only needs to be a readable directory containing `flake.lock`; Git
+history is not required. Never execute upstream Python,
+realize/build/switch/deploy/restart Nix, or infer
+invalidity merely from absence in defaults. Read [surfaces](reference/surfaces.md)
+and [output schema](reference/output-schema.md) when interpreting results.

skills/auditing-hermes-config/reference/audit.schema.json

@@ -0,0 +1,297 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "provenance",
+    "sourceRoles",
+    "applicationInventory",
+    "moduleInventory",
+    "local",
+    "comparison",
+    "completeness",
+    "requests",
+    "limits"
+  ],
+  "properties": {
+    "schemaVersion": { "const": 6 },
+    "sourceRoles": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/sourceRole" }
+    },
+    "provenance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "selection",
+        "lockedSha",
+        "resolvedSha",
+        "owner",
+        "repo",
+        "targetRepo",
+        "host",
+        "backend"
+      ],
+      "properties": {
+        "selection": { "enum": ["locked", "latest"] },
+        "lockedSha": { "$ref": "#/$defs/sha" },
+        "resolvedSha": { "$ref": "#/$defs/sha" },
+        "owner": { "type": "string", "minLength": 1 },
+        "repo": { "type": "string", "minLength": 1 },
+        "targetRepo": { "type": "string", "minLength": 1 },
+        "host": { "type": "string", "minLength": 1 },
+        "backend": { "enum": ["gh-api", "git-object"] },
+        "latestSha": { "$ref": "#/$defs/sha" },
+        "resolvedAt": { "type": "string", "format": "date-time" }
+      }
+    },
+    "applicationInventory": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["entries", "diagnostics", "completeness"],
+      "properties": {
+        "entries": { "type": "array", "items": { "$ref": "#/$defs/applicationEntry" } },
+        "diagnostics": { "type": "array", "items": { "type": "string" } },
+        "completeness": { "type": "string" },
+        "latestComparison": { "$ref": "#/$defs/latestComparison" }
+      }
+    },
+    "moduleInventory": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["entries", "completeness"],
+      "properties": {
+        "entries": { "type": "array", "items": { "$ref": "#/$defs/moduleEntry" } },
+        "completeness": { "type": "string" },
+        "latestComparison": { "$ref": "#/$defs/latestComparison" }
+      }
+    },
+    "local": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["available", "reason"],
+          "properties": { "available": { "const": false }, "reason": { "type": "string" } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "available",
+            "enabled",
+            "mode",
+            "configFileSet",
+            "nativeShape",
+            "applicationShape"
+          ],
+          "properties": {
+            "available": { "const": true },
+            "enabled": { "type": "boolean" },
+            "mode": { "enum": ["disabled", "generated", "configFile"] },
+            "configFileSet": { "type": "boolean" },
+            "nativeShape": { "type": "object" },
+            "applicationShape": { "type": ["object", "null"] }
+          }
+        }
+      ]
+    },
+    "comparison": {
+      "oneOf": [
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["available", "reason"],
+          "properties": { "available": { "const": false }, "reason": { "type": "string" } }
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "required": [
+            "available",
+            "applicationAvailable",
+            "presentInGeneratedOverride",
+            "applicationResults",
+            "catalogPathsAbsentFromGeneratedOverride",
+            "applicationMismatches",
+            "presentInEvaluatedNativeShape",
+            "catalogPathsAbsentFromEvaluatedNativeShape",
+            "moduleMismatches"
+          ],
+          "properties": {
+            "available": { "const": true },
+            "applicationAvailable": { "type": "boolean" },
+            "applicationReason": { "type": "string" },
+            "presentInGeneratedOverride": { "type": "array", "items": { "type": "string" } },
+            "applicationResults": { "type": "array", "items": { "$ref": "#/$defs/mismatch" } },
+            "catalogPathsAbsentFromGeneratedOverride": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "applicationMismatches": { "type": "array", "items": { "$ref": "#/$defs/mismatch" } },
+            "presentInEvaluatedNativeShape": { "type": "array", "items": { "type": "string" } },
+            "catalogPathsAbsentFromEvaluatedNativeShape": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "moduleMismatches": { "type": "array", "items": { "$ref": "#/$defs/mismatch" } }
+          }
+        }
+      ]
+    },
+    "completeness": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["overall", "localComparison", "inventory"],
+      "properties": {
+        "overall": { "type": "string" },
+        "localComparison": { "type": "string" },
+        "inventory": { "type": "string" }
+      }
+    },
+    "requests": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["githubApi", "githubSearch"],
+      "properties": {
+        "githubApi": { "type": "integer", "minimum": 0 },
+        "githubSearch": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "limits": { "type": "array", "items": { "type": "string" } }
+  },
+  "$defs": {
+    "sourceRole": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["role", "status", "required", "candidateCount", "inspectedCount", "sources"],
+      "properties": {
+        "role": { "type": "string" },
+        "status": {
+          "enum": [
+            "resolved",
+            "unresolved-no-candidate",
+            "candidates-found-none-validated",
+            "ambiguous-multiple-authoritative",
+            "extractor-unsupported"
+          ]
+        },
+        "required": { "type": "boolean" },
+        "candidateCount": { "type": "integer", "minimum": 0 },
+        "inspectedCount": { "type": "integer", "minimum": 0 },
+        "sources": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["path", "reason", "capabilities"],
+            "properties": {
+              "path": { "type": "string" },
+              "reason": { "type": "string" },
+              "capabilities": {
+                "type": "array",
+                "items": { "type": "string" }
+              }
+            }
+          }
+        }
+      }
+    },
+    "sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
+    "latestComparison": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["sha", "addedPaths", "removedPaths", "changedPaths"],
+      "properties": {
+        "sha": { "$ref": "#/$defs/sha" },
+        "addedPaths": { "type": "array", "items": { "type": "string" } },
+        "removedPaths": { "type": "array", "items": { "type": "string" } },
+        "changedPaths": { "type": "array", "items": { "type": "string" } }
+      }
+    },
+    "evidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["source", "line", "url", "excerpt"],
+      "properties": {
+        "source": { "type": "string" },
+        "line": { "type": "integer", "minimum": 1 },
+        "url": { "type": "string" },
+        "excerpt": { "type": "string" }
+      }
+    },
+    "applicationEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "path",
+        "expected",
+        "condition",
+        "dynamic",
+        "confidence",
+        "migration",
+        "evidence"
+      ],
+      "properties": {
+        "path": { "type": "string" },
+        "default": {},
+        "defaultShape": { "type": "string" },
+        "expected": { "type": "string" },
+        "condition": { "type": "string" },
+        "dynamic": { "type": "boolean" },
+        "confidence": {
+          "enum": [
+            "base-default",
+            "known-consumed-field",
+            "validation-open-dictionary",
+            "explicit-migration",
+            "explicit-invalid-shape"
+          ]
+        },
+        "migration": { "type": "string" },
+        "evidence": { "$ref": "#/$defs/evidence" }
+      }
+    },
+    "moduleEntry": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "type", "default", "example", "description", "mapping", "evidence"],
+      "properties": {
+        "path": { "type": "string" },
+        "type": { "type": ["string", "null"] },
+        "default": {},
+        "example": {},
+        "description": { "type": ["string", "null"] },
+        "mapping": {
+          "enum": ["service-behavior", "generates-application-config", "selects-application-config"]
+        },
+        "evidence": { "$ref": "#/$defs/evidence" }
+      }
+    },
+    "mismatch": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "type", "classification"],
+      "properties": {
+        "path": { "type": "string" },
+        "type": { "type": "string" },
+        "expected": { "type": "string" },
+        "classification": {
+          "enum": [
+            "current",
+            "valid-dynamic-extension",
+            "wrong-shape",
+            "deprecated-or-migrated",
+            "obsolete-or-wrong-shape",
+            "invalid-or-ignored",
+            "ignored-or-nonexistent",
+            "uncertain-needs-targeted-review"
+          ]
+        },
+        "evidence": { "$ref": "#/$defs/evidence" },
+        "candidates": { "type": "array", "items": { "$ref": "#/$defs/evidence" } }
+      }
+    }
+  }
+}

skills/auditing-hermes-config/reference/output-schema.md

@@ -0,0 +1,41 @@
+# Output schema
+
+The JSON report uses `schemaVersion: 6` and has exactly these top-level fields:
+
+- `schemaVersion`: report contract version
+- `provenance`: lock identity, selected exact SHA, target, host, and backend
+- `sourceRoles`: bounded semantic source-role resolution status, required flag,
+  candidate and inspected counts, validated capabilities, deterministic
+  selection reasons, and selected exact-tree paths
+- `applicationInventory`: option entries and an honest completeness statement
+- `moduleInventory`: native option entries and completeness statement
+- `local`: redacted evaluated native and Nix-generated override key/type shapes,
+  or an
+  unavailable reason
+- `comparison`: catalog paths present or absent in the evaluated shapes plus
+  all classified application paths and application/module mismatches; it does
+  not claim which values were explicitly assigned by local source
+- `completeness`: explicit overall, local-comparison, and inventory status
+- `requests`: GitHub API and code-search request counts
+- `limits`: unresolved and completeness qualifications
+
+Source roles are `resolved`, `unresolved-no-candidate`,
+`candidates-found-none-validated`, `ambiguous-multiple-authoritative`, or
+`extractor-unsupported`. Required foundational gaps stop inventory construction;
+optional gaps remain visible here and reduce the stated completeness in `limits`.
+Resolution enumerates the pinned commit tree once (with bounded subtree traversal
+if GitHub reports a truncated recursive tree), validates preferred paths by
+content signature, and otherwise reads only a bounded, deterministically ranked
+set of semantic candidates. Tests, generated files, translations, examples, and
+compatibility copies are excluded from fallback selection.
+
+Application entries include path, default and observed default shape where
+available, source-established expected shape, confidence, conditions, migration
+status, and exact-SHA evidence. A default alone is not treated as an accepted
+type contract. Module
+entries include path, type, default, example, description, mapping, and evidence.
+With `--latest`, each inventory also contains `latestComparison` with the exact
+latest SHA and added, removed, and changed paths; pinned inventory evidence stays
+authoritative.
+The machine-readable contract is [audit.schema.json](audit.schema.json). Both
+JSON and Markdown reports are created with mode `0600`.