Skip to content

Commit 5f093c6

Browse files
o-azo-az
committed
refactor: simplify Hermes configuration audit
1 parent c7f2785 commit 5f093c6

7 files changed

Lines changed: 1450 additions & 1692 deletions

File tree

‎skills/auditing-hermes-config/SKILL.md‎

Lines changed: 20 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -1,49 +1,30 @@
11
---
22
name: auditing-hermes-config
3-
description: Audits revision-pinned Hermes runtime configuration, native NixOS module options, and host configuration. Use when reviewing Hermes defaults, migrations, validity, module wiring, or revision drift.
4-
compatibility: Requires Node.js 20+. Nix is optional; static fallback is explicitly limited.
3+
description: Audits exact-revision Hermes application options, native NixOS module options, and one host's evaluated configuration.
4+
compatibility: Requires Node.js 20+, GitHub CLI authentication, and normally Nix.
55
---
66

77
# Auditing Hermes Config
88

9-
Keep three evidence surfaces separate: Hermes runtime configuration, the exact
10-
`hermes-agent` NixOS module pinned in `flake.lock`, and target host-effective
11-
configuration. Absence from defaults alone does not prove invalidity.
12-
13-
## Run
14-
15-
Set the installed skill directory and always identify the target explicitly:
9+
Run the one read-only command:
1610

1711
```sh
18-
SKILL_DIR=/absolute/path/to/skills/auditing-hermes-config
19-
node "$SKILL_DIR/scripts/hermes-config-audit.mjs" discover --target-repo /path/to/nixos-config --source /exact/worktree --cache /outside/cache
20-
node "$SKILL_DIR/scripts/hermes-config-audit.mjs" index --target-repo /path/to/nixos-config --source /exact/worktree --cache /outside/cache
21-
node "$SKILL_DIR/scripts/hermes-config-audit.mjs" audit --target-repo /path/to/nixos-config --host marley --source /exact/worktree --cache /outside/cache --json /outside/reports/audit.json --markdown /outside/reports/audit.md
22-
node "$SKILL_DIR/scripts/hermes-config-audit.mjs" compare --base /outside/base.json --target /outside/target.json
12+
node scripts/hermes-config-audit.mjs audit --target-repo /path/to/nixos-config --host HOST
2313
```
2414

25-
`--target-repo` is canonical. `HERMES_TARGET_REPO` and upward discovery are
26-
fallbacks; `--repo` is a compatibility alias. Outputs and cache must be outside
27-
the target, skills repository, and source worktree. Read
28-
[surfaces](reference/surfaces.md) before interpreting results and the
29-
[output schema](reference/output-schema.md) before consuming JSON.
30-
31-
## Discipline
32-
33-
Report **Proper**, **Repo today**, **Gap**, and **Path**. Treat unresolved values
34-
and incomplete dynamic schemas as uncertainty, never as proof of invalidity.
35-
`configFile` may bypass generated `settings`; do not realize derivations merely
36-
to inspect it.
37-
38-
The tool is read-only. Never run Python, build, switch, deploy, rebuild, restart,
39-
or edit target configuration. Snapshot permits at most one read-only `nix eval`
40-
per host invocation, disables import-from-derivation, redacts scalar host values,
41-
and falls back honestly when evaluation fails. Add
42-
`--host-adapter hosts/marley/hermes/settings.nix` for tracked static local evidence;
43-
without it static host evidence remains unresolved.
44-
45-
Static analysis is not effective configuration. Dynamic contracts and the
46-
structural Nix parser can be incomplete; `configFile` content stays unresolved
47-
without realization; offline Nix can fail. `--latest` is tested with a
48-
deterministic fake `gh`; live GitHub availability is not tested. Target dirty
49-
state is reported.
15+
It gets owner, repository, and exact SHA from the target's tracked `flake.lock`,
16+
checks `gh auth status`, reads only targeted files at that SHA through `gh api`,
17+
and performs exactly one `nix eval`. It writes mode-0600 JSON and Markdown to
18+
`$XDG_STATE_HOME/hermes-config-audit` (or `~/.local/state/...`) and prints both
19+
paths. `--output-dir` may select another directory outside both repositories.
20+
GitHub reads follow the [`gh api` manual](https://cli.github.com/manual/gh_api).
21+
22+
Use `--source /git/repository` offline; the repository need only contain the
23+
pinned commit because reads use `git show SHA:path`. `--no-nix` is an explicitly
24+
incomplete fallback. `--latest` keeps the pinned inventories authoritative and
25+
adds an explicit comparison with the current GitHub HEAD, so it still requires
26+
authenticated `gh` even when pinned reads use `--source`.
27+
28+
Never execute upstream Python, realize/build/switch/deploy/restart Nix, or infer
29+
invalidity merely from absence in defaults. Read [surfaces](reference/surfaces.md)
30+
and [output schema](reference/output-schema.md) when interpreting results.

‎skills/auditing-hermes-config/reference/audit.schema.json‎

Lines changed: 152 additions & 89 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,30 @@
11
{
22
"$schema": "https://json-schema.org/draft/2020-12/schema",
3-
"$id": "https://example.invalid/hermes-config-audit.schema.json",
43
"type": "object",
4+
"additionalProperties": false,
55
"required": [
66
"schemaVersion",
7-
"command",
8-
"generatedAt",
9-
"readOnly",
107
"provenance",
11-
"surfaces",
12-
"findings",
13-
"moduleSummary",
14-
"limits",
15-
"safety"
8+
"applicationInventory",
9+
"moduleInventory",
10+
"local",
11+
"comparison",
12+
"limits"
1613
],
1714
"properties": {
18-
"schemaVersion": { "const": 4 },
19-
"command": { "const": "audit" },
20-
"generatedAt": { "type": "string", "format": "date-time" },
21-
"readOnly": { "const": true },
15+
"schemaVersion": { "const": 5 },
2216
"provenance": {
2317
"type": "object",
18+
"additionalProperties": false,
2419
"required": [
2520
"selection",
2621
"lockedSha",
2722
"resolvedSha",
2823
"owner",
2924
"repo",
3025
"targetRepo",
31-
"targetDirty",
32-
"host"
26+
"host",
27+
"backend"
3328
],
3429
"properties": {
3530
"selection": { "enum": ["locked", "latest"] },
@@ -38,84 +33,122 @@
3833
"owner": { "type": "string", "minLength": 1 },
3934
"repo": { "type": "string", "minLength": 1 },
4035
"targetRepo": { "type": "string", "minLength": 1 },
41-
"targetDirty": { "type": "boolean" },
4236
"host": { "type": "string", "minLength": 1 },
37+
"backend": { "enum": ["gh-api", "git-object"] },
38+
"latestSha": { "$ref": "#/$defs/sha" },
4339
"resolvedAt": { "type": "string", "format": "date-time" }
44-
},
45-
"allOf": [
46-
{
47-
"if": { "properties": { "selection": { "const": "latest" } } },
48-
"then": { "required": ["resolvedAt"] }
49-
}
50-
]
40+
}
41+
},
42+
"applicationInventory": {
43+
"type": "object",
44+
"additionalProperties": false,
45+
"required": ["entries", "diagnostics", "completeness"],
46+
"properties": {
47+
"entries": { "type": "array", "items": { "$ref": "#/$defs/applicationEntry" } },
48+
"diagnostics": { "type": "array", "items": { "type": "string" } },
49+
"completeness": { "type": "string" },
50+
"latestComparison": { "$ref": "#/$defs/latestComparison" }
51+
}
5152
},
52-
"surfaces": {
53+
"moduleInventory": {
5354
"type": "object",
54-
"required": ["runtime", "module", "host"],
55+
"additionalProperties": false,
56+
"required": ["entries", "completeness"],
5557
"properties": {
56-
"runtime": {
58+
"entries": { "type": "array", "items": { "$ref": "#/$defs/moduleEntry" } },
59+
"completeness": { "type": "string" },
60+
"latestComparison": { "$ref": "#/$defs/latestComparison" }
61+
}
62+
},
63+
"local": {
64+
"oneOf": [
65+
{
5766
"type": "object",
58-
"required": ["contracts", "defaultCount"],
59-
"properties": {
60-
"contracts": { "type": "object" },
61-
"defaultCount": { "type": "integer", "minimum": 0 }
62-
}
67+
"additionalProperties": false,
68+
"required": ["available", "reason"],
69+
"properties": { "available": { "const": false }, "reason": { "type": "string" } }
6370
},
64-
"module": {
71+
{
6572
"type": "object",
73+
"additionalProperties": false,
6674
"required": [
67-
"source",
68-
"optionCount",
69-
"options",
70-
"completeness",
71-
"diagnostics",
72-
"generation"
75+
"available",
76+
"enabled",
77+
"mode",
78+
"configFileSet",
79+
"nativeShape",
80+
"applicationShape"
7381
],
7482
"properties": {
75-
"source": { "type": "string" },
76-
"optionCount": { "type": "integer" },
77-
"options": { "type": "array", "items": { "$ref": "#/$defs/option" } },
78-
"completeness": { "type": "string" },
79-
"diagnostics": { "type": "array", "items": { "type": "string" } },
80-
"generation": { "type": "object" }
83+
"available": { "const": true },
84+
"enabled": { "type": "boolean" },
85+
"mode": { "enum": ["disabled", "generated", "configFile"] },
86+
"configFileSet": { "type": "boolean" },
87+
"nativeShape": { "type": "object" },
88+
"applicationShape": { "type": ["object", "null"] }
8189
}
90+
}
91+
]
92+
},
93+
"comparison": {
94+
"oneOf": [
95+
{
96+
"type": "object",
97+
"additionalProperties": false,
98+
"required": ["available", "reason"],
99+
"properties": { "available": { "const": false }, "reason": { "type": "string" } }
82100
},
83-
"host": {
101+
{
84102
"type": "object",
103+
"additionalProperties": false,
85104
"required": [
86-
"host",
87-
"provenance",
88-
"evaluated",
89-
"runtimeDocument",
90-
"moduleValues",
91-
"localAssignments",
92-
"mode",
93-
"settingsEmpty",
94-
"configFileBypassesSettings",
95-
"limits"
96-
]
105+
"available",
106+
"applicationAvailable",
107+
"presentInGeneratedOverride",
108+
"catalogPathsAbsentFromGeneratedOverride",
109+
"applicationMismatches",
110+
"presentInEvaluatedNativeShape",
111+
"catalogPathsAbsentFromEvaluatedNativeShape",
112+
"moduleMismatches"
113+
],
114+
"properties": {
115+
"available": { "const": true },
116+
"applicationAvailable": { "type": "boolean" },
117+
"applicationReason": { "type": "string" },
118+
"presentInGeneratedOverride": { "type": "array", "items": { "type": "string" } },
119+
"catalogPathsAbsentFromGeneratedOverride": {
120+
"type": "array",
121+
"items": { "type": "string" }
122+
},
123+
"applicationMismatches": { "type": "array", "items": { "$ref": "#/$defs/mismatch" } },
124+
"presentInEvaluatedNativeShape": { "type": "array", "items": { "type": "string" } },
125+
"catalogPathsAbsentFromEvaluatedNativeShape": {
126+
"type": "array",
127+
"items": { "type": "string" }
128+
},
129+
"moduleMismatches": { "type": "array", "items": { "$ref": "#/$defs/mismatch" } }
130+
}
97131
}
98-
}
99-
},
100-
"findings": {
101-
"type": "array",
102-
"items": { "$ref": "#/$defs/finding" }
132+
]
103133
},
104-
"moduleSummary": {
134+
"limits": { "type": "array", "items": { "type": "string" } }
135+
},
136+
"$defs": {
137+
"sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
138+
"latestComparison": {
105139
"type": "object",
106-
"required": ["effectiveTopLevelValues", "optionCoverage"],
140+
"additionalProperties": false,
141+
"required": ["sha", "addedPaths", "removedPaths", "changedPaths"],
107142
"properties": {
108-
"effectiveTopLevelValues": { "type": "array" },
109-
"optionCoverage": { "const": "unavailable" }
143+
"sha": { "$ref": "#/$defs/sha" },
144+
"addedPaths": { "type": "array", "items": { "type": "string" } },
145+
"removedPaths": { "type": "array", "items": { "type": "string" } },
146+
"changedPaths": { "type": "array", "items": { "type": "string" } }
110147
}
111148
},
112-
"limits": { "type": "array", "items": { "type": "string" } },
113-
"safety": { "type": "object", "required": ["performed", "forbidden"] }
114-
},
115-
"$defs": {
116-
"sha": { "type": "string", "pattern": "^[0-9a-f]{40}$" },
117149
"evidence": {
118150
"type": "object",
151+
"additionalProperties": false,
119152
"required": ["source", "line", "url", "excerpt"],
120153
"properties": {
121154
"source": { "type": "string" },
@@ -124,41 +157,71 @@
124157
"excerpt": { "type": "string" }
125158
}
126159
},
127-
"option": {
160+
"applicationEntry": {
128161
"type": "object",
162+
"additionalProperties": false,
129163
"required": [
130164
"path",
131-
"typeExpression",
132-
"default",
133-
"example",
134-
"description",
135-
"declaration",
136-
"mappingEvidence"
165+
"expected",
166+
"condition",
167+
"dynamic",
168+
"confidence",
169+
"migration",
170+
"evidence"
137171
],
138172
"properties": {
139173
"path": { "type": "string" },
140-
"typeExpression": { "type": ["string", "null"] },
141-
"declaration": { "$ref": "#/$defs/evidence" },
142-
"mappingEvidence": { "type": "array" }
174+
"default": {},
175+
"defaultShape": { "type": "string" },
176+
"expected": { "type": "string" },
177+
"condition": { "type": "string" },
178+
"dynamic": { "type": "boolean" },
179+
"confidence": {
180+
"enum": [
181+
"base-default",
182+
"known-consumed-field",
183+
"validation-open-dictionary",
184+
"explicit-migration"
185+
]
186+
},
187+
"migration": { "type": "string" },
188+
"evidence": { "$ref": "#/$defs/evidence" }
143189
}
144190
},
145-
"finding": {
191+
"moduleEntry": {
146192
"type": "object",
147-
"required": ["path", "classification", "reason"],
193+
"additionalProperties": false,
194+
"required": ["path", "type", "default", "example", "description", "mapping", "evidence"],
148195
"properties": {
149196
"path": { "type": "string" },
197+
"type": { "type": ["string", "null"] },
198+
"default": {},
199+
"example": {},
200+
"description": { "type": ["string", "null"] },
201+
"mapping": {
202+
"enum": ["service-behavior", "generates-application-config", "selects-application-config"]
203+
},
204+
"evidence": { "$ref": "#/$defs/evidence" }
205+
}
206+
},
207+
"mismatch": {
208+
"type": "object",
209+
"additionalProperties": false,
210+
"required": ["path", "type", "classification"],
211+
"properties": {
212+
"path": { "type": "string" },
213+
"type": { "type": "string" },
214+
"expected": { "type": "string" },
150215
"classification": {
151216
"enum": [
152-
"current",
153-
"valid-dynamic-extension",
154-
"obsolete-or-wrong-shape",
155-
"ignored-or-nonexistent",
156-
"intentional-semantic-exception",
217+
"wrong-shape",
218+
"deprecated-or-migrated",
219+
"invalid-or-ignored",
157220
"uncertain-needs-targeted-review"
158221
]
159222
},
160-
"reason": { "type": "string" },
161-
"evidence": { "$ref": "#/$defs/evidence" }
223+
"evidence": { "$ref": "#/$defs/evidence" },
224+
"candidates": { "type": "array", "items": { "$ref": "#/$defs/evidence" } }
162225
}
163226
}
164227
}

0 commit comments

Comments
 (0)