Plan 61: Required Structure Rule Hardening (WIP)

PLAN.md

@@ -23,7 +23,7 @@ footer: |
 | 54  | ⛔     | [Conciseness Metrics Design and Implementation](plan/54_metrics-guide-tradeoffs.md)                             |
 | 56  | ⛔     | [Spike Ollama for Weasel Detection](plan/56_spike-ollama-weasel-detection.md)                                   |
 | 58  | ⛔     | [Select and Package Fast Weasel Classifier (CPU Fallback)](plan/58_classifier-model-selection-and-embedding.md) |
-| 61  | 🔲     | [Required Structure Rule Hardening](plan/61_required-structure-hardening.md)                                    |
+| 61  | 🔳     | [Required Structure Rule Hardening](plan/61_required-structure-hardening.md)                                    |
 | 62  | ✅     | [Corpus Acquisition and Taxonomy](plan/62_corpus-acquisition.md)                                                |
 | 64  | ✅     | [Spike Pure-Go Embedded Weasel Classifier](plan/64_spike-go-native-linear-classifier.md)                        |
 | 65  | 🔲     | [Spike WASM-Embedded Weasel Inference](plan/65_spike-wasm-embedded-inference.md)                                |

internal/rules/MDS020-required-structure/README.md

@@ -158,8 +158,9 @@ Describe the goal here.
 | Condition           | Message                                                                       |
 |---------------------|-------------------------------------------------------------------------------|
 | section missing     | missing required section "## Settings"                                        |
-| wrong level         | heading level mismatch: expected h2, got h3                                   |
-| extra section       | unexpected section "## Extra"                                                 |
+| wrong level         | heading level mismatch for "Settings": expected h2, got h3                    |
+| extra section       | unexpected section "## Extra" (expected "## Settings")                        |
+| out of order        | section "## Tasks" out of order: expected after "## Goal"                     |
 | heading sync        | heading does not match frontmatter: expected "MDS001" (from id), got "MDS002" |
 | body sync           | body does not match frontmatter field "description"                           |
 | front matter schema | front matter does not satisfy schema CUE constraints: ...                     |

internal/rules/MDS020-required-structure/bad/extra-section.md

@@ -0,0 +1,21 @@
+---
+settings:
+  schema: "../../internal/rules/MDS020-required-structure/bad/data/tmpl.md"
+diagnostics:
+  - line: 3
+    column: 1
+    message: 'unexpected section "## Extra" (expected "## Goal")'
+---
+# My Plan
+
+## Extra
+
+Extra description.
+
+## Goal
+
+Goal description.
+
+## Tasks
+
+Tasks description.

internal/rules/MDS020-required-structure/bad/out-of-order.md

@@ -0,0 +1,17 @@
+---
+settings:
+  schema: "../../internal/rules/MDS020-required-structure/bad/data/tmpl.md"
+diagnostics:
+  - line: 3
+    column: 1
+    message: 'section "## Tasks" out of order: expected after "## Goal"'
+---
+# My Plan
+
+## Tasks
+
+Tasks description.
+
+## Goal
+
+Goal description.

internal/rules/MDS020-required-structure/bad/wrong-level.md

@@ -0,0 +1,17 @@
+---
+settings:
+  schema: "../../internal/rules/MDS020-required-structure/bad/data/tmpl.md"
+diagnostics:
+  - line: 3
+    column: 1
+    message: 'heading level mismatch for "Goal": expected h2, got h3'
+---
+# My Plan
+
+### Goal
+
+Goal description.
+
+## Tasks
+
+Tasks description.

internal/rules/requiredstructure/rule.go

@@ -645,37 +645,44 @@ func checkStructure(
 ) []lint.Diagnostic {
 	var diags []lint.Diagnostic
 
+	// Map each literal required heading text to its schema indices,
+	// so that a doc heading seen at the "wrong" position can be
+	// recognized as an out-of-order required section rather than
+	// double-counted as both "unexpected" and "missing".
+	// Wildcard, `?` and field-interpolated headings are excluded
+	// because their match depends on context.
+	requiredByText := map[string][]int{}
+	for i, req := range sch.Headings {
+		if isSectionWildcard(req) || req.Text == "?" {
+			continue
+		}
+		if fieldinterp.ContainsField(req.Text) {
+			continue
+		}
+		requiredByText[req.Text] = append(requiredByText[req.Text], i)
+	}
+
+	claimed := make(map[int]bool)
 	docIdx := 0
 	allowExtra := false
-	for _, req := range sch.Headings {
+	for schIdx, req := range sch.Headings {
 		if isSectionWildcard(req) {
 			allowExtra = true
 			continue
 		}
+		if claimed[schIdx] {
+			continue
+		}
 
-		found := false
-		for docIdx < len(docHeadings) {
-			dh := docHeadings[docIdx]
-			if matchesSchema(req, dh) {
-				// Check level.
-				if dh.Level != req.Level {
-					diags = append(diags, makeDiag(f.Path, dh.Line,
-						fmt.Sprintf("heading level mismatch: expected h%d, got h%d",
-							req.Level, dh.Level)))
-				}
-				docIdx++
-				found = true
-				allowExtra = false
-				break
-			}
-			if !allowExtra {
-				diags = append(diags, makeDiag(f.Path, dh.Line,
-					fmt.Sprintf("unexpected section %q",
-						formatHeading(dh.Level, dh.Text))))
-			}
-			docIdx++
+		reqDiags, newIdx, found := matchRequired(
+			f, sch, docHeadings, docIdx, schIdx, requiredByText, claimed, allowExtra,
+		)
+		diags = append(diags, reqDiags...)
+		docIdx = newIdx
+		if found {
+			allowExtra = false
 		}
-		if !found {
+		if !found && !claimed[schIdx] {
 			diags = append(diags, makeDiag(f.Path, 1,
 				fmt.Sprintf("missing required section %q",
 					formatHeading(req.Level, req.Text))))
@@ -697,6 +704,76 @@ func checkStructure(
 	return diags
 }
 
+// matchRequired advances docIdx to find a doc heading matching req.
+// It emits diagnostics for intervening doc headings: "unexpected" when
+// they don't match any required section, or "out of order" when they
+// match a later required section (which is then claimed to avoid a
+// follow-up "missing required" for the same text).
+func matchRequired(
+	f *lint.File,
+	sch *parsedSchema,
+	docHeadings []docHeading,
+	docIdx, schIdx int,
+	requiredByText map[string][]int,
+	claimed map[int]bool,
+	allowExtra bool,
+) ([]lint.Diagnostic, int, bool) {
+	var diags []lint.Diagnostic
+	req := sch.Headings[schIdx]
+	for docIdx < len(docHeadings) {
+		dh := docHeadings[docIdx]
+		if matchesSchema(req, dh) {
+			if dh.Level != req.Level {
+				diags = append(diags, levelMismatchDiag(f, dh, req))
+			}
+			claimed[schIdx] = true
+			return diags, docIdx + 1, true
+		}
+		if ooIdx := nextUnclaimed(requiredByText[dh.Text], claimed, schIdx+1); ooIdx >= 0 {
+			other := sch.Headings[ooIdx]
+			diags = append(diags, makeDiag(f.Path, dh.Line,
+				fmt.Sprintf("section %q out of order: expected after %q",
+					formatHeading(dh.Level, dh.Text),
+					formatHeading(req.Level, req.Text))))
+			if dh.Level != other.Level {
+				diags = append(diags, levelMismatchDiag(f, dh, other))
+			}
+			claimed[ooIdx] = true
+			docIdx++
+			continue
+		}
+		if !allowExtra {
+			diags = append(diags, makeDiag(f.Path, dh.Line,
+				fmt.Sprintf("unexpected section %q (expected %q)",
+					formatHeading(dh.Level, dh.Text),
+					formatHeading(req.Level, req.Text))))
+		}
+		docIdx++
+	}
+	return diags, docIdx, false
+}
+
+// levelMismatchDiag builds a heading level-mismatch diagnostic that
+// names the offending heading so readers can locate it quickly.
+func levelMismatchDiag(
+	f *lint.File, dh docHeading, req schemaHeading,
+) lint.Diagnostic {
+	return makeDiag(f.Path, dh.Line,
+		fmt.Sprintf("heading level mismatch for %q: expected h%d, got h%d",
+			dh.Text, req.Level, dh.Level))
+}
+
+// nextUnclaimed returns the first index in candidates that is >= minIdx
+// and not yet claimed, or -1 if none qualifies.
+func nextUnclaimed(candidates []int, claimed map[int]bool, minIdx int) int {
+	for _, idx := range candidates {
+		if idx >= minIdx && !claimed[idx] {
+			return idx
+		}
+	}
+	return -1
+}
+
 // matchesSchema checks if a document heading matches a schema heading.
 func matchesSchema(req schemaHeading, doc docHeading) bool {
 	// Wildcard heading: matches any text at any level.

internal/rules/requiredstructure/rule_test.go

@@ -294,6 +294,66 @@ func TestCheck_WrongLevel(t *testing.T) {
 	expectDiagMsg(t, diags, "heading level mismatch")
 }
 
+// Level-mismatch diagnostics must name the offending heading so
+// readers can locate it in documents with many headings.
+func TestCheck_WrongLevel_NamesHeading(t *testing.T) {
+	schemaPath := writeSchema(t, "# ?\n\n## Settings\n")
+	r := &Rule{Schema: schemaPath}
+	f := newTestFile(t, "doc.md",
+		"# My Rule\n\n### Settings\n")
+	diags := r.Check(f)
+	expectDiagMsg(t, diags, `"Settings"`)
+	expectDiagMsg(t, diags, "expected h2, got h3")
+}
+
+// Unexpected-section diagnostics should tell the author which
+// required heading was expected at that position.
+func TestCheck_ExtraSection_NamesExpected(t *testing.T) {
+	schemaPath := writeSchema(t, "# ?\n\n## Goal\n")
+	r := &Rule{Schema: schemaPath}
+	f := newTestFile(t, "doc.md",
+		"# My Plan\n\n## Prerequisites\n\n## Goal\n")
+	diags := r.Check(f)
+	expectDiagMsg(t, diags, `unexpected section "## Prerequisites"`)
+	expectDiagMsg(t, diags, `expected "## Goal"`)
+}
+
+// Trailing extras (past the last required heading) have no
+// "expected next" to name, so the diagnostic should still be
+// emitted without an expected suffix.
+func TestCheck_ExtraSection_TrailingNoExpected(t *testing.T) {
+	schemaPath := writeSchema(t, "# ?\n\n## Goal\n")
+	r := &Rule{Schema: schemaPath}
+	f := newTestFile(t, "doc.md",
+		"# My Plan\n\n## Goal\n\n## Trailing\n")
+	diags := r.Check(f)
+	expectDiagMsg(t, diags, `unexpected section "## Trailing"`)
+}
+
+// When a required section appears but in the wrong order, the
+// rule should report it as out-of-order rather than double-counting
+// it as both "unexpected" and "missing required".
+func TestCheck_OutOfOrderSection(t *testing.T) {
+	schemaPath := writeSchema(t,
+		"# ?\n\n## Goal\n\n## Tasks\n")
+	r := &Rule{Schema: schemaPath}
+	f := newTestFile(t, "doc.md",
+		"# My Plan\n\n## Tasks\n\n## Goal\n")
+	diags := r.Check(f)
+	msgs := make([]string, len(diags))
+	for i, d := range diags {
+		msgs[i] = d.Message
+	}
+	// The document holds both required sections, so the rule must
+	// not emit a "missing required" diagnostic for either.
+	for _, m := range msgs {
+		if strings.Contains(m, "missing required section") {
+			t.Errorf("unexpected missing-required diagnostic: %q (all: %v)", m, msgs)
+		}
+	}
+	expectDiagMsg(t, diags, `out of order`)
+}
+
 func TestCheck_AllPresent(t *testing.T) {
 	schemaPath := writeSchema(t,
 		"# ?\n\n## Settings\n\n## Examples\n\n### Good\n\n### Bad\n")

plan/61_required-structure-hardening.md

@@ -1,7 +1,7 @@
 ---
 id: 61
 title: Required Structure Rule Hardening
-status: 🔲
+status: 🔳
 ---
 # Required Structure Rule Hardening