fix(pii): label detections by their source (ner vs pattern)

Pattern-matcher hits were stored and masked with an "ner:" prefix even
though no NER (Named Entity Recognition) model was involved, because the
redactor hard-coded the prefix for every detector. Thread a Source through
NERConfig (SourceNER / SourcePattern; empty defaults to ner for
back-compat) and build the synthetic id from it via NERConfig.patternID.

Pattern detections now carry pattern:<GROUP> ids and [REDACTED:pattern:<GROUP>]
masks; NER detections stay ner:<GROUP>. The resolver tags each detector with
its source, and the doc strings / swagger / api-instructions examples are
updated to match.

Assisted-by: claude-code:claude-opus-4-8 [Claude Code]

Author: richiejp

core/application/application.go

@@ -285,6 +285,7 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver {
 				0, // patterns are deterministic — no confidence floor
 				cfg.PIIDetectionDefaultAction(),
 				patternEntityActions(cfg),
+				pii.SourcePattern,
 			), true
 		}
 
@@ -294,6 +295,7 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver {
 			cfg.PIIDetectionMinScore(),
 			cfg.PIIDetectionDefaultAction(),
 			cfg.PIIDetectionEntityActions(),
+			pii.SourceNER,
 		), true
 	}
 }

core/http/endpoints/localai/api_instructions.go

@@ -102,7 +102,7 @@ var instructionDefs = []instructionDef{
 		Name:        "pii-filtering",
 		Description: "Inspect the NER-based PII filter applied to chat requests",
 		Tags:        []string{"pii"},
-		Intro:       "PII redaction is NER-based and request-side. A consuming model opts in with `pii: { enabled: true, detectors: [<model>] }` where each detector is a token-classification (token_classify) model. The detection policy lives on the detector model itself in a `pii_detection:` block: `{ min_score, default_action (mask|block|allow), entity_actions: { GROUP: action } }`. Multiple detectors union their hits; overlapping spans resolve to the strongest action (block > mask > allow). PII defaults OFF for non-proxy backends and ON for proxy-* (cloud passthroughs). GET /api/pii/events returns recent redaction events filtered by correlation_id / user_id / pattern_id (events carry `ner:<GROUP>` ids and an 8-char hash prefix, never the matched value; admin or local-user only). The legacy regex pattern tier and its endpoints (/api/pii/patterns, /test, /decide) were removed.",
+		Intro:       "PII redaction is NER-based and request-side. A consuming model opts in with `pii: { enabled: true, detectors: [<model>] }` where each detector is a token-classification (token_classify) model. The detection policy lives on the detector model itself in a `pii_detection:` block: `{ min_score, default_action (mask|block|allow), entity_actions: { GROUP: action } }`. Multiple detectors union their hits; overlapping spans resolve to the strongest action (block > mask > allow). PII defaults OFF for non-proxy backends and ON for proxy-* (cloud passthroughs). GET /api/pii/events returns recent redaction events filtered by correlation_id / user_id / pattern_id (events carry `<source>:<GROUP>` ids — e.g. `ner:EMAIL` for the neural detector, `pattern:ANTHROPIC_KEY` for the regex pattern tier — and an 8-char hash prefix, never the matched value; admin or local-user only). The legacy regex pattern tier and its endpoints (/api/pii/patterns, /test, /decide) were removed.",
 	},
 	{
 		Name:        "middleware-admin",

core/http/routes/pii.go

@@ -36,7 +36,7 @@ func RegisterPIIRoutes(e *echo.Echo, app *application.Application) {
 	// @Produce json
 	// @Param correlation_id query string false "Correlation ID join key"
 	// @Param user_id query string false "User id"
-	// @Param pattern_id query string false "Detector group id (e.g. ner:EMAIL)"
+	// @Param pattern_id query string false "Detector group id (e.g. ner:EMAIL, pattern:ANTHROPIC_KEY)"
 	// @Param kind query string false "Event kind: pii | proxy_connect | proxy_traffic"
 	// @Param limit query int false "Max events" default(100)
 	// @Success 200 {object} map[string]interface{}

core/services/routing/pii/ner.go

@@ -60,8 +60,22 @@ type NERConfig struct {
 	// entities silently" — useful when the model returns a broad
 	// taxonomy but the admin only cares about a subset.
 	DefaultAction Action
+
+	// Source labels where this detector's hits come from. It becomes the
+	// PatternID prefix on events and the [REDACTED:<id>] mask, so neural NER
+	// detections (Source "ner") and deterministic pattern-matcher detections
+	// (Source "pattern") are told apart in the events log and to the model.
+	// Empty defaults to "ner" for backward compatibility.
+	Source string
 }
 
+// Detector source labels (the PatternID prefix). Kept short and stable —
+// they appear in the events log and the [REDACTED:...] mask.
+const (
+	SourceNER     = "ner"
+	SourcePattern = "pattern"
+)
+
 // ResolveAction returns the action configured for a detected entity
 // group, falling back to DefaultAction. Returns ("", false) when the
 // entity should be ignored entirely (no override + no default).
@@ -82,23 +96,32 @@ func (c NERConfig) ResolveAction(group string) (Action, bool) {
 // downgrades it). Unknown per-entity actions are dropped (and logged by
 // validActions). This is the single conversion point the application-layer
 // resolver uses, so the detector model's policy reaches the redactor in
-// exactly one shape.
-func NERConfigFromRaw(detector NERDetector, minScore float32, defaultAction string, entityActions map[string]string) NERConfig {
+// exactly one shape. source labels the detector kind (SourceNER /
+// SourcePattern) and becomes the PatternID prefix; empty defaults to
+// SourceNER.
+func NERConfigFromRaw(detector NERDetector, minScore float32, defaultAction string, entityActions map[string]string, source string) NERConfig {
+	if source == "" {
+		source = SourceNER
+	}
 	return NERConfig{
 		Detector:      detector,
 		MinScore:      minScore,
 		DefaultAction: validActionOr(defaultAction, ActionMask),
 		EntityActions: validActions(entityActions),
+		Source:        source,
 	}
 }
 
-// nerPatternID returns the synthetic pattern ID that audit rows carry
-// for NER hits. Prefixing with "ner:" keeps these distinguishable from
-// regex pattern IDs in the events tab and in filter queries; admins
-// can switch off a single entity type with the same Disabled-pattern
-// machinery used for regex.
-func nerPatternID(group string) string {
-	return "ner:" + group
+// patternID returns the synthetic pattern ID that audit rows and masks carry
+// for this detector's hits, e.g. "ner:EMAIL" or "pattern:ANTHROPIC_KEY". The
+// source prefix keeps neural and deterministic detections distinguishable in
+// the events tab and in pattern_id filter queries.
+func (c NERConfig) patternID(group string) string {
+	source := c.Source
+	if source == "" {
+		source = SourceNER
+	}
+	return source + ":" + group
 }
 
 // errNERDetector is a NERDetector that always returns the wrapped

core/services/routing/pii/ner_test.go

@@ -131,23 +131,31 @@ var _ = Describe("RedactNER", func() {
 var _ = Describe("NERConfigFromRaw", func() {
 	det := &stubNERDetector{}
 
-	It("defaults an empty default_action to mask", func() {
-		cfg := NERConfigFromRaw(det, 0.4, "", nil)
+	It("defaults an empty default_action to mask and an empty source to ner", func() {
+		cfg := NERConfigFromRaw(det, 0.4, "", nil, "")
 		Expect(cfg.DefaultAction).To(Equal(ActionMask))
 		Expect(cfg.MinScore).To(BeNumerically("~", 0.4, 1e-6))
+		Expect(cfg.Source).To(Equal(SourceNER))
+		Expect(cfg.patternID("EMAIL")).To(Equal("ner:EMAIL"))
 	})
 
 	It("passes through valid actions and drops invalid ones", func() {
 		cfg := NERConfigFromRaw(det, 0, "block", map[string]string{
 			"PASSWORD": "block",
 			"EMAIL":    "mask",
 			"BOGUS":    "nonsense", // dropped
-		})
+		}, SourceNER)
 		Expect(cfg.DefaultAction).To(Equal(ActionBlock))
 		Expect(cfg.EntityActions).To(HaveKeyWithValue("PASSWORD", ActionBlock))
 		Expect(cfg.EntityActions).To(HaveKeyWithValue("EMAIL", ActionMask))
 		Expect(cfg.EntityActions).NotTo(HaveKey("BOGUS"))
 	})
+
+	It("prefixes pattern-detector hits with the pattern source", func() {
+		cfg := NERConfigFromRaw(det, 0, "mask", nil, SourcePattern)
+		Expect(cfg.Source).To(Equal(SourcePattern))
+		Expect(cfg.patternID("ANTHROPIC_KEY")).To(Equal("pattern:ANTHROPIC_KEY"))
+	})
 })
 
 var _ = Describe("NERConfig.ResolveAction", func() {

core/services/routing/pii/redactor.go

@@ -111,7 +111,7 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit,
 			"group", e.Group, "score", e.Score, "action", action,
 			"start", e.Start, "end", e.End, "text", e.Text)
 		hits = append(hits, rawHit{
-			patternID: nerPatternID(e.Group),
+			patternID: cfg.patternID(e.Group),
 			action:    action,
 			start:     e.Start,
 			end:       e.End,

core/services/routing/pii/redactor_test.go

@@ -30,6 +30,20 @@ var _ = Describe("RedactNER emission", func() {
 		Expect(res.Spans[0].HashPrefix).NotTo(BeEmpty(), "hash prefix must be set so audits can dedupe leaks")
 	})
 
+	It("labels pattern-detector hits with the pattern source, not ner", func() {
+		cfgs := []NERConfig{{
+			Detector:      &stubNERDetector{entities: []NEREntity{{Group: "ANTHROPIC_KEY", Start: 4, End: 24, Score: 1}}},
+			EntityActions: map[string]Action{"ANTHROPIC_KEY": ActionMask},
+			Source:        SourcePattern,
+		}}
+		res, err := RedactNER(ctx, "use sk-ant-aaaaaaaaaaaaaaaa now", cfgs)
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(ContainSubstring("[REDACTED:pattern:ANTHROPIC_KEY]"))
+		Expect(res.Redacted).NotTo(ContainSubstring("[REDACTED:ner:"))
+		Expect(res.Spans).To(HaveLen(1))
+		Expect(res.Spans[0].Pattern).To(Equal("pattern:ANTHROPIC_KEY"))
+	})
+
 	It("block leaves the matched span intact and sets Blocked", func() {
 		res, err := RedactNER(ctx, "token sk-abcdef here", oneShot("PASSWORD", ActionBlock, 6, 15))
 		Expect(err).NotTo(HaveOccurred())

core/services/routing/pii/types.go

@@ -64,7 +64,7 @@ const (
 type Span struct {
 	Start      int
 	End        int
-	Pattern    string  // synthetic detector id, "ner:<GROUP>"
+	Pattern    string  // synthetic detector id, "<source>:<GROUP>" (e.g. "ner:EMAIL", "pattern:ANTHROPIC_KEY")
 	HashPrefix string  // first 8 chars of sha256(matched value); audit-safe
 	Action     Action  // the action that fired for this span (after merge)
 	Score      float32 // detector confidence for the (winning) hit, 0..1