feat(pii): extend a detection over the next word (extend_to_next_word)

Address models tend to tag a postal code with high confidence but miss
or under-score the town right after it. Measured with
privacy-filter-multilingual on 24 synthetic German invoices: the postal
code was masked every time, while the town after it stayed in the clear
in 6 of 18 cases (4 not detected at all, 2 tagged CITY at 0.43/0.47,
below the configured floor). Masking the code alone leaves the pair half
redacted: "[REDACTED:ner:ZIPCODE] Ludwigshafen".

pii_detection.extend_to_next_word lists entity groups whose accepted
detections also cover the word that directly follows them on the same
line, with the same action and group. Horizontal whitespace is skipped,
a line break ends the search, and nothing is extended when no word
follows, so a hit never grows over bare whitespace or onto the next
line (or across the segment separator on the multi-message path).

The option is opt-in and empty by default; existing detector configs
behave exactly as before. Covered by Ginkgo specs in
core/services/routing/pii/extend_test.go (fail without the change), a
YAML round-trip spec, a registry entry for the model editor, and the
middleware and model-configuration docs.

Assisted-by: Claude Code:claude-opus-5-5
Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>
(cherry picked from commit c67c0782f8)

core/application/application.go

@@ -367,13 +367,15 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver {
 		}
 
 		det := piidetector.New(a.ModelLoader(), cfg, a.ApplicationConfig())
-		return pii.NERConfigFromRaw(
+		nc := pii.NERConfigFromRaw(
 			det,
 			cfg.PIIDetectionMinScore(),
 			cfg.PIIDetectionDefaultAction(),
 			cfg.PIIDetectionEntityActions(),
 			pii.SourceNER,
-		), true
+		)
+		nc.ExtendToNextWord = cfg.PIIDetectionExtendToNextWord()
+		return nc, true
 	}
 }
 

core/config/meta/registry.go

@@ -1055,6 +1055,13 @@ func DefaultRegistry() map[string]FieldMetaOverride {
 			Component:   "pii-pattern-list",
 			Order:       214,
 		},
+		"pii_detection.extend_to_next_word": {
+			Section:     "pii",
+			Label:       "Extend To Next Word",
+			Description: "Entity groups (e.g. ZIPCODE) whose detections also cover the word right after them on the same line, with the same action. Use it when the model tags one part of an identifying pair reliably but misses its neighbour, such as the town after a postal code.",
+			Component:   "string-list",
+			Order:       215,
+		},
 
 		// --- Cloud passthrough proxy ---
 		// These only have an effect when Backend is set to

core/config/model_config.go

@@ -653,6 +653,11 @@ type PIIDetectionConfig struct {
 	// subset (validated at load). Each match is reported under its Name as the
 	// entity group, so EntityActions/DefaultAction apply by Name.
 	Patterns []PIIPattern `yaml:"patterns,omitempty" json:"patterns,omitempty"`
+	// ExtendToNextWord lists entity groups (e.g. "ZIPCODE") whose accepted
+	// detections also cover the word right after them on the same line,
+	// with the same action - for a neighbour the model scores unreliably,
+	// such as the town after a postal code.
+	ExtendToNextWord []string `yaml:"extend_to_next_word,omitempty" json:"extend_to_next_word,omitempty"`
 }
 
 // PIIPattern is one operator-defined pattern on a pattern detector model. Name
@@ -732,6 +737,15 @@ func (c *ModelConfig) PIIDetectionEntityActions() map[string]string {
 	return out
 }
 
+// PIIDetectionExtendToNextWord returns the entity groups whose detections
+// are stretched over the following word (a fresh copy).
+func (c *ModelConfig) PIIDetectionExtendToNextWord() []string {
+	if len(c.PIIDetection.ExtendToNextWord) == 0 {
+		return nil
+	}
+	return append([]string(nil), c.PIIDetection.ExtendToNextWord...)
+}
+
 // IsPatternDetector reports whether this detector model matches secrets with
 // regex patterns (built-in and/or operator-defined) rather than a neural NER
 // model. Such a model runs entirely in-process (no backend / GGUF / VRAM); the

core/config/model_config_test.go

@@ -809,6 +809,16 @@ var _ = Describe("PII config accessors", func() {
 		Expect(cfg.PIIDetectionDefaultAction()).To(Equal("mask"))
 		Expect(cfg.PIIDetectionEntityActions()).To(HaveKeyWithValue("PASSWORD", "block"))
 	})
+
+	It("unmarshals pii_detection.extend_to_next_word from YAML", func() {
+		var cfg ModelConfig
+		raw := []byte("name: pf\npii_detection:\n  extend_to_next_word: [ZIPCODE]\n")
+		Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed())
+		Expect(cfg.PIIDetectionExtendToNextWord()).To(Equal([]string{"ZIPCODE"}))
+		got := cfg.PIIDetectionExtendToNextWord()
+		got[0] = "CITY"
+		Expect(cfg.PIIDetection.ExtendToNextWord[0]).To(Equal("ZIPCODE"), "accessor must return a fresh slice")
+	})
 })
 
 var _ = Describe("GGUF importer chat-default guard (reservedNonChatModel)", func() {

core/services/routing/pii/extend_test.go

@@ -0,0 +1,127 @@
+package pii
+
+import (
+	"context"
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// span returns the byte range of the first occurrence of sub in text, so the
+// stub detections below line up with the text they describe.
+func span(text, sub string) (int, int) {
+	i := strings.Index(text, sub)
+	Expect(i).To(BeNumerically(">=", 0), "fixture: %q not in text", sub)
+	return i, i + len(sub)
+}
+
+func ent(text, sub, group string, score float32) NEREntity {
+	s, e := span(text, sub)
+	return NEREntity{Group: group, Start: s, End: e, Score: score, Text: sub}
+}
+
+var _ = Describe("ExtendToNextWord", func() {
+	ctx := context.Background()
+
+	// An invoice line as an address model sees it: the postal code is tagged
+	// with high confidence, the town after it is not tagged at all.
+	const invoice = "Rechnung an Lena Kranzberger, Ahornweg 12, 67059 Ludwigshafen\nTel. 0612 345678"
+	detections := func(text string) []NEREntity {
+		return []NEREntity{
+			ent(text, "Lena", "FIRSTNAME", 0.95),
+			ent(text, "Kranzberger", "LASTNAME", 0.97),
+			ent(text, "67059", "ZIPCODE", 0.96),
+		}
+	}
+	cfg := func(text string, extend ...string) []NERConfig {
+		return []NERConfig{{
+			Detector:         &stubNERDetector{entities: detections(text)},
+			DefaultAction:    ActionMask,
+			ExtendToNextWord: extend,
+		}}
+	}
+
+	It("leaves the town in the clear without the option (the gap it closes)", func() {
+		res, err := RedactNER(ctx, invoice, cfg(invoice))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(ContainSubstring("Ludwigshafen"))
+	})
+
+	It("masks the word after a configured group together with it", func() {
+		res, err := RedactNER(ctx, invoice, cfg(invoice, "ZIPCODE"))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).NotTo(ContainSubstring("Ludwigshafen"))
+		Expect(res.Redacted).NotTo(ContainSubstring("67059"))
+		Expect(res.Redacted).To(ContainSubstring(", [REDACTED:ner:ZIPCODE]\nTel."), "stops at the line break")
+		s, _ := span(invoice, "67059")
+		_, e := span(invoice, "Ludwigshafen")
+		Expect(res.Spans).To(ContainElement(SatisfyAll(
+			HaveField("Start", s), HaveField("End", e), HaveField("Pattern", "ner:ZIPCODE"))))
+	})
+
+	It("only extends the configured groups", func() {
+		res, err := RedactNER(ctx, invoice, cfg(invoice, "CITY"))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(ContainSubstring("[REDACTED:ner:LASTNAME], Ahornweg"), "a name is not stretched")
+		Expect(res.Redacted).To(ContainSubstring("Ludwigshafen"))
+	})
+
+	It("does not cross a line break, and leaves the next line intact", func() {
+		text := "PLZ 67059\nLudwigshafen"
+		det := &stubNERDetector{entities: []NEREntity{ent(text, "67059", "ZIPCODE", 0.9)}}
+		res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToNextWord: []string{"ZIPCODE"}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("PLZ [REDACTED:ner:ZIPCODE]\nLudwigshafen"))
+	})
+
+	It("handles non-ASCII words, hyphens and a sentence-ending dot", func() {
+		text := "Lieferadresse 97070 Würzburg-Heidingsfeld. Danke"
+		det := &stubNERDetector{entities: []NEREntity{ent(text, "97070", "ZIPCODE", 0.9)}}
+		res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToNextWord: []string{"ZIPCODE"}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("Lieferadresse [REDACTED:ner:ZIPCODE]. Danke"))
+	})
+
+	It("does not extend over trailing whitespace or punctuation only", func() {
+		text := "PLZ 67059 , Rest"
+		det := &stubNERDetector{entities: []NEREntity{ent(text, "67059", "ZIPCODE", 0.9)}}
+		res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToNextWord: []string{"ZIPCODE"}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("PLZ [REDACTED:ner:ZIPCODE] , Rest"))
+	})
+
+	It("merges with a town the model did detect", func() {
+		text := "67059 Ludwigshafen am Rhein"
+		det := &stubNERDetector{entities: []NEREntity{
+			ent(text, "67059", "ZIPCODE", 0.9),
+			ent(text, "Ludwigshafen", "CITY", 0.6),
+		}}
+		res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToNextWord: []string{"ZIPCODE"}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("[REDACTED:ner:ZIPCODE] am Rhein"))
+		Expect(res.Spans).To(HaveLen(1))
+	})
+
+	It("applies on the segment path without crossing the segment separator", func() {
+		texts := []string{"Kunde: 67059", "Ludwigshafen ist schön"}
+		doc := strings.Join(texts, segmentSeparator)
+		det := &stubNERDetector{entities: []NEREntity{ent(doc, "67059", "ZIPCODE", 0.9)}}
+		res, err := RedactNERSegments(ctx, texts, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToNextWord: []string{"ZIPCODE"}}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res[0].Redacted).To(Equal("Kunde: [REDACTED:ner:ZIPCODE]"))
+		Expect(res[1].Redacted).To(Equal("Ludwigshafen ist schön"))
+	})
+
+	It("keeps a block action on the extended span", func() {
+		text := "Code 4711 Geheim"
+		det := &stubNERDetector{entities: []NEREntity{ent(text, "4711", "PIN", 0.9)}}
+		res, err := RedactNER(ctx, text, []NERConfig{{
+			Detector: det, EntityActions: map[string]Action{"PIN": ActionBlock}, ExtendToNextWord: []string{"PIN"},
+		}})
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Blocked).To(BeTrue())
+		_, e := span(text, "Geheim")
+		Expect(res.Spans[0].End).To(Equal(e))
+	})
+})

core/services/routing/pii/ner.go

@@ -3,6 +3,7 @@ package pii
 import (
 	"context"
 	"fmt"
+	"slices"
 )
 
 // NERDetector is the contract the redactor's encoder/NER tier expects.
@@ -67,6 +68,16 @@ type NERConfig struct {
 	// (Source "pattern") are told apart in the events log and to the model.
 	// Empty defaults to "ner" for backward compatibility.
 	Source string
+
+	// ExtendToNextWord lists entity groups whose accepted detections are
+	// stretched over the word that directly follows them on the same line
+	// (horizontal whitespace in between, no line break). The extension
+	// carries the hit's action and group. It exists for entities whose
+	// neighbour is identifying but scored unreliably: an address model
+	// that tags the postal code with high confidence often misses or
+	// under-scores the town after it ("67059 Ludwigshafen"), and masking
+	// the code alone leaves the town in the clear. Empty disables it.
+	ExtendToNextWord []string
 }
 
 // Detector source labels (the PatternID prefix). Kept short and stable —
@@ -112,6 +123,12 @@ func NERConfigFromRaw(detector NERDetector, minScore float32, defaultAction stri
 	}
 }
 
+// extendsToNextWord reports whether accepted hits of group are stretched
+// over the following word (see NERConfig.ExtendToNextWord).
+func (c NERConfig) extendsToNextWord(group string) bool {
+	return slices.Contains(c.ExtendToNextWord, group)
+}
+
 // patternID returns the synthetic pattern ID that audit rows and masks carry
 // for this detector's hits, e.g. "ner:EMAIL" or "pattern:ANTHROPIC_KEY". The
 // source prefix keeps neural and deterministic detections distinguishable in

core/services/routing/pii/redactor.go

@@ -6,6 +6,8 @@ import (
 	"encoding/hex"
 	"sort"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/mudler/xlog"
 )
@@ -187,20 +189,56 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit,
 				"group", e.Group, "start", e.Start, "end", e.End, "text_len", len(text))
 			continue
 		}
+		end := e.End
+		if cfg.extendsToNextWord(e.Group) {
+			end = nextWordEnd(text, e.End)
+		}
 		xlog.Debug("pii/ner: detection accepted",
 			"group", e.Group, "score", e.Score, "action", action,
-			"start", e.Start, "end", e.End, "text", e.Text)
+			"start", e.Start, "end", e.End, "extended_end", end, "text", e.Text)
 		hits = append(hits, rawHit{
 			patternID: cfg.patternID(e.Group),
 			action:    action,
 			start:     e.Start,
-			end:       e.End,
+			end:       end,
 			score:     e.Score,
 		})
 	}
 	return hits, nil
 }
 
+// nextWordEnd returns the end offset of the word that follows text[:end]:
+// horizontal whitespace is skipped (a line break ends the search), then
+// letters, digits, combining marks and the in-word joiners '-', '.' and
+// the apostrophe are consumed; trailing dots are left out (sentence end). When no
+// word follows, end is returned unchanged, so a hit is never stretched
+// over bare whitespace or onto the next line.
+func nextWordEnd(text string, end int) int {
+	i := end
+	for i < len(text) {
+		r, size := utf8.DecodeRuneInString(text[i:])
+		if r == '\n' || r == '\r' || !unicode.IsSpace(r) {
+			break
+		}
+		i += size
+	}
+	wordStart := i
+	for i < len(text) {
+		r, size := utf8.DecodeRuneInString(text[i:])
+		if !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r) && r != '-' && r != '.' && r != '\'' {
+			break
+		}
+		i += size
+	}
+	for i > wordStart && text[i-1] == '.' {
+		i--
+	}
+	if i == wordStart {
+		return end
+	}
+	return i
+}
+
 // mergeAndEmit handles the overlap-merge + masked-output step. Sorts by
 // start (stable on equal starts by descending action strength), drops
 // overlapping hits in favour of the stronger action, and walks the text

docs/content/advanced/model-configuration.md

@@ -1151,6 +1151,8 @@ PII redaction is NER-based and runs on the **request** (input) side. It has two
       PASSWORD: block
       CREDITCARD: block
       EMAIL: mask
+    extend_to_next_word:      # optional: also mask the word right after these
+      - ZIPCODE               # groups on the same line (e.g. the town after a postal code)
   ```
 
 - **Consuming models** opt in and reference one or more detectors by name - no per-consumer policy:

docs/content/operations/middleware.md

@@ -101,6 +101,28 @@ but leaves the text unchanged. The entity-group names are whatever the model
 emits (the privacy-filter family uses uppercase names like `EMAIL`,
 `PASSWORD`, `CREDITCARD`).
 
+#### Extending a detection to the next word
+
+Some identifying values come in pairs where the model reliably tags one half
+and misses the other. The typical case is an address line: the postal code is
+detected with high confidence, the town right after it is tagged with a low
+score or not at all, and masking the code alone leaves the town in the clear
+(`[REDACTED:ner:ZIPCODE] Ludwigshafen`). `extend_to_next_word` lists entity
+groups whose accepted detections also cover the word that directly follows
+them on the same line, with the same action and group:
+
+```yaml
+pii_detection:
+  min_score: 0.4
+  extend_to_next_word:
+    - ZIPCODE                 # "67059 Ludwigshafen" -> "[REDACTED:ner:ZIPCODE]"
+```
+
+A word is a run of letters, digits, combining marks, `-`, `'` and inner dots
+(a trailing dot is left out). The extension skips spaces and tabs but never a
+line break, and it does nothing when no word follows. Only one word is added,
+so a multi-word town ("Frankfurt am Main") keeps its tail.
+
 ### Pattern detector tier
 
 NER is the wrong tool for high-entropy, highly-regular **secrets** - API keys,