From e3341d21a0a5cb606ee824b24008e3470af7654c Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Sun, 4 Oct 2026 16:36:49 +0200 Subject: [PATCH] feat(pii): extend a detection over the previous word (extend_to_previous_word) The mirror of extend_to_next_word. A name model often tags the surname with high confidence and scores the first name in front of it below min_score, so "Gast Greta Waldmeister" became "Gast Greta [REDACTED:ner:LASTNAME]". extend_to_previous_word lists entity groups whose accepted detections also cover the word directly before them on the same line. The preceding word must consist of letters (plus '-' and the apostrophe): a word with digits - an amount, a date, a room number - is never taken, nor is a protected term, and the search never crosses a line break. Docs: operations/middleware.md and advanced/model-configuration.md. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Stefan Walcz (cherry picked from commit b1845b8bed085efefaec034c981250e49c2046cb) --- core/application/application.go | 1 + core/config/meta/registry.go | 7 ++ core/config/model_config.go | 14 +++ core/config/model_config_test.go | 11 +++ .../routing/pii/extend_previous_test.go | 94 +++++++++++++++++++ core/services/routing/pii/ner.go | 16 ++++ core/services/routing/pii/protected.go | 12 +++ core/services/routing/pii/redactor.go | 44 +++++++++ docs/content/advanced/model-configuration.md | 2 + docs/content/operations/middleware.md | 14 +++ 10 files changed, 215 insertions(+) create mode 100644 core/services/routing/pii/extend_previous_test.go diff --git a/core/application/application.go b/core/application/application.go index 62c6bbadf..000069722 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -375,6 +375,7 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver { pii.SourceNER, ) nc.ExtendToNextWord = cfg.PIIDetectionExtendToNextWord() + nc.ExtendToPreviousWord = cfg.PIIDetectionExtendToPreviousWord() terms, files := cfg.PIIDetectionProtectedTerms() nc.ProtectedTerms = pii.LoadProtectedTerms(terms, files, a.ApplicationConfig().SystemState.Model.ModelsPath) return nc, true diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 8b37a73a9..14bf794b7 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -1062,6 +1062,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Component: "string-list", Order: 215, }, + "pii_detection.extend_to_previous_word": { + Section: "pii", + Label: "Extend To Previous Word", + Description: "Entity groups (e.g. LASTNAME) whose detections also cover the word right before them on the same line, with the same action, when that word consists of letters only. Use it when the model tags a surname reliably but scores the first name before it below the threshold.", + Component: "string-list", + Order: 215, + }, "pii_detection.protected_terms": { Section: "pii", Label: "Protected Terms", diff --git a/core/config/model_config.go b/core/config/model_config.go index b0d86e105..88c1a48a2 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -661,6 +661,11 @@ type PIIDetectionConfig struct { // with the same action - for a neighbour the model scores unreliably, // such as the town after a postal code. ExtendToNextWord []string `yaml:"extend_to_next_word,omitempty" json:"extend_to_next_word,omitempty"` + // ExtendToPreviousWord lists entity groups (e.g. "LASTNAME") whose accepted + // detections also cover the word right before them on the same line, when + // that word consists of letters only - for a neighbour the model scores + // unreliably, such as the first name before a recognised surname. + ExtendToPreviousWord []string `yaml:"extend_to_previous_word,omitempty" json:"extend_to_previous_word,omitempty"` // ProtectedTerms are values that must pass the filter unchanged even // when the model tags them (e.g. known business names read as // surnames). They are hidden from the detector behind a placeholder and @@ -758,6 +763,15 @@ func (c *ModelConfig) PIIDetectionExtendToNextWord() []string { return append([]string(nil), c.PIIDetection.ExtendToNextWord...) } +// PIIDetectionExtendToPreviousWord returns the entity groups whose detections +// are stretched over the preceding word (a fresh copy). +func (c *ModelConfig) PIIDetectionExtendToPreviousWord() []string { + if len(c.PIIDetection.ExtendToPreviousWord) == 0 { + return nil + } + return append([]string(nil), c.PIIDetection.ExtendToPreviousWord...) +} + // PIIDetectionProtectedTerms returns the inline protected terms and the // configured term files (fresh copies). The files are read by the pii // package (pii.LoadProtectedTerms), relative to the models path. diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index f82bed656..57e5caccb 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -820,6 +820,17 @@ var _ = Describe("PII config accessors", func() { Expect(cfg.PIIDetection.ExtendToNextWord[0]).To(Equal("ZIPCODE"), "accessor must return a fresh slice") }) + It("unmarshals pii_detection.extend_to_previous_word from YAML", func() { + var cfg ModelConfig + raw := []byte("name: pf\npii_detection:\n extend_to_previous_word: [LASTNAME]\n") + Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed()) + Expect(cfg.PIIDetectionExtendToPreviousWord()).To(Equal([]string{"LASTNAME"})) + Expect(cfg.PIIDetectionExtendToNextWord()).To(BeNil(), "the two directions are independent") + got := cfg.PIIDetectionExtendToPreviousWord() + got[0] = "FIRSTNAME" + Expect(cfg.PIIDetection.ExtendToPreviousWord[0]).To(Equal("LASTNAME"), "accessor must return a fresh slice") + }) + It("unmarshals pii_detection.protected_terms and protected_terms_files from YAML", func() { var cfg ModelConfig raw := []byte("name: pf\npii_detection:\n protected_terms: [Hotel Seeblick]\n protected_terms_files: [pf/terms.txt]\n") diff --git a/core/services/routing/pii/extend_previous_test.go b/core/services/routing/pii/extend_previous_test.go new file mode 100644 index 000000000..02a521b2c --- /dev/null +++ b/core/services/routing/pii/extend_previous_test.go @@ -0,0 +1,94 @@ +package pii + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ExtendToPreviousWord", func() { + ctx := context.Background() + + // A hotel receipt as a name model sees it: the surname is tagged with high + // confidence, the first name in front of it scores below the threshold. + const receipt = "Hotel Seeblick, Gast Greta Waldmeister, Kastanienallee 97\nBetrag 12,50 EUR" + cfg := func(text string, extend ...string) []NERConfig { + return []NERConfig{{ + Detector: &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.97)}}, + DefaultAction: ActionMask, + ExtendToPreviousWord: extend, + }} + } + + It("leaves the first name in the clear without the option (the gap it closes)", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt)) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(ContainSubstring("Gast Greta [REDACTED:ner:LASTNAME]")) + }) + + It("masks the word before a configured group together with it", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt, "LASTNAME")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).NotTo(ContainSubstring("Greta")) + Expect(res.Redacted).To(ContainSubstring("Gast [REDACTED:ner:LASTNAME], Kastanienallee")) + s, _ := span(receipt, "Greta") + _, e := span(receipt, "Waldmeister") + Expect(res.Spans).To(ContainElement(SatisfyAll( + HaveField("Start", s), HaveField("End", e), HaveField("Pattern", "ner:LASTNAME")))) + }) + + It("only extends the configured groups", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt, "FIRSTNAME")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(ContainSubstring("Greta")) + }) + + It("never swallows a word with digits (amounts, dates, codes stay intact)", func() { + for _, text := range []string{"Betrag 12,50 Waldmeister", "am 11.08.2026 Waldmeister", "Zimmer 4711 Waldmeister", "Code A1b Waldmeister"} { + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(HaveSuffix(" [REDACTED:ner:LASTNAME]"), text) + Expect(len(res.Redacted)).To(BeNumerically(">", len(" [REDACTED:ner:LASTNAME]")), text) + Expect(res.Spans).To(HaveLen(1)) + Expect(res.Spans[0].Start).To(Equal(len(text)-len("Waldmeister")), "kept the word before: %q", text) + } + }) + + It("does not cross a line break, and leaves the previous line intact", func() { + text := "Greta\nWaldmeister" + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Greta\n[REDACTED:ner:LASTNAME]")) + }) + + It("does nothing at the start of the text or after punctuation only", func() { + for _, text := range []string{"Waldmeister zahlt", ", Waldmeister zahlt"} { + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(HaveSuffix("[REDACTED:ner:LASTNAME] zahlt"), text) + Expect(res.Spans[0].Start).To(Equal(len(text)-len("Waldmeister zahlt")), text) + } + }) + + It("handles non-ASCII words and hyphens", func() { + text := "Gast Anne-Sophie Müller-Lüdenscheidt" + det := &stubNERDetector{entities: []NEREntity{ent(text, "Müller-Lüdenscheidt", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Gast [REDACTED:ner:LASTNAME]")) + }) + + It("never swallows a protected term in front of the hit", func() { + text := "Seeblick Waldmeister" + // tags what it actually sees: after shielding, the protected term is a placeholder + det := &lexiconDetector{words: map[string]NEREntity{"Waldmeister": {Group: "LASTNAME", Score: 0.9}}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, + ExtendToPreviousWord: []string{"LASTNAME"}, ProtectedTerms: []string{"Seeblick"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Seeblick [REDACTED:ner:LASTNAME]")) + }) +}) diff --git a/core/services/routing/pii/ner.go b/core/services/routing/pii/ner.go index 65e4427d4..50b3b0647 100644 --- a/core/services/routing/pii/ner.go +++ b/core/services/routing/pii/ner.go @@ -79,6 +79,16 @@ type NERConfig struct { // the code alone leaves the town in the clear. Empty disables it. ExtendToNextWord []string + // ExtendToPreviousWord is the mirror of ExtendToNextWord: accepted + // detections of these groups also cover the word directly before them on + // the same line, but only when that word consists of letters (and the + // in-word joiners '-' and the apostrophe) - never digits, so amounts, + // dates and codes in front of a hit stay intact. It exists for a name + // model that tags the surname with high confidence and scores the first + // name in front of it below the threshold ("Gast Greta Waldmeister" + // -> "Gast Greta [REDACTED]"). Protected terms are never swallowed. + ExtendToPreviousWord []string + // ProtectedTerms are values that must reach the model unchanged even // though the detector would tag them, e.g. known business names that a // name model reads as surnames ("Hotel Seeblick" -> LASTNAME). Each @@ -138,6 +148,12 @@ func (c NERConfig) extendsToNextWord(group string) bool { return slices.Contains(c.ExtendToNextWord, group) } +// extendsToPreviousWord reports whether accepted hits of group are stretched +// over the preceding word (see NERConfig.ExtendToPreviousWord). +func (c NERConfig) extendsToPreviousWord(group string) bool { + return slices.Contains(c.ExtendToPreviousWord, group) +} + // patternID returns the synthetic pattern ID that audit rows and masks carry // for this detector's hits, e.g. "ner:EMAIL" or "pattern:ANTHROPIC_KEY". The // source prefix keeps neural and deterministic detections distinguishable in diff --git a/core/services/routing/pii/protected.go b/core/services/routing/pii/protected.go index 2e0111f3f..482cca258 100644 --- a/core/services/routing/pii/protected.go +++ b/core/services/routing/pii/protected.go @@ -250,6 +250,18 @@ func keepExtension(end, extended int, regions []shield) int { return extended } +// keepExtensionBefore is keepExtension for a hit stretched backwards to +// extended (< start): the extension is dropped when a protected region lies +// in the added range, so a protected term in front of a hit is never masked. +func keepExtensionBefore(start, extended int, regions []shield) int { + for _, r := range regions { + if r.origEnd > extended && r.origEnd <= start { + return start + } + } + return extended +} + type protectedFileEntry struct { modTime time.Time size int64 diff --git a/core/services/routing/pii/redactor.go b/core/services/routing/pii/redactor.go index ee08a496c..1ff467a84 100644 --- a/core/services/routing/pii/redactor.go +++ b/core/services/routing/pii/redactor.go @@ -214,6 +214,9 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, if cfg.extendsToNextWord(e.Group) && i == len(pieces)-1 { end = keepExtension(end, nextWordEnd(text, end), shields) } + if cfg.extendsToPreviousWord(e.Group) && i == 0 { + start = keepExtensionBefore(start, prevWordStart(text, start), shields) + } xlog.Debug("pii/ner: detection accepted", "group", e.Group, "score", e.Score, "action", action, "start", start, "end", end, "detected_end", pc[1], "text", e.Text) @@ -290,6 +293,47 @@ func nextWordEnd(text string, end int) int { return i } +// prevWordStart returns the start offset of the word that precedes +// text[start:]: horizontal whitespace before start is skipped (a line break +// ends the search), then letters, combining marks and the in-word joiners '-' +// and the apostrophe are consumed backwards. The word must consist of letters: +// when it touches a digit (an amount, a date, a code) or no word precedes, +// start is returned unchanged, so a hit is never stretched over a number, +// bare whitespace or onto the previous line. +func prevWordStart(text string, start int) int { + i := start + for i > 0 { + r, size := utf8.DecodeLastRuneInString(text[:i]) + if r == '\n' || r == '\r' || !unicode.IsSpace(r) { + break + } + i -= size + } + wordEnd := i + letters := 0 + for i > 0 { + r, size := utf8.DecodeLastRuneInString(text[:i]) + if unicode.IsLetter(r) { + letters++ + } else if !unicode.IsMark(r) && r != '-' && r != '\'' { + break + } + i -= size + } + if i > 0 { + if r, _ := utf8.DecodeLastRuneInString(text[:i]); unicode.IsDigit(r) { + return start // the word is glued to a number: leave it alone + } + } + for i < wordEnd && (text[i] == '-' || text[i] == '\'') { + i++ + } + if i == wordEnd || letters == 0 { + return start + } + return i +} + // mergeAndEmit handles the overlap-merge + masked-output step. Sorts by // start (stable on equal starts by descending action strength), drops // overlapping hits in favour of the stronger action, and walks the text diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index eaf4d9985..04454a43d 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -1153,6 +1153,8 @@ PII redaction is NER-based and runs on the **request** (input) side. It has two EMAIL: mask extend_to_next_word: # optional: also mask the word right after these - ZIPCODE # groups on the same line (e.g. the town after a postal code) + extend_to_previous_word: # optional: also mask the letters-only word right before these + - LASTNAME # groups on the same line (e.g. the first name before a surname) protected_terms: # optional: values that must pass unchanged even if tagged - Hotel Seeblick # (plus protected_terms_files: one term per line, in the models path) ``` diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index aca749e72..406ae467c 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -128,6 +128,20 @@ date (`ZIPCODE` followed by `12 Mar 2026` takes `12`), so it trades some over-masking for fewer leaks. Use it only for groups whose neighbour is identifying. +`extend_to_previous_word` is the mirror for the word *before* a detection. A +name model often tags the surname with high confidence and scores the first +name in front of it below `min_score` (`Gast Greta [REDACTED:ner:LASTNAME]`): + +```yaml +pii_detection: + extend_to_previous_word: + - LASTNAME # "Gast Greta Waldmeister" -> "Gast [REDACTED:ner:LASTNAME]" +``` + +Here the preceding word must consist of letters (plus `-` and `'`): a word +with digits - an amount, a date, a room number - is never taken, nor a +protected term. The search skips spaces and tabs but never a line break. + #### Protected terms The opposite failure is a value the model tags although it must reach the