diff --git a/core/application/application.go b/core/application/application.go index f519f5ef1..f5182d13d 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -372,6 +372,7 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver { pii.SourceNER, ) nc.ExtendToNextWord = cfg.PIIDetectionExtendToNextWord() + nc.ExtendToPreviousWord = cfg.PIIDetectionExtendToPreviousWord() terms, files := cfg.PIIDetectionProtectedTerms() nc.ProtectedTerms = pii.LoadProtectedTerms(terms, files, a.ApplicationConfig().SystemState.Model.ModelsPath) return nc, true diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 91ce5e30c..534bff5c9 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -1062,6 +1062,13 @@ func DefaultRegistry() map[string]FieldMetaOverride { Component: "string-list", Order: 215, }, + "pii_detection.extend_to_previous_word": { + Section: "pii", + Label: "Extend To Previous Word", + Description: "Entity groups (e.g. LASTNAME) whose detections also cover the word right before them on the same line, with the same action, when that word consists of letters only. Use it when the model tags a surname reliably but scores the first name before it below the threshold.", + Component: "string-list", + Order: 215, + }, "pii_detection.protected_terms": { Section: "pii", Label: "Protected Terms", diff --git a/core/config/model_config.go b/core/config/model_config.go index 2fd9345d4..16431aaeb 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -656,6 +656,11 @@ type PIIDetectionConfig struct { // with the same action - for a neighbour the model scores unreliably, // such as the town after a postal code. ExtendToNextWord []string `yaml:"extend_to_next_word,omitempty" json:"extend_to_next_word,omitempty"` + // ExtendToPreviousWord lists entity groups (e.g. "LASTNAME") whose accepted + // detections also cover the word right before them on the same line, when + // that word consists of letters only - for a neighbour the model scores + // unreliably, such as the first name before a recognised surname. + ExtendToPreviousWord []string `yaml:"extend_to_previous_word,omitempty" json:"extend_to_previous_word,omitempty"` // ProtectedTerms are values that must pass the filter unchanged even // when the model tags them (e.g. known business names read as // surnames). They are hidden from the detector behind a placeholder and @@ -753,6 +758,15 @@ func (c *ModelConfig) PIIDetectionExtendToNextWord() []string { return append([]string(nil), c.PIIDetection.ExtendToNextWord...) } +// PIIDetectionExtendToPreviousWord returns the entity groups whose detections +// are stretched over the preceding word (a fresh copy). +func (c *ModelConfig) PIIDetectionExtendToPreviousWord() []string { + if len(c.PIIDetection.ExtendToPreviousWord) == 0 { + return nil + } + return append([]string(nil), c.PIIDetection.ExtendToPreviousWord...) +} + // PIIDetectionProtectedTerms returns the inline protected terms and the // configured term files (fresh copies). The files are read by the pii // package (pii.LoadProtectedTerms), relative to the models path. diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index b68023a97..81586f299 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -801,6 +801,17 @@ var _ = Describe("PII config accessors", func() { Expect(cfg.PIIDetection.ExtendToNextWord[0]).To(Equal("ZIPCODE"), "accessor must return a fresh slice") }) + It("unmarshals pii_detection.extend_to_previous_word from YAML", func() { + var cfg ModelConfig + raw := []byte("name: pf\npii_detection:\n extend_to_previous_word: [LASTNAME]\n") + Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed()) + Expect(cfg.PIIDetectionExtendToPreviousWord()).To(Equal([]string{"LASTNAME"})) + Expect(cfg.PIIDetectionExtendToNextWord()).To(BeNil(), "the two directions are independent") + got := cfg.PIIDetectionExtendToPreviousWord() + got[0] = "FIRSTNAME" + Expect(cfg.PIIDetection.ExtendToPreviousWord[0]).To(Equal("LASTNAME"), "accessor must return a fresh slice") + }) + It("unmarshals pii_detection.protected_terms and protected_terms_files from YAML", func() { var cfg ModelConfig raw := []byte("name: pf\npii_detection:\n protected_terms: [Hotel Seeblick]\n protected_terms_files: [pf/terms.txt]\n") diff --git a/core/services/routing/pii/extend_previous_test.go b/core/services/routing/pii/extend_previous_test.go new file mode 100644 index 000000000..02a521b2c --- /dev/null +++ b/core/services/routing/pii/extend_previous_test.go @@ -0,0 +1,94 @@ +package pii + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ExtendToPreviousWord", func() { + ctx := context.Background() + + // A hotel receipt as a name model sees it: the surname is tagged with high + // confidence, the first name in front of it scores below the threshold. + const receipt = "Hotel Seeblick, Gast Greta Waldmeister, Kastanienallee 97\nBetrag 12,50 EUR" + cfg := func(text string, extend ...string) []NERConfig { + return []NERConfig{{ + Detector: &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.97)}}, + DefaultAction: ActionMask, + ExtendToPreviousWord: extend, + }} + } + + It("leaves the first name in the clear without the option (the gap it closes)", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt)) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(ContainSubstring("Gast Greta [REDACTED:ner:LASTNAME]")) + }) + + It("masks the word before a configured group together with it", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt, "LASTNAME")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).NotTo(ContainSubstring("Greta")) + Expect(res.Redacted).To(ContainSubstring("Gast [REDACTED:ner:LASTNAME], Kastanienallee")) + s, _ := span(receipt, "Greta") + _, e := span(receipt, "Waldmeister") + Expect(res.Spans).To(ContainElement(SatisfyAll( + HaveField("Start", s), HaveField("End", e), HaveField("Pattern", "ner:LASTNAME")))) + }) + + It("only extends the configured groups", func() { + res, err := RedactNER(ctx, receipt, cfg(receipt, "FIRSTNAME")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(ContainSubstring("Greta")) + }) + + It("never swallows a word with digits (amounts, dates, codes stay intact)", func() { + for _, text := range []string{"Betrag 12,50 Waldmeister", "am 11.08.2026 Waldmeister", "Zimmer 4711 Waldmeister", "Code A1b Waldmeister"} { + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(HaveSuffix(" [REDACTED:ner:LASTNAME]"), text) + Expect(len(res.Redacted)).To(BeNumerically(">", len(" [REDACTED:ner:LASTNAME]")), text) + Expect(res.Spans).To(HaveLen(1)) + Expect(res.Spans[0].Start).To(Equal(len(text)-len("Waldmeister")), "kept the word before: %q", text) + } + }) + + It("does not cross a line break, and leaves the previous line intact", func() { + text := "Greta\nWaldmeister" + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Greta\n[REDACTED:ner:LASTNAME]")) + }) + + It("does nothing at the start of the text or after punctuation only", func() { + for _, text := range []string{"Waldmeister zahlt", ", Waldmeister zahlt"} { + det := &stubNERDetector{entities: []NEREntity{ent(text, "Waldmeister", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(HaveSuffix("[REDACTED:ner:LASTNAME] zahlt"), text) + Expect(res.Spans[0].Start).To(Equal(len(text)-len("Waldmeister zahlt")), text) + } + }) + + It("handles non-ASCII words and hyphens", func() { + text := "Gast Anne-Sophie Müller-Lüdenscheidt" + det := &stubNERDetector{entities: []NEREntity{ent(text, "Müller-Lüdenscheidt", "LASTNAME", 0.9)}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, ExtendToPreviousWord: []string{"LASTNAME"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Gast [REDACTED:ner:LASTNAME]")) + }) + + It("never swallows a protected term in front of the hit", func() { + text := "Seeblick Waldmeister" + // tags what it actually sees: after shielding, the protected term is a placeholder + det := &lexiconDetector{words: map[string]NEREntity{"Waldmeister": {Group: "LASTNAME", Score: 0.9}}} + res, err := RedactNER(ctx, text, []NERConfig{{Detector: det, DefaultAction: ActionMask, + ExtendToPreviousWord: []string{"LASTNAME"}, ProtectedTerms: []string{"Seeblick"}}}) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Seeblick [REDACTED:ner:LASTNAME]")) + }) +}) diff --git a/core/services/routing/pii/ner.go b/core/services/routing/pii/ner.go index 65e4427d4..50b3b0647 100644 --- a/core/services/routing/pii/ner.go +++ b/core/services/routing/pii/ner.go @@ -79,6 +79,16 @@ type NERConfig struct { // the code alone leaves the town in the clear. Empty disables it. ExtendToNextWord []string + // ExtendToPreviousWord is the mirror of ExtendToNextWord: accepted + // detections of these groups also cover the word directly before them on + // the same line, but only when that word consists of letters (and the + // in-word joiners '-' and the apostrophe) - never digits, so amounts, + // dates and codes in front of a hit stay intact. It exists for a name + // model that tags the surname with high confidence and scores the first + // name in front of it below the threshold ("Gast Greta Waldmeister" + // -> "Gast Greta [REDACTED]"). Protected terms are never swallowed. + ExtendToPreviousWord []string + // ProtectedTerms are values that must reach the model unchanged even // though the detector would tag them, e.g. known business names that a // name model reads as surnames ("Hotel Seeblick" -> LASTNAME). Each @@ -138,6 +148,12 @@ func (c NERConfig) extendsToNextWord(group string) bool { return slices.Contains(c.ExtendToNextWord, group) } +// extendsToPreviousWord reports whether accepted hits of group are stretched +// over the preceding word (see NERConfig.ExtendToPreviousWord). +func (c NERConfig) extendsToPreviousWord(group string) bool { + return slices.Contains(c.ExtendToPreviousWord, group) +} + // patternID returns the synthetic pattern ID that audit rows and masks carry // for this detector's hits, e.g. "ner:EMAIL" or "pattern:ANTHROPIC_KEY". The // source prefix keeps neural and deterministic detections distinguishable in diff --git a/core/services/routing/pii/protected.go b/core/services/routing/pii/protected.go index 1eeceb8e5..1d94204d3 100644 --- a/core/services/routing/pii/protected.go +++ b/core/services/routing/pii/protected.go @@ -181,6 +181,18 @@ func keepExtension(end, extended int, regions []shield) int { return extended } +// keepExtensionBefore is keepExtension for a hit stretched backwards to +// extended (< start): the extension is dropped when a protected region lies +// in the added range, so a protected term in front of a hit is never masked. +func keepExtensionBefore(start, extended int, regions []shield) int { + for _, r := range regions { + if r.origEnd > extended && r.origEnd <= start { + return start + } + } + return extended +} + type protectedFileEntry struct { modTime time.Time size int64 diff --git a/core/services/routing/pii/redactor.go b/core/services/routing/pii/redactor.go index 2c30a913d..35df31624 100644 --- a/core/services/routing/pii/redactor.go +++ b/core/services/routing/pii/redactor.go @@ -210,6 +210,9 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, if cfg.extendsToNextWord(e.Group) && i == len(pieces)-1 { end = keepExtension(end, nextWordEnd(text, end), shields) } + if cfg.extendsToPreviousWord(e.Group) && i == 0 { + start = keepExtensionBefore(start, prevWordStart(text, start), shields) + } xlog.Debug("pii/ner: detection accepted", "group", e.Group, "score", e.Score, "action", action, "start", start, "end", end, "detected_end", pc[1], "text", e.Text) @@ -257,6 +260,47 @@ func nextWordEnd(text string, end int) int { return i } +// prevWordStart returns the start offset of the word that precedes +// text[start:]: horizontal whitespace before start is skipped (a line break +// ends the search), then letters, combining marks and the in-word joiners '-' +// and the apostrophe are consumed backwards. The word must consist of letters: +// when it touches a digit (an amount, a date, a code) or no word precedes, +// start is returned unchanged, so a hit is never stretched over a number, +// bare whitespace or onto the previous line. +func prevWordStart(text string, start int) int { + i := start + for i > 0 { + r, size := utf8.DecodeLastRuneInString(text[:i]) + if r == '\n' || r == '\r' || !unicode.IsSpace(r) { + break + } + i -= size + } + wordEnd := i + letters := 0 + for i > 0 { + r, size := utf8.DecodeLastRuneInString(text[:i]) + if unicode.IsLetter(r) { + letters++ + } else if !unicode.IsMark(r) && r != '-' && r != '\'' { + break + } + i -= size + } + if i > 0 { + if r, _ := utf8.DecodeLastRuneInString(text[:i]); unicode.IsDigit(r) { + return start // the word is glued to a number: leave it alone + } + } + for i < wordEnd && (text[i] == '-' || text[i] == '\'') { + i++ + } + if i == wordEnd || letters == 0 { + return start + } + return i +} + // mergeAndEmit handles the overlap-merge + masked-output step. Sorts by // start (stable on equal starts by descending action strength), drops // overlapping hits in favour of the stronger action, and walks the text diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index fa0918de8..0f48101f7 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -1111,6 +1111,8 @@ PII redaction is NER-based and runs on the **request** (input) side. It has two EMAIL: mask extend_to_next_word: # optional: also mask the word right after these - ZIPCODE # groups on the same line (e.g. the town after a postal code) + extend_to_previous_word: # optional: also mask the letters-only word right before these + - LASTNAME # groups on the same line (e.g. the first name before a surname) protected_terms: # optional: values that must pass unchanged even if tagged - Hotel Seeblick # (plus protected_terms_files: one term per line, in the models path) ``` diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index d36644baa..7411d16b7 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -121,6 +121,20 @@ A word is a run of letters, digits, combining marks, `-`, `'` and inner dots line break, and it does nothing when no word follows. Only one word is added, so a multi-word town ("Frankfurt am Main") keeps its tail. +`extend_to_previous_word` is the mirror for the word *before* a detection. A +name model often tags the surname with high confidence and scores the first +name in front of it below `min_score` (`Gast Greta [REDACTED:ner:LASTNAME]`): + +```yaml +pii_detection: + extend_to_previous_word: + - LASTNAME # "Gast Greta Waldmeister" -> "Gast [REDACTED:ner:LASTNAME]" +``` + +Here the preceding word must consist of letters (plus `-` and `'`): a word +with digits - an amount, a date, a room number - is never taken, nor a +protected term. The search skips spaces and tabs but never a line break. + #### Protected terms The opposite failure is a value the model tags although it must reach the