From a0e844d6112d1cfef6dc3c43ae439cc2376a7cca Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 13 Oct 2026 08:09:56 +0000 Subject: [PATCH] fix(pii): keep hyphen and apostrophe words whole around protected terms A protected term that is one half of "Anna-Lena" or "O'Brien" is not a whole word: shielding it left the term in the clear next to a masked rest. Hyphen and apostrophe joiners between letters now count as in-word, as they do for the word extensions. When a detection spanned the placeholder and the name after it, the space beside the protected term stayed inside the mask. Trim whitespace from the surviving pieces. Assisted-by: Claude Code:claude-sonnet-5-5 Signed-off-by: Ettore Di Giacinto --- core/services/routing/pii/protected.go | 44 +++++++++++++++++---- core/services/routing/pii/protected_test.go | 32 +++++++++++++++ core/services/routing/pii/redactor.go | 31 ++++++++++++++- 3 files changed, 98 insertions(+), 9 deletions(-) diff --git a/core/services/routing/pii/protected.go b/core/services/routing/pii/protected.go index af15fc276..2e0111f3f 100644 --- a/core/services/routing/pii/protected.go +++ b/core/services/routing/pii/protected.go @@ -137,6 +137,38 @@ func isWordRune(r rune) bool { return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) } +// isJoiner reports the characters that join two parts of one word (the +// hyphen in "Anna-Lena", the apostrophe in "O'Brien"), consistent with the +// word extensions in redactor.go. +func isJoiner(r rune) bool { return r == '-' || r == '\'' } + +// touchesWordBefore reports whether the text ends in a word character, or +// in a joiner that is itself preceded by one: a protected term that is the +// tail of "Anna-Lena" is not a whole word. +func touchesWordBefore(text string) bool { + r, size := utf8.DecodeLastRuneInString(text) + if isWordRune(r) { + return true + } + if !isJoiner(r) { + return false + } + r, _ = utf8.DecodeLastRuneInString(text[:len(text)-size]) + return isWordRune(r) +} + +// touchesWordAfter is the mirror of touchesWordBefore. +func touchesWordAfter(text string) bool { + r, size := utf8.DecodeRuneInString(text) + if isWordRune(r) { + return true + } + if !isJoiner(r) { + return false + } + r, _ = utf8.DecodeRuneInString(text[size:]) + return isWordRune(r) +} // shieldProtected replaces every whole-word occurrence of a protected term // with protectedPlaceholder and returns the text the detector should scan @@ -151,15 +183,11 @@ func shieldProtected(text string, re *regexp.Regexp) (string, []shield) { cursor := 0 for _, m := range re.FindAllStringIndex(text, -1) { s, e := m[0], m[1] - if s > 0 { - if r, _ := utf8.DecodeLastRuneInString(text[:s]); isWordRune(r) { - continue - } + if s > 0 && touchesWordBefore(text[:s]) { + continue } - if e < len(text) { - if r, _ := utf8.DecodeRuneInString(text[e:]); isWordRune(r) { - continue - } + if e < len(text) && touchesWordAfter(text[e:]) { + continue } b.WriteString(text[cursor:s]) scanStart := b.Len() diff --git a/core/services/routing/pii/protected_test.go b/core/services/routing/pii/protected_test.go index 68f6dbf71..fb01fffbc 100644 --- a/core/services/routing/pii/protected_test.go +++ b/core/services/routing/pii/protected_test.go @@ -125,6 +125,38 @@ var _ = Describe("ProtectedTerms", func() { Expect(texts[1][res[1].Spans[1].Start:res[1].Spans[1].End]).To(Equal("Kranzberger")) }) + It("does not shield a term that is part of a hyphenated or apostrophe word", func() { + text := "Anna-Lena Meier, O'Brien, Anna Meier" + det := &lexiconDetector{words: map[string]NEREntity{ + "Anna-Lena": {Group: "FIRSTNAME", Score: 0.9}, + "Brien": {Group: "LASTNAME", Score: 0.9}, + "Meier": {Group: "LASTNAME", Score: 0.9}, + }} + res, err := RedactNER(ctx, text, cfg(det, "Anna", "Brien")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("[REDACTED:ner:FIRSTNAME] [REDACTED:ner:LASTNAME], " + + "O'[REDACTED:ner:LASTNAME], Anna [REDACTED:ner:LASTNAME]")) + Expect(det.seen[0]).To(ContainSubstring("Anna-Lena")) + Expect(det.seen[0]).To(ContainSubstring(protectedPlaceholder+" Meier"), "a free-standing term is still shielded") + }) + + It("still shields a term next to a hyphen that joins no word", func() { + scan, regions := shieldProtected("Hotel Seeblick - Gast", protectedMatcher([]string{"Hotel Seeblick"})) + Expect(scan).To(Equal(protectedPlaceholder + " - Gast")) + Expect(regions).To(HaveLen(1)) + }) + + It("does not let a mask swallow the space next to a protected term", func() { + text := "Hotel Seeblick Meier" + det := &lexiconDetector{words: map[string]NEREntity{ + protectedPlaceholder + " Meier": {Group: "LASTNAME", Score: 0.9}, + }} + res, err := RedactNER(ctx, text, cfg(det, "Hotel Seeblick")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Hotel Seeblick [REDACTED:ner:LASTNAME]")) + Expect(text[res.Spans[0].Start:res.Spans[0].End]).To(Equal("Meier")) + }) + It("matches a term across no-break and other Unicode spaces", func() { text := "Hotel\u00a0Seeblick, Hotel\u202fSeeblick, Hotel\u2003Seeblick, Seeblick" det := &lexiconDetector{words: map[string]NEREntity{"Seeblick": {Group: "LASTNAME", Score: 0.9}}} diff --git a/core/services/routing/pii/redactor.go b/core/services/routing/pii/redactor.go index 7097f186b..ee08a496c 100644 --- a/core/services/routing/pii/redactor.go +++ b/core/services/routing/pii/redactor.go @@ -202,7 +202,7 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, } pieces := [][2]int{{e.Start, e.End}} if shields != nil { - pieces = unshieldSpan(e.Start, e.End, shields) + pieces = trimSpans(text, unshieldSpan(e.Start, e.End, shields)) if len(pieces) == 0 { xlog.Debug("pii/ner: detection dropped (protected term)", "group", e.Group, "score", e.Score, "start", e.Start, "end", e.End) @@ -229,6 +229,35 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, return hits, nil } +// trimSpans cuts leading and trailing whitespace off each span and drops +// the ones that are empty afterwards. Cutting a protected term out of a +// detection leaves the space next to it on the surviving piece, and a +// mask must not swallow it ("Hotel Seeblick[REDACTED]"). +func trimSpans(text string, spans [][2]int) [][2]int { + out := spans[:0] + for _, sp := range spans { + s, e := sp[0], sp[1] + for s < e { + r, size := utf8.DecodeRuneInString(text[s:e]) + if !unicode.IsSpace(r) { + break + } + s += size + } + for e > s { + r, size := utf8.DecodeLastRuneInString(text[s:e]) + if !unicode.IsSpace(r) { + break + } + e -= size + } + if s < e { + out = append(out, [2]int{s, e}) + } + } + return out +} + // nextWordEnd returns the end offset of the word that follows text[:end]: // horizontal whitespace is skipped (a line break ends the search), then // letters, digits, combining marks and the in-word joiners '-', '.' and