fix(pii): keep hyphen and apostrophe words whole around protected terms

A protected term that is one half of "Anna-Lena" or "O'Brien" is not a
whole word: shielding it left the term in the clear next to a masked
rest. Hyphen and apostrophe joiners between letters now count as
in-word, as they do for the word extensions.

When a detection spanned the placeholder and the name after it, the
space beside the protected term stayed inside the mask. Trim whitespace
from the surviving pieces.

Assisted-by: Claude Code:claude-sonnet-5-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/services/routing/pii/protected.go

@@ -137,6 +137,38 @@ func isWordRune(r rune) bool {
 	return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r)
 }
 
+// isJoiner reports the characters that join two parts of one word (the
+// hyphen in "Anna-Lena", the apostrophe in "O'Brien"), consistent with the
+// word extensions in redactor.go.
+func isJoiner(r rune) bool { return r == '-' || r == '\'' }
+
+// touchesWordBefore reports whether the text ends in a word character, or
+// in a joiner that is itself preceded by one: a protected term that is the
+// tail of "Anna-Lena" is not a whole word.
+func touchesWordBefore(text string) bool {
+	r, size := utf8.DecodeLastRuneInString(text)
+	if isWordRune(r) {
+		return true
+	}
+	if !isJoiner(r) {
+		return false
+	}
+	r, _ = utf8.DecodeLastRuneInString(text[:len(text)-size])
+	return isWordRune(r)
+}
+
+// touchesWordAfter is the mirror of touchesWordBefore.
+func touchesWordAfter(text string) bool {
+	r, size := utf8.DecodeRuneInString(text)
+	if isWordRune(r) {
+		return true
+	}
+	if !isJoiner(r) {
+		return false
+	}
+	r, _ = utf8.DecodeRuneInString(text[size:])
+	return isWordRune(r)
+}
 
 // shieldProtected replaces every whole-word occurrence of a protected term
 // with protectedPlaceholder and returns the text the detector should scan
@@ -151,15 +183,11 @@ func shieldProtected(text string, re *regexp.Regexp) (string, []shield) {
 	cursor := 0
 	for _, m := range re.FindAllStringIndex(text, -1) {
 		s, e := m[0], m[1]
-		if s > 0 {
-			if r, _ := utf8.DecodeLastRuneInString(text[:s]); isWordRune(r) {
-				continue
-			}
+		if s > 0 && touchesWordBefore(text[:s]) {
+			continue
 		}
-		if e < len(text) {
-			if r, _ := utf8.DecodeRuneInString(text[e:]); isWordRune(r) {
-				continue
-			}
+		if e < len(text) && touchesWordAfter(text[e:]) {
+			continue
 		}
 		b.WriteString(text[cursor:s])
 		scanStart := b.Len()

core/services/routing/pii/protected_test.go

@@ -125,6 +125,38 @@ var _ = Describe("ProtectedTerms", func() {
 		Expect(texts[1][res[1].Spans[1].Start:res[1].Spans[1].End]).To(Equal("Kranzberger"))
 	})
 
+	It("does not shield a term that is part of a hyphenated or apostrophe word", func() {
+		text := "Anna-Lena Meier, O'Brien, Anna Meier"
+		det := &lexiconDetector{words: map[string]NEREntity{
+			"Anna-Lena": {Group: "FIRSTNAME", Score: 0.9},
+			"Brien":     {Group: "LASTNAME", Score: 0.9},
+			"Meier":     {Group: "LASTNAME", Score: 0.9},
+		}}
+		res, err := RedactNER(ctx, text, cfg(det, "Anna", "Brien"))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("[REDACTED:ner:FIRSTNAME] [REDACTED:ner:LASTNAME], " +
+			"O'[REDACTED:ner:LASTNAME], Anna [REDACTED:ner:LASTNAME]"))
+		Expect(det.seen[0]).To(ContainSubstring("Anna-Lena"))
+		Expect(det.seen[0]).To(ContainSubstring(protectedPlaceholder+" Meier"), "a free-standing term is still shielded")
+	})
+
+	It("still shields a term next to a hyphen that joins no word", func() {
+		scan, regions := shieldProtected("Hotel Seeblick - Gast", protectedMatcher([]string{"Hotel Seeblick"}))
+		Expect(scan).To(Equal(protectedPlaceholder + " - Gast"))
+		Expect(regions).To(HaveLen(1))
+	})
+
+	It("does not let a mask swallow the space next to a protected term", func() {
+		text := "Hotel Seeblick Meier"
+		det := &lexiconDetector{words: map[string]NEREntity{
+			protectedPlaceholder + " Meier": {Group: "LASTNAME", Score: 0.9},
+		}}
+		res, err := RedactNER(ctx, text, cfg(det, "Hotel Seeblick"))
+		Expect(err).NotTo(HaveOccurred())
+		Expect(res.Redacted).To(Equal("Hotel Seeblick [REDACTED:ner:LASTNAME]"))
+		Expect(text[res.Spans[0].Start:res.Spans[0].End]).To(Equal("Meier"))
+	})
+
 	It("matches a term across no-break and other Unicode spaces", func() {
 		text := "Hotel\u00a0Seeblick, Hotel\u202fSeeblick, Hotel\u2003Seeblick, Seeblick"
 		det := &lexiconDetector{words: map[string]NEREntity{"Seeblick": {Group: "LASTNAME", Score: 0.9}}}

core/services/routing/pii/redactor.go

@@ -202,7 +202,7 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit,
 		}
 		pieces := [][2]int{{e.Start, e.End}}
 		if shields != nil {
-			pieces = unshieldSpan(e.Start, e.End, shields)
+			pieces = trimSpans(text, unshieldSpan(e.Start, e.End, shields))
 			if len(pieces) == 0 {
 				xlog.Debug("pii/ner: detection dropped (protected term)",
 					"group", e.Group, "score", e.Score, "start", e.Start, "end", e.End)
@@ -229,6 +229,35 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit,
 	return hits, nil
 }
 
+// trimSpans cuts leading and trailing whitespace off each span and drops
+// the ones that are empty afterwards. Cutting a protected term out of a
+// detection leaves the space next to it on the surviving piece, and a
+// mask must not swallow it ("Hotel Seeblick[REDACTED]").
+func trimSpans(text string, spans [][2]int) [][2]int {
+	out := spans[:0]
+	for _, sp := range spans {
+		s, e := sp[0], sp[1]
+		for s < e {
+			r, size := utf8.DecodeRuneInString(text[s:e])
+			if !unicode.IsSpace(r) {
+				break
+			}
+			s += size
+		}
+		for e > s {
+			r, size := utf8.DecodeLastRuneInString(text[s:e])
+			if !unicode.IsSpace(r) {
+				break
+			}
+			e -= size
+		}
+		if s < e {
+			out = append(out, [2]int{s, e})
+		}
+	}
+	return out
+}
+
 // nextWordEnd returns the end offset of the word that follows text[:end]:
 // horizontal whitespace is skipped (a line break ends the search), then
 // letters, digits, combining marks and the in-word joiners '-', '.' and