From 01f71772a564005ee1139d0bec013ff408fd7739 Mon Sep 17 00:00:00 2001 From: Stefan Walcz Date: Sat, 3 Oct 2026 12:19:49 +0200 Subject: [PATCH] feat(pii): protected terms that pass the NER filter unchanged MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A name model reads many business names as surnames. Measured with privacy-filter-multilingual on 24 synthetic German receipts and invoices: the supplier name ("Hotel Seeblick", "Café Lindenhof", ...) was masked as LASTNAME (0.78-0.98) or FIRSTNAME in 9 of 12 receipts that carried one. The supplier is exactly what the upstream model needs to categorise a receipt, and no score floor helps at those confidences. pii_detection.protected_terms (inline) and pii_detection.protected_terms_files (files inside the models path, one term per line, '#' comments, re-read when size or mtime change) list values that must pass unchanged. Each whole-word, case-insensitive occurrence is replaced by a neutral placeholder before the detector runs; every detection is mapped back onto the original text with the placeholder parts cut out, so a protected term is never masked or blocked, the detector never sees it as context, and event offsets keep addressing the original text. A word extension (extend_to_next_word) is dropped rather than run into a protected term. Terms shorter than three characters are ignored. A missing or unreadable file is logged and skipped: fewer protected terms means more masking, never less. Paths are verified to stay inside the models path. Opt-in, empty by default. Covered by Ginkgo specs in core/services/routing/pii/protected_test.go (detector-visible text, exact restoration, straddling detections, whole-word/case/line-wrap matching, the segment path, file loading and reload), a YAML round-trip spec, registry entries and the middleware/model-configuration docs. Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Stefan Walcz --- core/application/application.go | 2 + core/config/meta/registry.go | 14 ++ core/config/model_config.go | 16 ++ core/config/model_config_test.go | 11 + core/services/routing/pii/ner.go | 9 + core/services/routing/pii/protected.go | 236 +++++++++++++++++++ core/services/routing/pii/protected_test.go | 157 ++++++++++++ core/services/routing/pii/redactor.go | 52 ++-- docs/content/advanced/model-configuration.md | 2 + docs/content/operations/middleware.md | 31 +++ 10 files changed, 513 insertions(+), 17 deletions(-) create mode 100644 core/services/routing/pii/protected.go create mode 100644 core/services/routing/pii/protected_test.go diff --git a/core/application/application.go b/core/application/application.go index c0546fb84..f519f5ef1 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -372,6 +372,8 @@ func (a *Application) PIINERResolver() pii.NERDetectorResolver { pii.SourceNER, ) nc.ExtendToNextWord = cfg.PIIDetectionExtendToNextWord() + terms, files := cfg.PIIDetectionProtectedTerms() + nc.ProtectedTerms = pii.LoadProtectedTerms(terms, files, a.ApplicationConfig().SystemState.Model.ModelsPath) return nc, true } } diff --git a/core/config/meta/registry.go b/core/config/meta/registry.go index 5fe4b1f5b..91ce5e30c 100644 --- a/core/config/meta/registry.go +++ b/core/config/meta/registry.go @@ -1062,6 +1062,20 @@ func DefaultRegistry() map[string]FieldMetaOverride { Component: "string-list", Order: 215, }, + "pii_detection.protected_terms": { + Section: "pii", + Label: "Protected Terms", + Description: "Values that must pass the filter unchanged even when the model tags them, e.g. known business names a name model reads as surnames. Hidden from the detector behind a placeholder and restored afterwards; matched whole-word and case-insensitively, minimum three characters.", + Component: "string-list", + Order: 216, + }, + "pii_detection.protected_terms_files": { + Section: "pii", + Label: "Protected Terms Files", + Description: "Files inside the models path with one protected term per line ('#' starts a comment). Re-read when they change, so an external process can keep the list current without a restart. A missing file is skipped with a warning.", + Component: "string-list", + Order: 217, + }, // --- Cloud passthrough proxy --- // These only have an effect when Backend is set to diff --git a/core/config/model_config.go b/core/config/model_config.go index b09f0bb30..2fd9345d4 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -656,6 +656,15 @@ type PIIDetectionConfig struct { // with the same action - for a neighbour the model scores unreliably, // such as the town after a postal code. ExtendToNextWord []string `yaml:"extend_to_next_word,omitempty" json:"extend_to_next_word,omitempty"` + // ProtectedTerms are values that must pass the filter unchanged even + // when the model tags them (e.g. known business names read as + // surnames). They are hidden from the detector behind a placeholder and + // restored afterwards. Matched whole-word and case-insensitively. + ProtectedTerms []string `yaml:"protected_terms,omitempty" json:"protected_terms,omitempty"` + // ProtectedTermsFiles adds terms from files (one per line, '#' comments), + // resolved inside the models path and re-read when they change, so an + // external process can keep the list current without a restart. + ProtectedTermsFiles []string `yaml:"protected_terms_files,omitempty" json:"protected_terms_files,omitempty"` } // PIIPattern is one operator-defined pattern on a pattern detector model. Name @@ -744,6 +753,13 @@ func (c *ModelConfig) PIIDetectionExtendToNextWord() []string { return append([]string(nil), c.PIIDetection.ExtendToNextWord...) } +// PIIDetectionProtectedTerms returns the inline protected terms and the +// configured term files (fresh copies). The files are read by the pii +// package (pii.LoadProtectedTerms), relative to the models path. +func (c *ModelConfig) PIIDetectionProtectedTerms() (terms, files []string) { + return slices.Clone(c.PIIDetection.ProtectedTerms), slices.Clone(c.PIIDetection.ProtectedTermsFiles) +} + // IsPatternDetector reports whether this detector model matches secrets with // regex patterns (built-in and/or operator-defined) rather than a neural NER // model. Such a model runs entirely in-process (no backend / GGUF / VRAM); the diff --git a/core/config/model_config_test.go b/core/config/model_config_test.go index 13fd32d98..b68023a97 100644 --- a/core/config/model_config_test.go +++ b/core/config/model_config_test.go @@ -800,6 +800,17 @@ var _ = Describe("PII config accessors", func() { got[0] = "CITY" Expect(cfg.PIIDetection.ExtendToNextWord[0]).To(Equal("ZIPCODE"), "accessor must return a fresh slice") }) + + It("unmarshals pii_detection.protected_terms and protected_terms_files from YAML", func() { + var cfg ModelConfig + raw := []byte("name: pf\npii_detection:\n protected_terms: [Hotel Seeblick]\n protected_terms_files: [pf/terms.txt]\n") + Expect(yaml.Unmarshal(raw, &cfg)).To(Succeed()) + terms, files := cfg.PIIDetectionProtectedTerms() + Expect(terms).To(Equal([]string{"Hotel Seeblick"})) + Expect(files).To(Equal([]string{"pf/terms.txt"})) + terms[0] = "changed" + Expect(cfg.PIIDetection.ProtectedTerms[0]).To(Equal("Hotel Seeblick"), "accessor must return a fresh slice") + }) }) var _ = Describe("GGUF importer chat-default guard (reservedNonChatModel)", func() { diff --git a/core/services/routing/pii/ner.go b/core/services/routing/pii/ner.go index ed5ca69e5..65e4427d4 100644 --- a/core/services/routing/pii/ner.go +++ b/core/services/routing/pii/ner.go @@ -78,6 +78,15 @@ type NERConfig struct { // under-scores the town after it ("67059 Ludwigshafen"), and masking // the code alone leaves the town in the clear. Empty disables it. ExtendToNextWord []string + + // ProtectedTerms are values that must reach the model unchanged even + // though the detector would tag them, e.g. known business names that a + // name model reads as surnames ("Hotel Seeblick" -> LASTNAME). Each + // whole-word, case-insensitive occurrence is replaced by a neutral + // placeholder before detection and restored afterwards; no detection + // can mask or block it. Terms shorter than three characters are + // ignored. Empty disables it. + ProtectedTerms []string } // Detector source labels (the PatternID prefix). Kept short and stable — diff --git a/core/services/routing/pii/protected.go b/core/services/routing/pii/protected.go new file mode 100644 index 000000000..fefa0e981 --- /dev/null +++ b/core/services/routing/pii/protected.go @@ -0,0 +1,236 @@ +package pii + +import ( + "bufio" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "sync" + "time" + "unicode" + "unicode/utf8" + + "github.com/mudler/xlog" + + "github.com/mudler/LocalAI/pkg/utils" +) + +// protectedPlaceholder stands in for a protected term while the detector +// runs. It carries no letters a name/address model could read as PII and +// is restored verbatim afterwards, so its exact shape only matters as +// neutral context for the encoder. +const protectedPlaceholder = "[PROTECTED]" + +// minProtectedTermRunes is the shortest term that is honoured. Shorter +// terms ("AG", "DB") would shield every matching token in a document, +// including ones that are part of real PII. +const minProtectedTermRunes = 3 + +// shield maps one protected occurrence between the original text +// [origStart, origEnd) and the scanned text [scanStart, scanEnd). +type shield struct { + origStart, origEnd int + scanStart, scanEnd int +} + +var protectedMatchers sync.Map // key: "\x00"-joined normalised terms -> *regexp.Regexp + +// protectedMatcher compiles (once per distinct term list) a +// case-insensitive matcher for the terms, longest first so that "Hotel +// Seeblick Spa" wins over "Hotel Seeblick". Whitespace inside a term +// matches any whitespace run, so a term still matches across a line wrap. +// Returns nil when no usable term remains. +func protectedMatcher(terms []string) *regexp.Regexp { + var norm []string + for _, t := range terms { + t = strings.Join(strings.Fields(t), " ") + if utf8.RuneCountInString(t) < minProtectedTermRunes { + continue + } + norm = append(norm, t) + } + if len(norm) == 0 { + return nil + } + slices.SortFunc(norm, func(a, b string) int { + if d := len(b) - len(a); d != 0 { + return d + } + return strings.Compare(a, b) + }) + norm = slices.Compact(norm) + key := strings.Join(norm, "\x00") + if re, ok := protectedMatchers.Load(key); ok { + return re.(*regexp.Regexp) + } + alts := make([]string, len(norm)) + for i, t := range norm { + words := strings.Fields(t) + for j, w := range words { + words[j] = regexp.QuoteMeta(w) + } + alts[i] = strings.Join(words, `\s+`) + } + re := regexp.MustCompile(`(?i)(?:` + strings.Join(alts, "|") + `)`) + protectedMatchers.Store(key, re) + return re +} + +func isWordRune(r rune) bool { + return unicode.IsLetter(r) || unicode.IsDigit(r) || unicode.IsMark(r) +} + +// shieldProtected replaces every whole-word occurrence of a protected term +// with protectedPlaceholder and returns the text the detector should scan +// plus the mapping back to the original. With no matcher or no match the +// text is returned unchanged and regions is nil. +func shieldProtected(text string, re *regexp.Regexp) (string, []shield) { + if re == nil || text == "" { + return text, nil + } + var regions []shield + var b strings.Builder + cursor := 0 + for _, m := range re.FindAllStringIndex(text, -1) { + s, e := m[0], m[1] + if s > 0 { + if r, _ := utf8.DecodeLastRuneInString(text[:s]); isWordRune(r) { + continue + } + } + if e < len(text) { + if r, _ := utf8.DecodeRuneInString(text[e:]); isWordRune(r) { + continue + } + } + b.WriteString(text[cursor:s]) + scanStart := b.Len() + b.WriteString(protectedPlaceholder) + regions = append(regions, shield{origStart: s, origEnd: e, scanStart: scanStart, scanEnd: b.Len()}) + cursor = e + } + if regions == nil { + return text, nil + } + b.WriteString(text[cursor:]) + return b.String(), regions +} + +// unshieldSpan maps a detection [start, end) in the scanned text back to +// the original text. The parts that fall on a placeholder are cut out — +// a protected term is never masked or blocked — so one detection can come +// back as zero, one or two pieces. +func unshieldSpan(start, end int, regions []shield) [][2]int { + toOrig := func(p int) int { + delta := 0 + for _, r := range regions { + if r.scanEnd > p { + break + } + delta += (r.origEnd - r.origStart) - (r.scanEnd - r.scanStart) + } + return p + delta + } + var out [][2]int + cur := start + for _, r := range regions { + if r.scanEnd <= cur { + continue + } + if r.scanStart >= end { + break + } + if r.scanStart > cur { + out = append(out, [2]int{toOrig(cur), toOrig(r.scanStart)}) + } + cur = r.scanEnd + } + if cur < end { + out = append(out, [2]int{toOrig(cur), toOrig(end)}) + } + return out +} + +// keepExtension returns extended when the word extension [end, extended) +// does not run into a protected occurrence, and end otherwise: a protected +// term is never masked, and the extension is dropped rather than cut so +// no dangling whitespace is swallowed. +func keepExtension(end, extended int, regions []shield) int { + for _, r := range regions { + if r.origStart >= end && r.origStart < extended { + return end + } + } + return extended +} + +type protectedFileEntry struct { + modTime time.Time + size int64 + terms []string +} + +var ( + protectedFilesMu sync.Mutex + protectedFiles = map[string]protectedFileEntry{} +) + +// LoadProtectedTerms returns the inline terms plus the terms read from +// each file (one term per line; blank lines and lines starting with '#' +// are skipped). File paths resolve against baseDir (the models path) and +// must stay inside it. A file is re-read only when its size or modification time +// changes, so an external process can keep it current without a restart. +// +// A missing or unreadable file is logged and skipped. That is the safe +// direction for a PII filter: fewer protected terms means more masking, +// never less. +func LoadProtectedTerms(inline, files []string, baseDir string) []string { + out := append([]string(nil), inline...) + for _, f := range files { + if err := utils.VerifyPath(f, baseDir); err != nil { + xlog.Warn("pii: protected terms file outside the models path; skipping", "file", f, "error", err) + continue + } + path := filepath.Join(baseDir, f) + terms, err := readProtectedFile(path) + if err != nil { + xlog.Warn("pii: protected terms file not readable; skipping", "file", path, "error", err) + continue + } + out = append(out, terms...) + } + return out +} + +func readProtectedFile(path string) ([]string, error) { + st, err := os.Stat(path) + if err != nil { + return nil, err + } + protectedFilesMu.Lock() + defer protectedFilesMu.Unlock() + if c, ok := protectedFiles[path]; ok && c.modTime.Equal(st.ModTime()) && c.size == st.Size() { + return c.terms, nil + } + fh, err := os.Open(path) + if err != nil { + return nil, err + } + defer fh.Close() + var terms []string + sc := bufio.NewScanner(fh) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + terms = append(terms, line) + } + if err := sc.Err(); err != nil { + return nil, err + } + protectedFiles[path] = protectedFileEntry{modTime: st.ModTime(), size: st.Size(), terms: terms} + return terms, nil +} diff --git a/core/services/routing/pii/protected_test.go b/core/services/routing/pii/protected_test.go new file mode 100644 index 000000000..d3d77c54c --- /dev/null +++ b/core/services/routing/pii/protected_test.go @@ -0,0 +1,157 @@ +package pii + +import ( + "context" + "os" + "path/filepath" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// lexiconDetector tags every occurrence of its words in the text it is +// given, the way a NER model reacts to what it actually sees. It records +// the scanned texts so specs can assert what reached the detector. +type lexiconDetector struct { + words map[string]NEREntity // value: Group + Score; offsets are filled in + seen []string +} + +func (d *lexiconDetector) Detect(_ context.Context, text string) ([]NEREntity, error) { + d.seen = append(d.seen, text) + var out []NEREntity + for w, tmpl := range d.words { + for off := 0; ; { + i := strings.Index(text[off:], w) + if i < 0 { + break + } + s := off + i + out = append(out, NEREntity{Group: tmpl.Group, Score: tmpl.Score, Start: s, End: s + len(w), Text: w}) + off = s + len(w) + } + } + return out, nil +} + +var _ = Describe("ProtectedTerms", func() { + ctx := context.Background() + + // A hotel invoice as a name/address model sees it: the business name is + // read as a surname (measured: LASTNAME 0.78-0.98), the guest's name and + // the postal code are tagged, the town is not. + const hotel = "Hotel Seeblick, Gast Lena Kranzberger, Am Mühlbach 12, 67059 Ludwigshafen\nÜbernachtung netto 84,00 EUR" + newDet := func() *lexiconDetector { + return &lexiconDetector{words: map[string]NEREntity{ + "Seeblick": {Group: "LASTNAME", Score: 0.81}, + "Lena": {Group: "FIRSTNAME", Score: 0.95}, + "Kranzberger": {Group: "LASTNAME", Score: 0.97}, + "Mühlbach": {Group: "STREET", Score: 0.48}, + "67059": {Group: "ZIPCODE", Score: 0.96}, + }} + } + cfg := func(det NERDetector, terms ...string) []NERConfig { + return []NERConfig{{ + Detector: det, MinScore: 0.4, DefaultAction: ActionMask, + ExtendToNextWord: []string{"ZIPCODE"}, ProtectedTerms: terms, + }} + } + + It("masks the business name without protection (the gap it closes)", func() { + res, err := RedactNER(ctx, hotel, cfg(newDet())) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(HavePrefix("Hotel [REDACTED:ner:LASTNAME],")) + }) + + It("keeps a protected term verbatim, hides it from the detector, and still masks the identity", func() { + det := newDet() + res, err := RedactNER(ctx, hotel, cfg(det, "hotel seeblick")) // learned memories store lower-case keys + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Hotel Seeblick, Gast [REDACTED:ner:FIRSTNAME] [REDACTED:ner:LASTNAME], " + + "Am [REDACTED:ner:STREET] 12, [REDACTED:ner:ZIPCODE]\nÜbernachtung netto 84,00 EUR")) + Expect(det.seen).To(HaveLen(1)) + Expect(det.seen[0]).NotTo(ContainSubstring("Seeblick"), "the detector must not see the protected value") + Expect(det.seen[0]).To(HavePrefix(protectedPlaceholder + ", Gast Lena")) + // Spans address the ORIGINAL text, not the scanned one. + for _, sp := range res.Spans { + Expect(hotel[sp.Start:sp.End]).To(BeElementOf("Lena", "Kranzberger", "Mühlbach", "67059 Ludwigshafen")) + } + }) + + It("cuts the protected part out of a detection that straddles it", func() { + text := "Hotel Seeblick, Gast Lena" + det := &lexiconDetector{words: map[string]NEREntity{ + // what the detector sees after shielding: one span over placeholder + name + protectedPlaceholder + ", Gast Lena": {Group: "LASTNAME", Score: 0.9}, + }} + res, err := RedactNER(ctx, text, cfg(det, "Hotel Seeblick")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("Hotel Seeblick[REDACTED:ner:LASTNAME]")) + Expect(res.Spans).To(HaveLen(1)) + Expect(text[res.Spans[0].Start:res.Spans[0].End]).To(Equal(", Gast Lena")) + }) + + It("matches whole words only, case-insensitively, across a line wrap, and ignores short terms", func() { + text := "Café Lindenhof\nCAFÉ\nLINDENHOF Lindenhofer AG Lindenhof" + det := &lexiconDetector{words: map[string]NEREntity{ + "Lindenhof": {Group: "LASTNAME", Score: 0.9}, + "AG": {Group: "ORGANIZATION", Score: 0.9}, + }} + res, err := RedactNER(ctx, text, cfg(det, "Café Lindenhof", "AG")) + Expect(err).NotTo(HaveOccurred()) + // "Café Lindenhof" and "CAFÉ\nLINDENHOF" are protected; the bare + // "Lindenhof" at the end is not the term and is masked; "AG" is too + // short to protect; "Lindenhofer" is a different word. + Expect(res.Redacted).To(Equal("Café Lindenhof\nCAFÉ\nLINDENHOF [REDACTED:ner:LASTNAME]er " + + "[REDACTED:ner:ORGANIZATION] [REDACTED:ner:LASTNAME]")) + }) + + It("never stretches a word extension onto a protected term", func() { + text := "PLZ 67059 Hotel Seeblick" + det := &lexiconDetector{words: map[string]NEREntity{"67059": {Group: "ZIPCODE", Score: 0.9}}} + res, err := RedactNER(ctx, text, cfg(det, "Hotel Seeblick")) + Expect(err).NotTo(HaveOccurred()) + Expect(res.Redacted).To(Equal("PLZ [REDACTED:ner:ZIPCODE] Hotel Seeblick")) + }) + + It("maps offsets back per segment on the multi-message path", func() { + texts := []string{"Beleg von Hotel Seeblick", "Gast Lena Kranzberger"} + res, err := RedactNERSegments(ctx, texts, cfg(newDet(), "Hotel Seeblick")) + Expect(err).NotTo(HaveOccurred()) + Expect(res[0].Redacted).To(Equal("Beleg von Hotel Seeblick")) + Expect(res[1].Redacted).To(Equal("Gast [REDACTED:ner:FIRSTNAME] [REDACTED:ner:LASTNAME]")) + Expect(texts[1][res[1].Spans[1].Start:res[1].Spans[1].End]).To(Equal("Kranzberger")) + }) + + It("is a no-op without terms", func() { + scan, regions := shieldProtected(hotel, protectedMatcher(nil)) + Expect(scan).To(Equal(hotel)) + Expect(regions).To(BeNil()) + }) +}) + +var _ = Describe("LoadProtectedTerms", func() { + It("reads term files inside the base path, skips comments, and re-reads on change", func() { + dir := GinkgoT().TempDir() + f := filepath.Join(dir, "lieferanten.txt") + Expect(os.WriteFile(f, []byte("# gelernte Lieferanten\nhotel seeblick\n\n café lindenhof \n"), 0o644)).To(Succeed()) + Expect(LoadProtectedTerms([]string{"Tankstelle Rheinblick"}, []string{"lieferanten.txt"}, dir)). + To(Equal([]string{"Tankstelle Rheinblick", "hotel seeblick", "café lindenhof"})) + + Expect(os.WriteFile(f, []byte("druckerei morgenrot\n"), 0o644)).To(Succeed()) + later := time.Now().Add(2 * time.Second) + Expect(os.Chtimes(f, later, later)).To(Succeed()) + Expect(LoadProtectedTerms(nil, []string{"lieferanten.txt"}, dir)).To(Equal([]string{"druckerei morgenrot"})) + }) + + It("skips a missing file and a path outside the base path (fewer terms = more masking)", func() { + dir := GinkgoT().TempDir() + outside := filepath.Join(filepath.Dir(dir), "outside.txt") + Expect(os.WriteFile(outside, []byte("secret name\n"), 0o644)).To(Succeed()) + DeferCleanup(os.Remove, outside) + Expect(LoadProtectedTerms([]string{"x-inline"}, []string{"missing.txt", "../outside.txt"}, dir)). + To(Equal([]string{"x-inline"})) + }) +}) diff --git a/core/services/routing/pii/redactor.go b/core/services/routing/pii/redactor.go index c83af1d6a..2c30a913d 100644 --- a/core/services/routing/pii/redactor.go +++ b/core/services/routing/pii/redactor.go @@ -152,11 +152,18 @@ func RedactNERSegments(ctx context.Context, texts []string, cfgs []NERConfig) ([ // Entities below MinScore or with no resolved action are dropped — the // detector doesn't know which entity groups the admin cares about, so // the policy filters here. +// +// Protected terms (NERConfig.ProtectedTerms) are swapped for a neutral +// placeholder before the detector runs and every detection is mapped +// back onto the original text with the placeholder parts cut out, so a +// protected term is never masked or blocked and the detector never sees +// it as context. func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, error) { if cfg.Detector == nil || text == "" { return nil, nil } - entities, err := cfg.Detector.Detect(ctx, text) + scan, shields := shieldProtected(text, protectedMatcher(cfg.ProtectedTerms)) + entities, err := cfg.Detector.Detect(ctx, scan) if err != nil { return nil, err } @@ -181,28 +188,39 @@ func collectNERHits(ctx context.Context, text string, cfg NERConfig) ([]rawHit, "start", e.Start, "end", e.End, "text", e.Text) continue } - if e.Start < 0 || e.End <= e.Start || e.End > len(text) { + if e.Start < 0 || e.End <= e.Start || e.End > len(scan) { // Defensive: the backend should return byte offsets into the - // original text, but a misconfigured model could produce + // scanned text, but a misconfigured model could produce // garbage. Skip rather than panic on slice OOB. xlog.Warn("pii/ner: detection has out-of-range offsets; skipping", - "group", e.Group, "start", e.Start, "end", e.End, "text_len", len(text)) + "group", e.Group, "start", e.Start, "end", e.End, "text_len", len(scan)) continue } - end := e.End - if cfg.extendsToNextWord(e.Group) { - end = nextWordEnd(text, e.End) + pieces := [][2]int{{e.Start, e.End}} + if shields != nil { + pieces = unshieldSpan(e.Start, e.End, shields) + if len(pieces) == 0 { + xlog.Debug("pii/ner: detection dropped (protected term)", + "group", e.Group, "score", e.Score, "start", e.Start, "end", e.End) + continue + } + } + for i, pc := range pieces { + start, end := pc[0], pc[1] + if cfg.extendsToNextWord(e.Group) && i == len(pieces)-1 { + end = keepExtension(end, nextWordEnd(text, end), shields) + } + xlog.Debug("pii/ner: detection accepted", + "group", e.Group, "score", e.Score, "action", action, + "start", start, "end", end, "detected_end", pc[1], "text", e.Text) + hits = append(hits, rawHit{ + patternID: cfg.patternID(e.Group), + action: action, + start: start, + end: end, + score: e.Score, + }) } - xlog.Debug("pii/ner: detection accepted", - "group", e.Group, "score", e.Score, "action", action, - "start", e.Start, "end", e.End, "extended_end", end, "text", e.Text) - hits = append(hits, rawHit{ - patternID: cfg.patternID(e.Group), - action: action, - start: e.Start, - end: end, - score: e.Score, - }) } return hits, nil } diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index cdf5bde57..fa0918de8 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -1111,6 +1111,8 @@ PII redaction is NER-based and runs on the **request** (input) side. It has two EMAIL: mask extend_to_next_word: # optional: also mask the word right after these - ZIPCODE # groups on the same line (e.g. the town after a postal code) + protected_terms: # optional: values that must pass unchanged even if tagged + - Hotel Seeblick # (plus protected_terms_files: one term per line, in the models path) ``` - **Consuming models** opt in and reference one or more detectors by name - no per-consumer policy: diff --git a/docs/content/operations/middleware.md b/docs/content/operations/middleware.md index bc3acce73..d36644baa 100644 --- a/docs/content/operations/middleware.md +++ b/docs/content/operations/middleware.md @@ -121,6 +121,37 @@ A word is a run of letters, digits, combining marks, `-`, `'` and inner dots line break, and it does nothing when no word follows. Only one word is added, so a multi-word town ("Frankfurt am Main") keeps its tail. +#### Protected terms + +The opposite failure is a value the model tags although it must reach the +upstream model: a business name read as a surname (`Hotel Seeblick` -> +`Hotel [REDACTED:ner:LASTNAME]`) turns an invoice into one the model cannot +categorise. `protected_terms` lists such values; `protected_terms_files` +adds files inside the models path with one term per line (`#` starts a +comment), re-read whenever they change, so an external process can keep the +list current without a restart: + +```yaml +pii_detection: + protected_terms: + - Hotel Seeblick + protected_terms_files: + - privacy-filter/known-suppliers.txt +``` + +Each whole-word, case-insensitive occurrence (whitespace inside a term +matches any whitespace run) is replaced by a neutral placeholder before the +detector runs and restored afterwards. The detector never sees the value, +so it cannot tag it or let it colour the context around it, and the parts +of a detection that fall on the placeholder are cut out. Event offsets +still address the original text. Terms shorter than three characters are +ignored, and a missing or unreadable file is skipped with a warning (fewer +protected terms means more masking, never less). + +A protected term passes **everywhere** in the text: if the same string is +also a person's name, that occurrence passes too. Keep the list to values +that are safe to send. + ### Pattern detector tier NER is the wrong tool for high-entropy, highly-regular **secrets** - API keys,