mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 17:18:49 -04:00
fix(pii): bound the protected matcher cache
Term-file reloads retain each compiled matcher indefinitely. Limit the cache to 32 lists and verify that evicted lists still match on reload. Assisted-by: Codex:GPT-6 Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>
This commit is contained in:
1 parent
01f71772a5
commit
dce0eacfcf
2 files changed
+36
-6
No files matched your search
@@ -35,10 +35,18 @@ type shield struct {
|
||||
scanStart, scanEnd int
|
||||
}
|
||||
|
||||
var protectedMatchers sync.Map // key: "\x00"-joined normalised terms -> *regexp.Regexp
|
||||
// File reloads can produce indefinitely many distinct lists. Retain only a
|
||||
// bounded set of compiled matchers; eviction changes no detection policy.
|
||||
var protectedMatchers = struct {
|
||||
sync.Mutex
|
||||
entries map[string]*regexp.Regexp
|
||||
order []string
|
||||
}{entries: make(map[string]*regexp.Regexp)}
|
||||
|
||||
// protectedMatcher compiles (once per distinct term list) a
|
||||
// case-insensitive matcher for the terms, longest first so that "Hotel
|
||||
const maxProtectedMatchers = 32
|
||||
|
||||
// protectedMatcher caches a case-insensitive matcher for the terms,
|
||||
// longest first so that "Hotel
|
||||
// Seeblick Spa" wins over "Hotel Seeblick". Whitespace inside a term
|
||||
// matches any whitespace run, so a term still matches across a line wrap.
|
||||
// Returns nil when no usable term remains.
|
||||
@@ -62,8 +70,10 @@ func protectedMatcher(terms []string) *regexp.Regexp {
|
||||
})
|
||||
norm = slices.Compact(norm)
|
||||
key := strings.Join(norm, "\x00")
|
||||
if re, ok := protectedMatchers.Load(key); ok {
|
||||
return re.(*regexp.Regexp)
|
||||
protectedMatchers.Lock()
|
||||
defer protectedMatchers.Unlock()
|
||||
if re, ok := protectedMatchers.entries[key]; ok {
|
||||
return re
|
||||
}
|
||||
alts := make([]string, len(norm))
|
||||
for i, t := range norm {
|
||||
@@ -74,7 +84,12 @@ func protectedMatcher(terms []string) *regexp.Regexp {
|
||||
alts[i] = strings.Join(words, `\s+`)
|
||||
}
|
||||
re := regexp.MustCompile(`(?i)(?:` + strings.Join(alts, "|") + `)`)
|
||||
protectedMatchers.Store(key, re)
|
||||
if len(protectedMatchers.order) == maxProtectedMatchers {
|
||||
delete(protectedMatchers.entries, protectedMatchers.order[0])
|
||||
protectedMatchers.order = protectedMatchers.order[1:]
|
||||
}
|
||||
protectedMatchers.entries[key] = re
|
||||
protectedMatchers.order = append(protectedMatchers.order, key)
|
||||
return re
|
||||
}
|
||||
|
||||
|
||||
@@ -155,3 +155,18 @@ var _ = Describe("LoadProtectedTerms", func() {
|
||||
To(Equal([]string{"x-inline"}))
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("Protected matcher cache", func() {
|
||||
It("releases obsolete term lists while preserving matching after eviction", func() {
|
||||
terms := []string{"Evicted Supplier"}
|
||||
first := protectedMatcher(terms)
|
||||
for i := 0; i < 64; i++ {
|
||||
protectedMatcher([]string{strings.Repeat("x", i+3)})
|
||||
}
|
||||
reloaded := protectedMatcher(terms)
|
||||
Expect(reloaded).NotTo(BeIdenticalTo(first), "old file revisions must not remain cached indefinitely")
|
||||
text, regions := shieldProtected("Evicted Supplier, customer Lena", reloaded)
|
||||
Expect(text).To(Equal(protectedPlaceholder + ", customer Lena"))
|
||||
Expect(regions).To(HaveLen(1))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user