mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
fix(routing): keep embedder fingerprint stable across probed reasoning
A chat completion sent to an embedding model whose config sets use_tokenizer_template runs the thinking probe, and persistProbedReasoning writes the probed reasoning slots back into the shared model config. EmbedderFingerprint hashed the whole config, so that single request moved the fingerprint and every knn router built on the embedder failed with ErrLiveEmbeddingMismatch (HTTP 503) until LocalAI was restarted. Reasoning settings never change the embedding space, so leave them out of the hash. Docs-Not-Needed: internal fingerprint computation, no user-facing config, flag or API change Assisted-by: Claude Code:claude-opus-5-5 Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>
This commit is contained in:
1 parent
0102751b31
commit
46163dc1d4
2 files changed
+30
-1
No files matched your search
@@ -13,6 +13,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/backend"
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/services/routing/corpus"
|
||||
"github.com/mudler/LocalAI/pkg/reasoning"
|
||||
"gopkg.in/yaml.v3"
|
||||
)
|
||||
|
||||
@@ -119,7 +120,14 @@ func (a *Application) EmbedderFingerprint(modelName string) (string, error) {
|
||||
if cfg == nil {
|
||||
return "", fmt.Errorf("embedding model %q not available", modelName)
|
||||
}
|
||||
raw, err := yaml.Marshal(cfg)
|
||||
// Reasoning never changes the embedding space, but a chat request against
|
||||
// this model may persist probed reasoning slots into the shared config
|
||||
// (persistProbedReasoning). Hashing them would move the fingerprint on
|
||||
// such a request and leave every knn router built on this embedder failing
|
||||
// with ErrLiveEmbeddingMismatch until restart.
|
||||
hashed := *cfg
|
||||
hashed.ReasoningConfig = reasoning.Config{}
|
||||
raw, err := yaml.Marshal(&hashed)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("fingerprint embedding model %q config: %w", modelName, err)
|
||||
}
|
||||
|
||||
@@ -137,6 +137,27 @@ var _ = Describe("router_factories lazy config resolution", func() {
|
||||
Expect(updated).NotTo(Equal(replaced))
|
||||
})
|
||||
|
||||
It("stays stable when a chat request persists probed reasoning slots", func() {
|
||||
// A chat completion sent to an embedding model whose config sets
|
||||
// use_tokenizer_template runs the thinking probe, and
|
||||
// persistProbedReasoning writes the probed reasoning slots back into
|
||||
// the shared config. Reasoning never changes the embedding space, so
|
||||
// the fingerprint must not move — otherwise every knn router built
|
||||
// on this embedder fails with ErrLiveEmbeddingMismatch until restart.
|
||||
writeCfg("emb-probe", "llama-cpp")
|
||||
before, err := app.EmbedderFingerprint("emb-probe")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
disable, prefill := true, false
|
||||
app.backendLoader.UpdateModelConfig("emb-probe", func(c *config.ModelConfig) {
|
||||
c.ReasoningConfig.DisableReasoning = &disable
|
||||
c.ReasoningConfig.DisableReasoningTagPrefill = &prefill
|
||||
})
|
||||
after, err := app.EmbedderFingerprint("emb-probe")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(after).To(Equal(before))
|
||||
})
|
||||
|
||||
It("rejects an unknown model", func() {
|
||||
_, err := app.EmbedderFingerprint("missing")
|
||||
Expect(err).To(HaveOccurred())
|
||||
|
||||
Reference in new issue
Block a user