mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
1cd7d63c7b
#10970 gave the four PredictOptions RPCs a model-identity check so a backend reached through a stale distributed route rejects the request instead of answering from whatever model it holds (#10952). Every other modality shares that exposure: the route is cached by host:port, a worker can recycle a stopped backend's port for another model's backend, and a liveness-only probe cannot tell a stale row from a valid one. Extends the same mechanism to the 21 remaining request messages that reach a backend through the router, using the pattern #10970 established rather than a parallel one: - proto: ModelIdentity on each modality request message. - controller: populated from ModelConfig.Model at the call site that also builds ModelOptions, so load-time and request-time values are equal by construction. - backends: one generic guard in pkg/grpc/server.go (27 Go backends), the method set in backend/python/common (36 Python backends), llama-cpp (AudioTranscription/Stream, Rerank, Score) and privacy-filter (TokenClassify). - reconcile already drops the stale row on IsModelMismatch; no change. TTSRequest and SoundGenerationRequest get a SEPARATE ModelIdentity field rather than reusing their existing `model`: FileStagingClient rewrites `model` to a worker-local path, so comparing it would reject valid requests in exactly the configuration this guards. AudioEncode/AudioDecode are deliberately left unguarded: the opus codec backend is loaded from a literal rather than a ModelConfig, so no value carries the equality guarantee the comparison depends on. The four bidirectional stream RPCs are out of scope; they bypass reconcile. Empty means skip on both sides, so an old controller, an old backend, and the bare request structs in tests/e2e-backends all keep working. Assisted-by: Claude Code:claude-opus-4-8 [Read] [Edit] [Bash] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
62 lines
2.7 KiB
Go
62 lines
2.7 KiB
Go
package backend
|
|
|
|
// Specs for the TTSRequest assembly that carries the per-request
|
|
// instructions/params from the OpenAI `instructions` field (and the LocalAI
|
|
// `params` extension) through to the gRPC boundary. Before this plumbing the
|
|
// instruction value was dropped before reaching the backend; these specs pin
|
|
// that it now survives, and that the empty case stays backward compatible.
|
|
|
|
import (
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("newTTSRequest", func() {
|
|
It("attaches the instructions when a per-request value is set", func() {
|
|
req := newTTSRequest("hi", "/m", "alloy", "/out.wav", "en", "cheerful narrator", nil, "m")
|
|
Expect(req.Instructions).ToNot(BeNil())
|
|
Expect(req.GetInstructions()).To(Equal("cheerful narrator"))
|
|
Expect(req.GetText()).To(Equal("hi"))
|
|
Expect(req.GetVoice()).To(Equal("alloy"))
|
|
Expect(req.GetDst()).To(Equal("/out.wav"))
|
|
Expect(req.GetLanguage()).To(Equal("en"))
|
|
})
|
|
|
|
It("leaves instructions unset when empty so backends fall back to YAML", func() {
|
|
req := newTTSRequest("hi", "/m", "", "/out.wav", "", "", nil, "m")
|
|
Expect(req.Instructions).To(BeNil())
|
|
Expect(req.GetInstructions()).To(Equal(""))
|
|
})
|
|
|
|
It("forwards per-request params through to the backend", func() {
|
|
params := map[string]string{"exaggeration": "0.7", "cfg_weight": "0.3"}
|
|
req := newTTSRequest("hi", "/m", "", "/out.wav", "", "", params, "m")
|
|
Expect(req.GetParams()).To(HaveKeyWithValue("exaggeration", "0.7"))
|
|
Expect(req.GetParams()).To(HaveKeyWithValue("cfg_weight", "0.3"))
|
|
})
|
|
|
|
It("leaves params nil when none are supplied", func() {
|
|
req := newTTSRequest("hi", "/m", "", "/out.wav", "", "", nil, "m")
|
|
Expect(req.GetParams()).To(BeNil())
|
|
})
|
|
})
|
|
|
|
// TTSRequest carries TWO model strings and they are not interchangeable.
|
|
// `Model` is a path that FileStagingClient.TTS rewrites into a worker-local
|
|
// absolute path in distributed mode; `ModelIdentity` is the untranslated
|
|
// ModelConfig.Model the backend compares against what it loaded (#10952).
|
|
// Comparing `Model` instead would reject valid requests in exactly the
|
|
// configuration this guards, so these specs pin them as separate inputs.
|
|
var _ = Describe("newTTSRequest model identity", func() {
|
|
It("carries the untranslated identity alongside the staged model path", func() {
|
|
req := newTTSRequest("hi", "/worker/local/staged.onnx", "", "/out.wav", "", "", nil, "voices/piper-en.onnx")
|
|
Expect(req.GetModelIdentity()).To(Equal("voices/piper-en.onnx"))
|
|
Expect(req.GetModel()).To(Equal("/worker/local/staged.onnx"))
|
|
})
|
|
|
|
It("keeps the identity empty when the config names no model, so the backend skips", func() {
|
|
req := newTTSRequest("hi", "/m", "", "/out.wav", "", "", nil, "")
|
|
Expect(req.GetModelIdentity()).To(BeEmpty())
|
|
})
|
|
})
|