mirror of
https://github.com/mudler/LocalAI.git
synced 2026-08-02 03:20:12 -04:00
fix(distributed): reject wrong-model requests on the remaining modalities (#10990)
#10970 gave the four PredictOptions RPCs a model-identity check so a backend reached through a stale distributed route rejects the request instead of answering from whatever model it holds (#10952). Every other modality shares that exposure: the route is cached by host:port, a worker can recycle a stopped backend's port for another model's backend, and a liveness-only probe cannot tell a stale row from a valid one. Extends the same mechanism to the 21 remaining request messages that reach a backend through the router, using the pattern #10970 established rather than a parallel one: - proto: ModelIdentity on each modality request message. - controller: populated from ModelConfig.Model at the call site that also builds ModelOptions, so load-time and request-time values are equal by construction. - backends: one generic guard in pkg/grpc/server.go (27 Go backends), the method set in backend/python/common (36 Python backends), llama-cpp (AudioTranscription/Stream, Rerank, Score) and privacy-filter (TokenClassify). - reconcile already drops the stale row on IsModelMismatch; no change. TTSRequest and SoundGenerationRequest get a SEPARATE ModelIdentity field rather than reusing their existing `model`: FileStagingClient rewrites `model` to a worker-local path, so comparing it would reject valid requests in exactly the configuration this guards. AudioEncode/AudioDecode are deliberately left unguarded: the opus codec backend is loaded from a literal rather than a ModelConfig, so no value carries the equality guarantee the comparison depends on. The four bidirectional stream RPCs are out of scope; they bypass reconcile. Empty means skip on both sides, so an old controller, an old backend, and the bare request structs in tests/e2e-backends all keep working. Assisted-by: Claude Code:claude-opus-4-8 [Read] [Edit] [Bash] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
committed by
GitHub
parent
a784cf669f
commit
1cd7d63c7b
@@ -28,8 +28,14 @@ type TranscriptionRequest struct {
|
||||
TimestampGranularities []string
|
||||
}
|
||||
|
||||
func (r *TranscriptionRequest) toProto(threads uint32) *proto.TranscriptRequest {
|
||||
// modelIdentity is ModelConfig.Model, the value the backend received as
|
||||
// ModelOptions.Model at LoadModel, so it can reject a request that reached it
|
||||
// through a stale distributed route (#10952). It is a parameter rather than a
|
||||
// TranscriptionRequest field because the request is built by HTTP handlers that
|
||||
// have no ModelConfig, while every caller of this method does.
|
||||
func (r *TranscriptionRequest) toProto(threads uint32, modelIdentity string) *proto.TranscriptRequest {
|
||||
return &proto.TranscriptRequest{
|
||||
ModelIdentity: modelIdentity,
|
||||
Dst: r.Audio,
|
||||
Language: r.Language,
|
||||
Translate: r.Translate,
|
||||
@@ -85,7 +91,7 @@ func ModelTranscriptionWithOptions(ctx context.Context, req TranscriptionRequest
|
||||
audioSnippet = trace.AudioSnippet(req.Audio, appConfig.TracingMaxBodyBytes)
|
||||
}
|
||||
|
||||
r, err := transcriptionModel.AudioTranscription(ctx, req.toProto(uint32(*modelConfig.Threads)))
|
||||
r, err := transcriptionModel.AudioTranscription(ctx, req.toProto(uint32(*modelConfig.Threads), modelConfig.Model))
|
||||
if err != nil {
|
||||
if appConfig.EnableTracing {
|
||||
errData := map[string]any{
|
||||
@@ -158,7 +164,7 @@ func ModelTranscriptionStream(ctx context.Context, req TranscriptionRequest, ml
|
||||
return err
|
||||
}
|
||||
|
||||
pbReq := req.toProto(uint32(*modelConfig.Threads))
|
||||
pbReq := req.toProto(uint32(*modelConfig.Threads), modelConfig.Model)
|
||||
pbReq.Stream = true
|
||||
|
||||
return transcriptionModel.AudioTranscriptionStream(ctx, pbReq, func(chunk *proto.TranscriptStreamResponse) {
|
||||
|
||||
Reference in New Issue
Block a user