mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 18:09:05 -04:00
465d488c90
fix(distributed): reject wrong-model requests at the backend (#10952) In distributed mode the controller caches a NodeModel row naming a backend's host:port. A worker can recycle a stopped backend's gRPC port for a different model's backend, and probeHealth verifies liveness rather than identity, so the probe succeeds against whatever now occupies the port and the request is dispatched to the wrong backend. The caller gets a silent wrong-model answer. Nothing in the request could catch this: PredictOptions had no model field, so model identity crossed the wire only in ModelOptions.Model at LoadModel time, and the cached-hit path issues no LoadModel. Every backend's "model not loaded" guard checks a nil handle, which a process holding a different model passes, so the stale row was never dropped either. Add PredictOptions.ModelIdentity and enforce it at the point of use: - The controller populates it in gRPCPredictOpts from ModelConfig.Model, the same expression ModelOptions feeds to model.WithModel and therefore the same value the backend received as ModelOptions.Model. Both are read from one config value in one function, so they are equal by construction and the comparison cannot false-reject. - Backends compare it against what they loaded and return NOT_FOUND with a fixed sentinel. Enforced in pkg/grpc/server.go (27 Go backends), an interceptor in backend/python/common (all 36 Python backends, no per-backend change), and the llama-cpp / ik-llama-cpp / ds4 C++ servers. That is every backend with real exposure: kokoros answers all four RPCs with unimplemented and privacy-filter implements none of them. - The router's reconcile drops the stale replica row on a mismatch, so the next request reloads somewhere correct. Empty means "skip the check" on both sides: a controller that predates the field sends nothing, a backend loaded by such a controller has nothing to compare, and the C++ server synthesizes PredictOptions internally for ASR. That keeps upgrades working in both directions. Scoped to the four PredictOptions RPCs. TTSRequest.model and SoundGenerationRequest.model are deliberately NOT validated: FileStagingClient already rewrites them to worker-local absolute paths, so in distributed mode they already differ from the load-time value and comparing them would reject valid requests. IsModelMismatch requires both the NOT_FOUND code and the sentinel, unlike the neighbouring helpers which accept either. insightface's Embedding returns NOT_FOUND "no face detected" on a PredictOptions RPC, and a code-only check would drop a healthy replica row on every faceless image. Assisted-by: Claude Code:claude-opus-4-8[1m] [Read] [Edit] [Bash] Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
109 lines
4.9 KiB
Go
109 lines
4.9 KiB
Go
package grpcerrors_test
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
|
|
"github.com/mudler/LocalAI/pkg/grpc/grpcerrors"
|
|
"google.golang.org/grpc/codes"
|
|
"google.golang.org/grpc/status"
|
|
)
|
|
|
|
func TestGRPCErrors(t *testing.T) {
|
|
RegisterFailHandler(Fail)
|
|
RunSpecs(t, "grpcerrors test suite")
|
|
}
|
|
|
|
var _ = Describe("grpcerrors", func() {
|
|
DescribeTable("IsModelNotLoaded",
|
|
func(err error, want bool) {
|
|
Expect(grpcerrors.IsModelNotLoaded(err)).To(Equal(want))
|
|
},
|
|
Entry("nil", nil, false),
|
|
Entry("typed via constructor", grpcerrors.ModelNotLoaded("parakeet-cpp"), true),
|
|
Entry("typed code only", status.Error(codes.FailedPrecondition, "anything"), true),
|
|
Entry("legacy message (Unknown code)", errors.New("parakeet-cpp: model not loaded"), true),
|
|
Entry("legacy message mixed case", errors.New("Backend: Model Not Loaded"), true),
|
|
Entry("unrelated error", errors.New("context deadline exceeded"), false),
|
|
Entry("unrelated grpc code", status.Error(codes.Unavailable, "connection refused"), false),
|
|
)
|
|
|
|
It("ModelNotLoaded carries FailedPrecondition", func() {
|
|
Expect(status.Code(grpcerrors.ModelNotLoaded("whisper"))).To(Equal(codes.FailedPrecondition))
|
|
})
|
|
|
|
DescribeTable("IsModelMismatch",
|
|
func(err error, want bool) {
|
|
Expect(grpcerrors.IsModelMismatch(err)).To(Equal(want))
|
|
},
|
|
Entry("nil", nil, false),
|
|
Entry("typed via constructor", grpcerrors.ModelMismatch("llama-cpp", "a.gguf", "b.gguf"), true),
|
|
// The sentinel is what makes this detectable, not the code alone.
|
|
// insightface's Embedding returns NOT_FOUND "no face detected" on a
|
|
// PredictOptions RPC (backend/python/insightface/backend.py:127). A
|
|
// code-only check would make the router drop a healthy replica row on
|
|
// every faceless image, so this entry must stay false.
|
|
Entry("insightface no-face NotFound", status.Error(codes.NotFound, "no face detected"), false),
|
|
Entry("insightface no-face in both images",
|
|
status.Error(codes.NotFound, "no face detected in one or both images"), false),
|
|
Entry("unrelated NotFound", status.Error(codes.NotFound, "Job 7 not found"), false),
|
|
// Must not overlap with the model-not-loaded signal, which the router
|
|
// handles identically today but for a different reason.
|
|
Entry("model not loaded is not a mismatch", grpcerrors.ModelNotLoaded("llama-cpp"), false),
|
|
Entry("unrelated error", errors.New("context deadline exceeded"), false),
|
|
)
|
|
|
|
It("ModelMismatch carries NotFound and names both models", func() {
|
|
err := grpcerrors.ModelMismatch("llama-cpp", "a.gguf", "b.gguf")
|
|
Expect(status.Code(err)).To(Equal(codes.NotFound))
|
|
Expect(err.Error()).To(ContainSubstring(grpcerrors.ModelMismatchSentinel))
|
|
Expect(err.Error()).To(ContainSubstring("a.gguf"))
|
|
Expect(err.Error()).To(ContainSubstring("b.gguf"))
|
|
})
|
|
|
|
It("IsModelNotLoaded does not claim a mismatch", func() {
|
|
Expect(grpcerrors.IsModelNotLoaded(grpcerrors.ModelMismatch("llama-cpp", "a", "b"))).To(BeFalse())
|
|
})
|
|
|
|
DescribeTable("IsLiveTranscriptionUnsupported",
|
|
func(err error, want bool) {
|
|
Expect(grpcerrors.IsLiveTranscriptionUnsupported(err)).To(Equal(want))
|
|
},
|
|
Entry("nil", nil, false),
|
|
Entry("typed via constructor", grpcerrors.LiveTranscriptionUnsupported("parakeet-cpp", "not a streaming model"), true),
|
|
Entry("typed code only", status.Error(codes.Unimplemented, "anything"), true),
|
|
Entry("stale stub message (Unknown code)", errors.New("rpc error: method AudioTranscriptionLive unimplemented"), true),
|
|
Entry("unrelated error", errors.New("context deadline exceeded"), false),
|
|
Entry("model not loaded is NOT unsupported", grpcerrors.ModelNotLoaded("parakeet-cpp"), false),
|
|
)
|
|
|
|
It("LiveTranscriptionUnsupported carries Unimplemented, not FailedPrecondition", func() {
|
|
err := grpcerrors.LiveTranscriptionUnsupported("parakeet-cpp", "reason")
|
|
Expect(status.Code(err)).To(Equal(codes.Unimplemented))
|
|
// FailedPrecondition is claimed by IsModelNotLoaded — the two
|
|
// signals must never alias.
|
|
Expect(grpcerrors.IsModelNotLoaded(err)).To(BeFalse())
|
|
})
|
|
|
|
DescribeTable("IsUnimplemented",
|
|
func(err error, want bool) {
|
|
Expect(grpcerrors.IsUnimplemented(err)).To(Equal(want))
|
|
},
|
|
Entry("nil", nil, false),
|
|
Entry("typed code", status.Error(codes.Unimplemented, "method Free not implemented"), true),
|
|
Entry("stale stub message (Unknown code)", errors.New("rpc error: code = Unimplemented desc = "), true),
|
|
Entry("unrelated error", errors.New("context deadline exceeded"), false),
|
|
Entry("unrelated grpc code", status.Error(codes.Unavailable, "connection refused"), false),
|
|
Entry("model not loaded is NOT unimplemented", grpcerrors.ModelNotLoaded("parakeet-cpp"), false),
|
|
)
|
|
|
|
It("StreamTranscriptionUnsupported carries Unimplemented and is not ModelNotLoaded", func() {
|
|
err := grpcerrors.StreamTranscriptionUnsupported("parakeet-cpp", "not a streaming model")
|
|
Expect(status.Code(err)).To(Equal(codes.Unimplemented))
|
|
Expect(grpcerrors.IsModelNotLoaded(err)).To(BeFalse())
|
|
})
|
|
})
|