mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-08-02 11:30:44 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(distributed): reject wrong-model requests on the remaining modalities (#10990)

Browse Source 
#10970 gave the four PredictOptions RPCs a model-identity check so a
backend reached through a stale distributed route rejects the request
instead of answering from whatever model it holds (#10952). Every other
modality shares that exposure: the route is cached by host:port, a worker
can recycle a stopped backend's port for another model's backend, and a
liveness-only probe cannot tell a stale row from a valid one.

Extends the same mechanism to the 21 remaining request messages that reach
a backend through the router, using the pattern #10970 established rather
than a parallel one:

- proto: ModelIdentity on each modality request message.
- controller: populated from ModelConfig.Model at the call site that also
  builds ModelOptions, so load-time and request-time values are equal by
  construction.
- backends: one generic guard in pkg/grpc/server.go (27 Go backends), the
  method set in backend/python/common (36 Python backends), llama-cpp
  (AudioTranscription/Stream, Rerank, Score) and privacy-filter
  (TokenClassify).
- reconcile already drops the stale row on IsModelMismatch; no change.

TTSRequest and SoundGenerationRequest get a SEPARATE ModelIdentity field
rather than reusing their existing `model`: FileStagingClient rewrites
`model` to a worker-local path, so comparing it would reject valid
requests in exactly the configuration this guards.

AudioEncode/AudioDecode are deliberately left unguarded: the opus codec
backend is loaded from a literal rather than a ModelConfig, so no value
carries the equality guarantee the comparison depends on. The four
bidirectional stream RPCs are out of scope; they bypass reconcile.

Empty means skip on both sides, so an old controller, an old backend, and
the bare request structs in tests/e2e-backends all keep working.


Assisted-by: Claude Code:claude-opus-4-8 [Read] [Edit] [Bash]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
mudler's LocalAI [bot]
2026-07-20 21:58:19 +02:00
committed by GitHub
parent a784cf669f
commit 1cd7d63c7b
29 changed files with 1104 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files

101
backend/backend.proto
View File

@@ -136,6 +136,10 @@ message MetricsResponse {
message TokenClassifyRequest {
string text = 1;
float threshold = 2;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 3;
}
// TokenClassifyEntity is one detected entity span. Byte offsets are
@@ -173,6 +177,10 @@ message ScoreRequest {
// candidates differ in length and the consumer wants a per-token
// measure comparable across them (PMI-style scoring).
bool length_normalize = 4;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}
// CandidateScore is one row in the ScoreResponse, matching by index
@@ -204,6 +212,10 @@ message RerankRequest {
string query = 1;
repeated string documents = 2;
int32 top_n = 3;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}
message RerankResult {
@@ -336,7 +348,14 @@ message PredictOptions {
// paths (core/services/nodes/file_staging_client.go), so in distributed mode
// they already differ from the load-time value and comparing them would
// reject valid requests. Extending identity to those RPCs needs a separate
// field carrying the untranslated value.
// field carrying the untranslated value - which is exactly what
// TTSRequest.ModelIdentity and SoundGenerationRequest.ModelIdentity are.
//
// Every other request message that reaches a backend through the distributed
// router now carries the same ModelIdentity field, populated from the same
// ModelConfig.Model. FileStagingClient rewrites Src/Dst/Voice/Model/
// StartImage/EndImage/Audio and never ModelIdentity, so what the backend
// compares is always what the controller sent.
string ModelIdentity = 54;
// 24 was never assigned; reserve it so it is not silently reused.
@@ -510,6 +529,10 @@ message TranscriptRequest {
float temperature = 8;
repeated string timestamp_granularities = 9;
bool stream = 10;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 11;
}
message TranscriptResult {
@@ -588,6 +611,10 @@ message GenerateImageRequest {
// Reference images for models that support them (e.g., Flux Kontext)
repeated string ref_images = 12;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 13;
}
message GenerateVideoRequest {
@@ -607,6 +634,10 @@ message GenerateVideoRequest {
// Backend-specific per-request generation parameters. Values are strings
// and are validated/coerced by the selected backend.
map<string, string> params = 14;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 15;
}
message TTSRequest {
@@ -624,10 +655,26 @@ message TTSRequest {
// (e.g. Chatterbox exaggeration/cfg_weight/temperature). Values are strings and
// coerced by the backend; unset leaves the backend's configured defaults.
map<string, string> params = 7;
// ModelIdentity is a SEPARATE field from `model` above and carries the
// UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
// request that reached it through a stale distributed route (#10952).
//
// `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
// SoundGeneration path rewrite it into a worker-local absolute path
// (core/services/nodes/file_staging_client.go), while the load-time value is
// untranslated. In distributed mode - exactly the configuration this guards -
// the two already differ, so comparing them would reject valid requests.
//
// Empty means "no identity supplied" and backends MUST skip the check.
string ModelIdentity = 8;
}
message VADRequest {
repeated float audio = 1;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 2;
}
message VADSegment {
@@ -659,6 +706,10 @@ message DiarizeRequest {
float min_duration_on = 8; // discard segments shorter than this (seconds); 0 = backend default
float min_duration_off = 9; // merge gaps shorter than this (seconds); 0 = backend default
bool include_text = 10; // when the backend can emit per-segment transcript for free, ask it to populate `text`
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 11;
}
message DiarizeSegment {
@@ -693,6 +744,18 @@ message SoundGenerationRequest {
optional string language = 14;
optional string timesignature = 15;
optional bool instrumental = 17;
// ModelIdentity is a SEPARATE field from `model` above and carries the
// UNTRANSLATED controller-side ModelConfig.Model, so a backend can reject a
// request that reached it through a stale distributed route (#10952).
//
// `model` cannot be reused for this: FileStagingClient.TTS/.TTSStream and the
// SoundGeneration path rewrite it into a worker-local absolute path
// (core/services/nodes/file_staging_client.go), while the load-time value is
// untranslated. In distributed mode - exactly the configuration this guards -
// the two already differ, so comparing them would reject valid requests.
//
// Empty means "no identity supplied" and backends MUST skip the check.
string ModelIdentity = 18;
}
message TokenizationResponse {
@@ -732,6 +795,10 @@ message DetectOptions {
repeated float points = 3; // Point coordinates as [x1, y1, label1, x2, y2, label2, ...] (label: 1=pos, 0=neg)
repeated float boxes = 4; // Box coordinates as [x1, y1, x2, y2, ...]
float threshold = 5; // Detection confidence threshold
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 6;
}
message Detection {
@@ -754,6 +821,10 @@ message SoundDetectionRequest {
string src = 1; // audio file path (LocalAI writes the upload to disk)
int32 top_k = 2; // number of top tags to return (0 = all classes)
float threshold = 3; // optional: drop tags scoring below this
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}
message SoundClass {
@@ -778,6 +849,10 @@ message DepthRequest {
bool include_points = 7; // back-project to a 3D point cloud (DualDPT)
float points_conf_thresh = 8; // keep points with confidence >= this threshold
repeated string exports = 9; // requested exports: "glb", "colmap"
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 10;
}
message DepthResponse {
@@ -809,6 +884,10 @@ message FaceVerifyRequest {
string img2 = 2; // base64-encoded image
float threshold = 3; // cosine-distance threshold; 0 = use backend default
bool anti_spoofing = 4; // run MiniFASNet liveness on each image; failed liveness forces verified=false
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}
message FaceVerifyResponse {
@@ -830,6 +909,10 @@ message FaceAnalyzeRequest {
string img = 1; // base64-encoded image
repeated string actions = 2; // subset of ["age","gender","emotion","race"]; empty = all-supported
bool anti_spoofing = 3;
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 4;
}
message FaceAnalysis {
@@ -862,6 +945,10 @@ message VoiceVerifyRequest {
string audio2 = 2; // path to second audio clip
float threshold = 3; // cosine-distance threshold; 0 = use backend default
bool anti_spoofing = 4; // reserved for future AASIST bolt-on
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}
message VoiceVerifyResponse {
@@ -876,6 +963,10 @@ message VoiceVerifyResponse {
message VoiceAnalyzeRequest {
string audio = 1; // path to audio clip
repeated string actions = 2; // subset of ["age","gender","emotion"]; empty = all-supported
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 3;
}
message VoiceAnalysis {
@@ -894,6 +985,10 @@ message VoiceAnalyzeResponse {
message VoiceEmbedRequest {
string audio = 1; // path to audio clip
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 2;
}
message VoiceEmbedResponse {
@@ -988,6 +1083,10 @@ message AudioTransformRequest {
string reference_path = 2; // optional auxiliary; empty => zero-fill
string dst = 3; // required, output file path
map<string, string> params = 4; // backend-specific tuning
// ModelIdentity names the model this request is for; see
// PredictOptions.ModelIdentity for the full rationale. Empty means "no
// identity supplied" and backends MUST skip the check.
string ModelIdentity = 5;
}
message AudioTransformResult {

14
backend/cpp/llama-cpp/grpc-server.cpp
View File

@@ -1416,7 +1416,11 @@ public:
// field and for the synthetic PredictOptions this server builds internally
// for ASR, and the loaded side is empty when such a controller performed
// the load. A false rejection is worse than the miss it prevents.
grpc::Status checkModelIdentity(const backend::PredictOptions* request) {
// Templated over the request type: every guarded request message exposes
// modelidentity(), and one body keeps the rule identical across modalities
// rather than repeating it per RPC.
template <typename Request>
grpc::Status checkModelIdentity(const Request* request) {
if (request == nullptr || request->modelidentity().empty()) {
return grpc::Status::OK;
}
@@ -2846,6 +2850,8 @@ public:
}
grpc::Status Rerank(ServerContext* context, const backend::RerankRequest* request, backend::RerankResult* rerankResult) override {
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
if (!params_base.embedding || params_base.pooling_type != LLAMA_POOLING_TYPE_RANK) {
return grpc::Status(grpc::StatusCode::UNIMPLEMENTED, "This server does not support reranking. Start it with `--reranking` and without `--embedding`");
}
@@ -2970,6 +2976,8 @@ public:
grpc::Status Score(ServerContext* context, const backend::ScoreRequest* request, backend::ScoreResponse* response) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
if (params_base.model.path.empty()) {
return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
}
@@ -3428,6 +3436,8 @@ public:
backend::TranscriptResult* response) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
backend::Reply reply;
grpc::Status st = runTranscriptionAsCompletion(context, request, &reply);
@@ -3446,6 +3456,8 @@ public:
grpc::ServerWriter<backend::TranscriptStreamResponse>* writer) override {
auto auth = checkAuth(context);
if (!auth.ok()) return auth;
auto identity = checkModelIdentity(request);
if (!identity.ok()) return identity;
// Buffered streaming: run the transcription as a normal chat
// completion, then emit one delta + one final event. Real

36
backend/cpp/privacy-filter/grpc-server.cpp
View File

@@ -41,6 +41,11 @@ namespace {
// per loaded model. g_mu guards (re)load against in-flight classification.
std::mutex g_mu;
pf_ctx * g_ctx = nullptr;
// The ModelOptions.Model this process loaded, compared against
// TokenClassifyRequest.ModelIdentity so a request that arrived through a stale
// distributed route is rejected rather than answered from the wrong model
// (#10952). Guarded by g_mu like the rest of the engine state.
std::string g_loaded_model_identity;
std::atomic<Server *> g_server{nullptr};
// Resolve the device string the engine expects ("cpu" / "gpu" / "cuda" /
@@ -113,17 +118,48 @@ public:
}
g_ctx = ctx;
// Record what we loaded so TokenClassify can reject a request meant
// for a different model. request->model(), not modelfile(): it is the
// value the controller also sends as ModelIdentity, and the two are
// read from the same ModelConfig.Model (#10952).
g_loaded_model_identity = request->model();
result->set_success(true);
result->set_message("privacy-filter loaded (" + device + ")");
return GStatus::OK;
}
// checkModelIdentity mirrors pkg/grpc/server.go,
// backend/python/common/model_identity.py and the llama-cpp server. In
// distributed mode a worker can recycle a stopped backend's gRPC port for
// another model's backend, and the controller's liveness-only probe cannot
// tell a stale cached route from a valid one, so the backend has to catch
// it. Either side empty means "skip": the request side is empty for a
// controller that predates the field, the loaded side when such a
// controller performed the load. A false rejection is worse than the miss.
// Callers must already hold g_mu.
GStatus checkModelIdentity(const backend::TokenClassifyRequest * request) {
if (request == nullptr || request->modelidentity().empty()) {
return GStatus::OK;
}
if (g_loaded_model_identity.empty() ||
g_loaded_model_identity == request->modelidentity()) {
return GStatus::OK;
}
// NOT_FOUND plus this exact sentinel is the cross-language contract
// the router matches on (grpcerrors.ModelMismatchSentinel).
return GStatus(StatusCode::NOT_FOUND,
"privacy-filter: model identity mismatch: loaded \"" +
g_loaded_model_identity + "\", requested \"" +
request->modelidentity() + "\"");
}
GStatus TokenClassify(ServerContext *, const backend::TokenClassifyRequest * request,
backend::TokenClassifyResponse * response) override {
std::lock_guard<std::mutex> lock(g_mu);
if (!g_ctx) {
return GStatus(StatusCode::FAILED_PRECONDITION, "Model not loaded");
}
if (GStatus id = checkModelIdentity(request); !id.ok()) return id;
const std::string & text = request->text();
if (text.empty()) {

52
backend/python/common/model_identity.py
View File

@@ -6,7 +6,8 @@ model's backend, and the controller's health probe checks liveness rather than
identity, so the request is dispatched to whatever now occupies the port and
the caller gets a silent wrong-model answer (#10952).
PredictOptions.ModelIdentity carries the model the request is for, so the
Every request message that reaches a backend through the distributed router
carries a ModelIdentity field naming the model the request is for, so the
backend can reject it at the point of use. This module enforces that for every
Python backend at once: all 36 of them build their server through
grpc_auth.get_auth_interceptors(), so wiring it there needs no per-backend
@@ -31,16 +32,51 @@ MODEL_MISMATCH_SENTINEL = "model identity mismatch"
_LOAD_METHOD = "/backend.Backend/LoadModel"
# The four RPCs that carry PredictOptions. Nothing else has an identity field.
# TTS and SoundGeneration are excluded on purpose: their `model` field is
# already rewritten to a worker-local path by the controller's
# FileStagingClient, so comparing it would reject valid requests.
# Every RPC whose request message carries a ModelIdentity field. This set IS
# the enforcement surface for all 36 Python backends: an RPC missing here is
# silently unprotected, so model_identity_test.py pins the full list.
#
# The guard reads request.ModelIdentity generically, so nothing here is
# modality-specific — a backend that does not implement an RPC simply never
# sees it.
#
# TTS and SoundGeneration are guarded on ModelIdentity, NOT on their `model`
# field: the controller's FileStagingClient rewrites `model` to a worker-local
# absolute path, so comparing that would reject valid requests in distributed
# mode. ModelIdentity is a separate, untranslated field for exactly that reason.
#
# AudioEncode/AudioDecode are absent deliberately: the opus codec backend they
# target is loaded from a literal rather than a ModelConfig, so no value carries
# the load-time/request-time equality guarantee this comparison depends on.
_GUARDED_METHODS = frozenset(
(
# PredictOptions RPCs (#10970)
"/backend.Backend/Predict",
"/backend.Backend/PredictStream",
"/backend.Backend/Embedding",
"/backend.Backend/TokenizeString",
# Remaining modalities
"/backend.Backend/GenerateImage",
"/backend.Backend/GenerateVideo",
"/backend.Backend/TTS",
"/backend.Backend/TTSStream",
"/backend.Backend/SoundGeneration",
"/backend.Backend/AudioTranscription",
"/backend.Backend/AudioTranscriptionStream",
"/backend.Backend/Detect",
"/backend.Backend/Depth",
"/backend.Backend/FaceVerify",
"/backend.Backend/FaceAnalyze",
"/backend.Backend/VoiceVerify",
"/backend.Backend/VoiceAnalyze",
"/backend.Backend/VoiceEmbed",
"/backend.Backend/Rerank",
"/backend.Backend/TokenClassify",
"/backend.Backend/Score",
"/backend.Backend/VAD",
"/backend.Backend/Diarize",
"/backend.Backend/SoundDetection",
"/backend.Backend/AudioTransform",
)
)
@@ -87,8 +123,10 @@ class ModelIdentityState:
def _rebuild(handler, behavior):
"""Return a copy of `handler` with its behavior replaced.
Only unary-request handlers are ever passed here: LoadModel and the four
guarded RPCs all take a single request message.
Only unary-request handlers are ever passed here: LoadModel and every
entry in _GUARDED_METHODS take a single request message. The bidirectional
streams (AudioTranscriptionLive, AudioTransformStream, AudioToAudioStream,
Forward) are not guarded and never reach this function.
"""
if handler.response_streaming:
return grpc.unary_stream_rpc_method_handler(

88
backend/python/common/model_identity_test.py
View File

@@ -214,9 +214,95 @@ class TestInterceptorBehavior(unittest.TestCase):
def test_unguarded_rpcs_pass_through_untouched(self):
handler = _handler(lambda request, context: "served")
wrapped = self._intercept("/backend.Backend/TTS", handler)
wrapped = self._intercept("/backend.Backend/Health", handler)
self.assertIs(wrapped, handler)
# Every modality request message now carries a ModelIdentity field, so every
# modality RPC shares the guard. The set below is the enforcement surface for
# all 36 Python backends at once: an RPC missing from it is silently
# unprotected, which is the failure mode this class exists to catch.
_EXPECTED_MODALITY_METHODS = (
"/backend.Backend/GenerateImage",
"/backend.Backend/GenerateVideo",
"/backend.Backend/TTS",
"/backend.Backend/TTSStream",
"/backend.Backend/SoundGeneration",
"/backend.Backend/AudioTranscription",
"/backend.Backend/AudioTranscriptionStream",
"/backend.Backend/Detect",
"/backend.Backend/Depth",
"/backend.Backend/FaceVerify",
"/backend.Backend/FaceAnalyze",
"/backend.Backend/VoiceVerify",
"/backend.Backend/VoiceAnalyze",
"/backend.Backend/VoiceEmbed",
"/backend.Backend/Rerank",
"/backend.Backend/TokenClassify",
"/backend.Backend/Score",
"/backend.Backend/VAD",
"/backend.Backend/Diarize",
"/backend.Backend/SoundDetection",
"/backend.Backend/AudioTransform",
)
class TestModalityMethods(unittest.TestCase):
def setUp(self):
self.interceptor = model_identity.ModelIdentityInterceptor()
self.interceptor.state.record("a.gguf")
self.served = []
def _call(self, method, identity, response_streaming=False):
def behavior(request, context):
self.served.append(method)
return "served"
handler = _handler(behavior, response_streaming=response_streaming)
wrapped = self.interceptor.intercept_service(
lambda _: handler, _FakeCallDetails(method)
)
fn = wrapped.unary_stream if response_streaming else wrapped.unary_unary
return fn(_Request(ModelIdentity=identity), _FakeContext())
def test_every_modality_rpc_is_guarded(self):
for method in _EXPECTED_MODALITY_METHODS:
with self.subTest(method=method):
self.assertIn(
method,
model_identity._GUARDED_METHODS,
"{} is unprotected on all Python backends".format(method),
)
def test_every_modality_rpc_rejects_a_mismatch(self):
for method in _EXPECTED_MODALITY_METHODS:
streaming = method.endswith("Stream")
with self.subTest(method=method):
with self.assertRaises(_Aborted):
self._call(method, "b.gguf", response_streaming=streaming)
self.assertEqual(self.served, [], "no request may reach the model")
def test_every_modality_rpc_serves_a_match(self):
for method in _EXPECTED_MODALITY_METHODS:
streaming = method.endswith("Stream")
self._call(method, "a.gguf", response_streaming=streaming)
self.assertEqual(len(self.served), len(_EXPECTED_MODALITY_METHODS))
# Compatibility: an old controller sends nothing, and the e2e backend suite
# drives real backends with bare request structs.
def test_every_modality_rpc_serves_without_an_identity(self):
for method in _EXPECTED_MODALITY_METHODS:
streaming = method.endswith("Stream")
self._call(method, "", response_streaming=streaming)
self.assertEqual(len(self.served), len(_EXPECTED_MODALITY_METHODS))
# AudioEncode/AudioDecode stay out: the opus codec backend is loaded from a
# literal, not a ModelConfig, so no value carries the structural guarantee
# the comparison depends on.
def test_codec_rpcs_stay_unguarded(self):
for method in ("/backend.Backend/AudioEncode", "/backend.Backend/AudioDecode"):
self.assertNotIn(method, model_identity._GUARDED_METHODS)
if __name__ == "__main__":
unittest.main()