From 763bf6fbab5ad85cc603eef062e09cd49b1bf487 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 08:22:16 +0200 Subject: [PATCH 1/7] chore: :arrow_up: Update 0xShug0/audio.cpp to `f6277c1695a83cf388a8282c1c1a8757cf626f18` (#11899) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index cb6c96553..811fc1115 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=c41ffefb44c16345c0ac57a0030f4cd3b5fbc4b8 +AUDIO_CPP_VERSION?=f6277c1695a83cf388a8282c1c1a8757cf626f18 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 5728f78f2d19fd2f91eaab6277c937e9e2afb8f3 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 08:22:41 +0200 Subject: [PATCH 2/7] chore: :arrow_up: Update ggml-org/llama.cpp to `465e49b9cea78a68b9c244ffb48d0ee24a82873d` (#11897) :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index bb60a6624..3f158f49a 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=74a7c897f049c17e7080423aa2111776eff6ebbf +LLAMA_VERSION?=465e49b9cea78a68b9c244ffb48d0ee24a82873d LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= From f94ce520cec68748053f4fd73d4c7618e3378ede Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 12:56:38 +0200 Subject: [PATCH 3/7] chore: :arrow_up: Update antirez/ds4 to `9ab705347c1775e7599ede7eb81a6255ec7dccb5` (#11808) * :arrow_up: Update antirez/ds4 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(ds4): link upstream image helpers The ds4 bump adds vision calls to the engine object. Link the new image preprocessing object into every backend target. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/ds4/CMakeLists.txt | 7 ++++--- backend/cpp/ds4/Makefile | 21 +++++++++++---------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/backend/cpp/ds4/CMakeLists.txt b/backend/cpp/ds4/CMakeLists.txt index 0535a8a44..5783db942 100644 --- a/backend/cpp/ds4/CMakeLists.txt +++ b/backend/cpp/ds4/CMakeLists.txt @@ -84,9 +84,10 @@ elseif(DS4_GPU STREQUAL "cpu") set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o") endif() -# Upstream splits distributed inference, tensor-parallel transport, the SSD -# expert cache, and layer placement into GPU-agnostic translation units. Link -# them regardless of DS4_GPU. +# Upstream splits image preprocessing, distributed inference, tensor-parallel +# transport, the SSD expert cache, and layer placement into GPU-agnostic +# translation units. Link them regardless of DS4_GPU. +list(APPEND DS4_OBJS "${DS4_DIR}/ds4_image.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_distributed.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_tp.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_ssd.o") diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index 807a01a09..455b554d8 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 +# Upstream pin lives below as DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5 # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 +DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5 DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -79,21 +79,22 @@ else endif endif -# Upstream splits distributed inference, tensor-parallel transport, the SSD -# expert cache, and layer placement into GPU-agnostic translation units. They -# are shared by every GPU mode, so append them unconditionally below. +# Upstream splits image preprocessing, distributed inference, tensor-parallel +# transport, the SSD expert cache, and layer placement into GPU-agnostic +# translation units. They are shared by every GPU mode, so append them +# unconditionally below. ifeq ($(BUILD_TYPE),cublas) CMAKE_ARGS += -DDS4_GPU=cuda - DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \ + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \ cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \ cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o else ifeq ($(UNAME_S),Darwin) CMAKE_ARGS += -DDS4_GPU=metal - DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else # CPU reference path (Linux only - macOS CPU path is broken by VM bug per ds4 README). CMAKE_ARGS += -DDS4_GPU=cpu - DS4_OBJ_TARGET := ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + DS4_OBJ_TARGET := ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o endif ifneq ($(NATIVE),true) @@ -120,9 +121,9 @@ ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) +$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET) else ifeq ($(UNAME_S),Darwin) - +$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + +$(MAKE) -C ds4 ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else - +$(MAKE) -C ds4 ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + +$(MAKE) -C ds4 ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o endif grpc-server: ds4/ds4.o From 57f802aa7ae08bb99655647d74cd802b843e3d75 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 12:56:55 +0200 Subject: [PATCH 4/7] chore: :arrow_up: Update leejet/stable-diffusion.cpp to `d8fb10c02977c8ca999f3fb4e02df9ecf10f7ba6` (#11898) * :arrow_up: Update leejet/stable-diffusion.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(stablediffusion): adapt streaming options Upstream now selects segmented weight streaming automatically and removes the stream_layers field. Keep the old LocalAI option as a no-op for existing model configurations. Assisted-by: Codex:gpt-5 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/go/stablediffusion-ggml/Makefile | 2 +- backend/go/stablediffusion-ggml/cpp/gosd.cpp | 13 ++++++------- docs/content/features/image-generation.md | 4 ++-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 1f8e70402..e39a1d09c 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=6b3edaaf32cc19e5bb2d819c788bd557eddc8eba +STABLEDIFFUSION_GGML_VERSION?=d8fb10c02977c8ca999f3fb4e02df9ecf10f7ba6 CMAKE_ARGS+=-DGGML_MAX_NAME=128 diff --git a/backend/go/stablediffusion-ggml/cpp/gosd.cpp b/backend/go/stablediffusion-ggml/cpp/gosd.cpp index 12cc4a83e..7722e8d06 100644 --- a/backend/go/stablediffusion-ggml/cpp/gosd.cpp +++ b/backend/go/stablediffusion-ggml/cpp/gosd.cpp @@ -401,7 +401,6 @@ int load_model(const char *model, char *model_path, char* options[], int threads const char *params_backend_arg = ""; const char *rpc_servers_arg = ""; const char *max_vram_arg = ""; - bool stream_layers = false; int n_threads = threads; enum sd_type_t wtype = SD_TYPE_COUNT; @@ -510,7 +509,10 @@ int load_model(const char *model, char *model_path, char* options[], int threads if (!strcmp(optname, "params_backend")) params_backend_arg = strdup(optval); if (!strcmp(optname, "rpc_servers")) rpc_servers_arg = strdup(optval); if (!strcmp(optname, "max_vram")) max_vram_arg = strdup(optval); - if (!strcmp(optname, "stream_layers")) stream_layers = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0); + if (!strcmp(optname, "stream_layers")) { + // Retained as a no-op for existing configurations. Upstream now + // selects segmented weight streaming automatically. + } // vae_decode_only is still accepted for backwards compatibility with // existing gallery configs, but upstream dropped the option (the model @@ -650,11 +652,9 @@ int load_model(const char *model, char *model_path, char* options[], int threads ctx_params.rpc_servers = env_rpc_servers; } } - // max_vram: GiB budget or per-backend spec for graph-cut segmented param - // offload ("0" = disabled, "-1" = auto). stream_layers only has effect when - // max_vram is set. + // max_vram is an optional GiB budget or per-backend spec for automatic + // graph-cut execution. A zero value uses the live free-VRAM budget. if (strlen(max_vram_arg) > 0) ctx_params.max_vram = max_vram_arg; - ctx_params.stream_layers = stream_layers; ctx_params.diffusion_flash_attn = diffusion_flash_attn; ctx_params.tae_preview_only = tae_preview_only; ctx_params.diffusion_conv_direct = diffusion_conv_direct; @@ -1438,4 +1438,3 @@ int unload() { free_sd_ctx(sd_c); return 0; } - diff --git a/docs/content/features/image-generation.md b/docs/content/features/image-generation.md index d94c0f9b4..1df6058b4 100644 --- a/docs/content/features/image-generation.md +++ b/docs/content/features/image-generation.md @@ -79,8 +79,8 @@ When a model does not fit entirely in VRAM, the following `options:` control whe |--------|---------|-------------| | `backend` | `backend:clip=cpu,vae=cuda0,diffusion=vulkan0` | Runtime (compute) backend assignment per component. Use `cpu` to place a component's compute on the CPU. Component keys include `te` (text encoder / CLIP), `vae`, `diffusion`, `controlnet`. | | `params_backend` | `params_backend:diffusion=disk,clip=cpu` | Where parameters (weights) are stored. Supports `cpu`, `disk` (mmap weights from disk to save RAM/VRAM), or per-component specs. | -| `max_vram` | `max_vram:8` or `max_vram:-1` | VRAM budget (in GiB) for graph-cut segmented parameter offload. `0` disables it, `-1` auto-selects (free VRAM minus ~1 GiB). Also accepts per-backend budgets. | -| `stream_layers` | `stream_layers:true` | Enable residency + prefetch streaming on top of `max_vram` (no effect unless `max_vram` is set). | +| `max_vram` | `max_vram:8` or `max_vram:-1` | Optional per-device VRAM budget (in GiB) for managed weights and automatic graph-cut execution. `0` uses live free VRAM without an explicit cap; a negative value reserves that many GiB of free VRAM. Also accepts per-backend budgets. | +| `stream_layers` | `stream_layers:true` | Deprecated compatibility option. Segmented weight streaming is now selected automatically, so this value is ignored. | | `rpc_servers` | `rpc_servers:localhost:50052,192.168.1.3:50052` | Comma-separated list of `host:port` RPC servers to offload compute to. | | `pulid_weights_path` | `pulid_weights_path:pulid.safetensors` | Path to PuLID-Flux weights for identity injection. | From 77b8241c51849b8b378b3c890e7156a98a8beec1 Mon Sep 17 00:00:00 2001 From: "Plamen K. Kosseff" <333840+blackd@users.noreply.github.com> Date: Mon, 7 Sep 2026 17:36:58 +0300 Subject: [PATCH 5/7] docs(integrations): add Distribution Packages section (#11904) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Community-maintained packagings that currently track releases — Homebrew, ALT Sisyphus and the Gentoo local-ai overlay — with a note that versions may lag. Placement and scope as discussed in the issue. Assisted-by: Claude:claude-fable-5 Signed-off-by: Plamen K. Kosseff --- docs/content/integrations.md | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/docs/content/integrations.md b/docs/content/integrations.md index 0bcf69e99..8d8bd4104 100644 --- a/docs/content/integrations.md +++ b/docs/content/integrations.md @@ -18,6 +18,15 @@ Feel free to open up a Pull request (by clicking at the "Edit page" below) to ge - [Helm chart](https://github.com/go-skynet/helm-charts) - Deploy LocalAI on Kubernetes - [GitHub Actions](https://github.com/marketplace/actions/start-localai) - Use LocalAI in CI/CD workflows +### Distribution Packages + +Community-maintained packages of LocalAI. Package versions and +availability may lag upstream releases. + +- [Homebrew](https://formulae.brew.sh/formula/localai) - `brew install localai` +- [ALT Sisyphus](https://packages.altlinux.org/en/sisyphus/srpms/localai/) - `localai` package +- [Gentoo overlay](https://git.ipnmod.org/packages/local-ai-overlay) - `sci-ml/local-ai` and per-backend packages, overlay `local-ai` in Gentoo's repository index + ### Web UIs - [localai-admin](https://github.com/Jirubizu/localai-admin) From fb8b7a359a56af8d24eed1b44264f2187e314a9c Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 17:12:37 +0200 Subject: [PATCH 6/7] fix(distributed): stage sound detection audio (#11907) * fix(distributed): stage sound detection audio Sound detection passes frontend temporary paths directly to remote workers, unlike transcription. Stage the WAV before classification so CED can read it without a shared temporary directory. Preserve the original request for retries and propagate staging errors without calling the backend. Cover staging, request preservation, and error handling with regression tests. Assisted-by: Codex:GPT-6 golangci-lint * test(distributed): verify routed sound staging Call sound detection through the client returned by SmartRouter.Route. This checks interface dispatch through both routing wrappers, rather than constructing FileStagingClient directly. The test fails without the sound-staging override and passes with it. Assisted-by: Codex:GPT-6 golangci-lint --------- Co-authored-by: Ettore Di Giacinto --- core/services/nodes/file_staging_client.go | 14 +++ .../file_staging_sound_detection_test.go | 94 +++++++++++++++++++ docs/content/features/audio-classification.md | 4 + 3 files changed, 112 insertions(+) create mode 100644 core/services/nodes/file_staging_sound_detection_test.go diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index bfc202c82..d6c14cf2e 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -16,6 +16,7 @@ import ( pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/xlog" ggrpc "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) // FileStagingClient wraps a grpc.Backend to transparently handle file transfer @@ -342,6 +343,19 @@ func (f *FileStagingClient) SoundGeneration(ctx context.Context, in *pb.SoundGen return result, nil } +func (f *FileStagingClient) SoundDetection(ctx context.Context, in *pb.SoundDetectionRequest, opts ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) { + if in.Src != "" && isFilePath(in.Src) { + backendPath, _, err := f.stageInputFile(ctx, requestID(), in.Src, "inputs") + if err != nil { + return nil, fmt.Errorf("staging audio for sound detection: %w", err) + } + // Keep the frontend path available if the caller retries on another node. + in = proto.Clone(in).(*pb.SoundDetectionRequest) + in.Src = backendPath + } + return f.Backend.SoundDetection(ctx, in, opts...) +} + func (f *FileStagingClient) AudioTranscription(ctx context.Context, in *pb.TranscriptRequest, opts ...ggrpc.CallOption) (*pb.TranscriptResult, error) { reqID := requestID() diff --git a/core/services/nodes/file_staging_sound_detection_test.go b/core/services/nodes/file_staging_sound_detection_test.go new file mode 100644 index 000000000..e66b18f5f --- /dev/null +++ b/core/services/nodes/file_staging_sound_detection_test.go @@ -0,0 +1,94 @@ +package nodes + +import ( + "context" + "errors" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +type soundStagingBackend struct { + grpc.Backend + request *pb.SoundDetectionRequest + err error +} + +func (b *soundStagingBackend) SoundDetection(_ context.Context, in *pb.SoundDetectionRequest, _ ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) { + b.request = in + return &pb.SoundDetectionResponse{}, b.err +} + +type soundStagingFailure struct{ FileStager } + +func (s *soundStagingFailure) EnsureRemote(context.Context, string, string, string) (string, error) { + return "", errors.New("upload failed") +} + +type soundRouteFactory struct{ client grpc.Backend } + +func (f *soundRouteFactory) NewClient(string, bool) grpc.Backend { return f.client } + +var _ = Describe("FileStagingClient sound detection", func() { + It("stages sound audio through the client returned by SmartRouter.Route", func(ctx SpecContext) { + node := &BackendNode{ID: "worker-1", Name: "worker", Address: "10.0.0.1:50051"} + reg := &fakeModelRouter{ + findAndLockNode: node, + findAndLockNM: &NodeModel{NodeID: node.ID, ModelName: "ced", Address: "10.0.0.1:9001"}, + } + backend := &soundStagingBackend{Backend: &stubBackend{healthResult: true}} + stager := &fakeFileStager{} + router := NewSmartRouter(reg, SmartRouterOptions{ + ClientFactory: &soundRouteFactory{client: backend}, + FileStager: stager, + Unloader: &fakeUnloader{}, + }) + result, err := router.Route(ctx, "ced", "ced.gguf", "ced", "", nil, false) + Expect(err).NotTo(HaveOccurred()) + Expect(result).NotTo(BeNil()) + defer result.Release() + request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf"} + _, err = result.Client.SoundDetection(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(stager.ensureCalls).To(HaveLen(1)) + Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src)) + Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key)) + Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav")) + }) + + It("stages audio on the worker without changing the caller's request", func(ctx SpecContext) { + backend := &soundStagingBackend{} + stager := &fakeFileStager{} + client := NewFileStagingClient(backend, stager, "worker-1") + request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf", TopK: 5, Threshold: 0.25} + _, err := client.SoundDetection(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(stager.ensureCalls).To(HaveLen(1)) + Expect(stager.ensureCalls[0].nodeID).To(Equal("worker-1")) + Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src)) + Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key)) + Expect(backend.request.ModelIdentity).To(Equal(request.ModelIdentity)) + Expect(backend.request.TopK).To(Equal(request.TopK)) + Expect(backend.request.Threshold).To(Equal(request.Threshold)) + Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav")) + }) + It("does not call the backend when staging fails", func(ctx SpecContext) { + backend := &soundStagingBackend{} + client := NewFileStagingClient(backend, &soundStagingFailure{}, "worker-1") + _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: "/tmp/clip.wav"}) + Expect(err).To(MatchError(ContainSubstring("upload failed"))) + Expect(backend.request).To(BeNil()) + }) + It("passes through requests without a file and preserves backend errors", func(ctx SpecContext) { + failure := errors.New("classifier failed") + backend := &soundStagingBackend{err: failure} + stager := &fakeFileStager{} + client := NewFileStagingClient(backend, stager, "worker-1") + _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{}) + Expect(err).To(MatchError(failure)) + Expect(stager.ensureCalls).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md index 045074340..09759ab72 100644 --- a/docs/content/features/audio-classification.md +++ b/docs/content/features/audio-classification.md @@ -11,6 +11,10 @@ LocalAI exposes this through the `/v1/audio/classification` endpoint, modelled a Because classification is exposed as a regular OpenAI-style endpoint, any HTTP client works - there is no Python dependency on the consumer side. +In distributed mode, LocalAI stages uploaded audio and realtime sound-detection +windows on the selected worker before classification. The API server and worker +do not need a shared temporary directory. + ## Endpoint ``` From 88d19567b8a9dbaaff8f8e210eb01868d288354a Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 17:40:27 +0200 Subject: [PATCH 7/7] feat(faces): replay saved face enrollments (#11908) Accept original embeddings and timestamps so clients can restore faces when the in-memory store restarts. Derive stable IDs from exact vectors to make registration retries preserve identity without duplicate entries. Assisted-by: Codex:GPT-6 golangci-lint Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/http/endpoints/localai/face_register.go | 30 ++++--- .../endpoints/localai/face_register_test.go | 80 +++++++++++++++++++ core/schema/localai.go | 10 ++- core/services/facerecognition/registry.go | 1 + core/services/facerecognition/replay_test.go | 67 ++++++++++++++++ .../facerecognition/store_registry.go | 23 +++++- docs/content/features/face-recognition.md | 37 ++++++++- 7 files changed, 231 insertions(+), 17 deletions(-) create mode 100644 core/http/endpoints/localai/face_register_test.go create mode 100644 core/services/facerecognition/replay_test.go diff --git a/core/http/endpoints/localai/face_register.go b/core/http/endpoints/localai/face_register.go index fbeb29e0c..9cd40b456 100644 --- a/core/http/endpoints/localai/face_register.go +++ b/core/http/endpoints/localai/face_register.go @@ -1,6 +1,7 @@ package localai import ( + "errors" "net/http" "github.com/labstack/echo/v4" @@ -33,22 +34,31 @@ func FaceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a return echo.NewHTTPError(http.StatusBadRequest, "name is required") } - img, err := decodeImageInput(input.Img) - if err != nil { - return err + if (input.Img == "") == (len(input.Embedding) == 0) { + return echo.NewHTTPError(http.StatusBadRequest, "provide exactly one of img or embedding") } - - xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) - embedding, err := backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) - if err != nil { - return mapBackendError(err) + embedding := input.Embedding + if len(embedding) == 0 { + img, err := decodeImageInput(input.Img) + if err != nil { + return err + } + xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) + embedding, err = backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) + if err != nil { + return mapBackendError(err) + } } stored, err := registry.Register(c.Request().Context(), embedding, facerecognition.Metadata{ - Name: input.Name, - Labels: input.Labels, + Name: input.Name, + RegisteredAt: input.RegisteredAt, + Labels: input.Labels, }) if err != nil { + if errors.Is(err, facerecognition.ErrInvalidEmbedding) || errors.Is(err, facerecognition.ErrDimensionMismatch) { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } return err } return c.JSON(http.StatusOK, schema.FaceRegisterResponse{ diff --git a/core/http/endpoints/localai/face_register_test.go b/core/http/endpoints/localai/face_register_test.go new file mode 100644 index 000000000..16970345a --- /dev/null +++ b/core/http/endpoints/localai/face_register_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT + +package localai_test + +import ( + "context" + "net/http" + "net/http/httptest" + "time" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/facerecognition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type registrationRecorder struct { + facerecognition.Registry + vector []float32 + meta facerecognition.Metadata + err error +} + +func (r *registrationRecorder) Register(_ context.Context, v []float32, m facerecognition.Metadata) (facerecognition.Metadata, error) { + r.vector = v + r.meta = m + m.ID = "saved-id" + return m, r.err +} + +var _ = Describe("Face registration replay", func() { + var reg *registrationRecorder + call := func(in schema.FaceRegisterRequest) (*httptest.ResponseRecorder, error) { + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/face/register", nil), rec) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &in) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{}) + // No model loader: replay must not call the embedding backend. + err := FaceRegisterEndpoint(nil, nil, nil, reg)(c) + return rec, err + } + BeforeEach(func() { reg = ®istrationRecorder{} }) + It("accepts the saved vector and timestamp without running inference", func() { + at := time.Now().UTC() + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{1, 0}, RegisteredAt: at, Labels: map[string]string{"client_id": "alice"}} + in.Model = "faces" + rec, err := call(in) + Expect(err).NotTo(HaveOccurred()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(reg.vector).To(Equal(in.Embedding)) + Expect(reg.meta.RegisteredAt).To(Equal(at)) + Expect(reg.meta.Labels).To(Equal(in.Labels)) + Expect(rec.Body.String()).To(ContainSubstring("saved-id")) + }) + It("rejects ambiguous and missing inputs before inference", func() { + for _, in := range []schema.FaceRegisterRequest{ + {Name: "Alice"}, + {Name: "Alice", Img: "image", Embedding: []float32{1, 0}}, + } { + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + Expect(reg.vector).To(BeNil()) + } + }) + It("reports invalid vectors as a client error", func() { + reg.err = facerecognition.ErrInvalidEmbedding + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{0, 0}} + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + }) +}) diff --git a/core/schema/localai.go b/core/schema/localai.go index e160badad..a40f90c8d 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -353,10 +353,12 @@ type FaceEmbedResponse struct { // FaceRegisterRequest enrolls a face into the 1:N recognition store. type FaceRegisterRequest struct { BasicModelRequest - Img string `json:"img"` - Name string `json:"name"` - Labels map[string]string `json:"labels,omitempty"` - Store string `json:"store,omitempty"` // vector store model; empty = local-store default + RegisteredAt time.Time `json:"registered_at,omitempty"` // original enrollment time when replaying a saved embedding + Embedding []float32 `json:"embedding,omitempty"` + Img string `json:"img"` + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` + Store string `json:"store,omitempty"` // vector store model; empty = local-store default } type FaceRegisterResponse struct { diff --git a/core/services/facerecognition/registry.go b/core/services/facerecognition/registry.go index adc9d9200..ae781dbb2 100644 --- a/core/services/facerecognition/registry.go +++ b/core/services/facerecognition/registry.go @@ -56,5 +56,6 @@ type Match struct { var ( ErrNotFound = errors.New("facerecognition: id not found") ErrEmptyEmbedding = errors.New("facerecognition: embedding is empty") + ErrInvalidEmbedding = errors.New("facerecognition: embedding must be finite and nonzero") ErrDimensionMismatch = errors.New("facerecognition: embedding dimension mismatch") ) diff --git a/core/services/facerecognition/replay_test.go b/core/services/facerecognition/replay_test.go new file mode 100644 index 000000000..21c965905 --- /dev/null +++ b/core/services/facerecognition/replay_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT + +package facerecognition + +import ( + "context" + "encoding/json" + "math" + "sync" + "testing" + "time" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +func TestEnrollmentReplay(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Enrollment replay") } + +type replayStore struct { + grpc.Backend + mu sync.Mutex + entries map[string][]byte +} + +func (s *replayStore) StoresSet(_ context.Context, in *pb.StoresSetOptions, _ ...ggrpc.CallOption) (*pb.Result, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i, k := range in.Keys { + b, _ := json.Marshal(k.Floats) + s.entries[string(b)] = append([]byte(nil), in.Values[i].Bytes...) + } + return &pb.Result{Success: true}, nil +} + +var _ = Describe("Enrollment replay", func() { + It("keeps the identity across registry instances and a cleared store", func(ctx SpecContext) { + storage := &replayStore{entries: map[string][]byte{}} + newRegistry := func() Registry { + return NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + } + vector := []float32{1, 0, 0, 0} + meta := Metadata{Name: "Alice", RegisteredAt: time.Now().UTC()} + first, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + again, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + storage.entries = map[string][]byte{} + restored, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(restored).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + }) + It("rejects zero and non-finite embeddings before writing", func(ctx SpecContext) { + for _, v := range [][]float32{{0, 0}, {float32(math.NaN()), 1}, {float32(math.Inf(1)), 1}} { + storage := &replayStore{entries: map[string][]byte{}} + reg := NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + _, err := reg.Register(ctx, v, Metadata{Name: "Alice"}) + Expect(err).To(HaveOccurred()) + Expect(storage.entries).To(BeEmpty()) + } + }) +}) diff --git a/core/services/facerecognition/store_registry.go b/core/services/facerecognition/store_registry.go index d4fd0d971..abf7ee1ac 100644 --- a/core/services/facerecognition/store_registry.go +++ b/core/services/facerecognition/store_registry.go @@ -2,8 +2,10 @@ package facerecognition import ( "context" + "encoding/binary" "encoding/json" "fmt" + "math" "sort" "sync" "time" @@ -57,13 +59,32 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta if r.dim != 0 && len(embedding) != r.dim { return Metadata{}, fmt.Errorf("%w: expected %d, got %d", ErrDimensionMismatch, r.dim, len(embedding)) } + var norm float64 + key := make([]byte, 4*len(embedding)) + for i, value := range embedding { + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return Metadata{}, ErrInvalidEmbedding + } + norm += float64(value) * float64(value) + // The store treats negative and positive zero as the same key. + if value == 0 { + value = 0 + } + binary.LittleEndian.PutUint32(key[i*4:], math.Float32bits(value)) + } + if norm == 0 { + return Metadata{}, ErrInvalidEmbedding + } backend, err := r.resolve(ctx, r.storeName) if err != nil { return Metadata{}, fmt.Errorf("facerecognition: resolve store: %w", err) } - meta.ID = uuid.NewString() + // The vector store upserts by the exact embedding. Derive the ID from the + // same key so replaying a saved vector preserves identity across replicas + // and after the in-memory store restarts. + meta.ID = uuid.NewSHA1(uuid.NewSHA1(uuid.NameSpaceOID, []byte(r.storeName)), key).String() if meta.RegisteredAt.IsZero() { meta.RegisteredAt = time.Now().UTC() } diff --git a/docs/content/features/face-recognition.md b/docs/content/features/face-recognition.md index 8b037eec7..33fa29fc4 100644 --- a/docs/content/features/face-recognition.md +++ b/docs/content/features/face-recognition.md @@ -73,9 +73,9 @@ Detect faces and analyze demographics (buffalo entries populate age / gender; YuNet + SFace returns regions only): ```bash -curl -sX POST http://localhost:8080/v1/face/detect \ +curl -sX POST http://localhost:8080/v1/detection \ -H "Content-Type: application/json" \ - -d '{"model": "face-detect-buffalo-l", "img": "https://example.com/group.jpg"}' + -d '{"model": "face-detect-buffalo-l", "image": "https://example.com/group.jpg"}' curl -sX POST http://localhost:8080/v1/face/analyze \ -H "Content-Type: application/json" \ @@ -141,6 +141,39 @@ Response: } ``` +## Restore enrollments after a restart + +The default identity store is in memory. Clients can keep an enrollment record +and replay it with `POST /v1/face/register` after a restart. Extract the embedding +once with `/v1/face/embed`, then save the exact returned vector, model, name, +labels, and enrollment timestamp. Submit `embedding` instead of `img`: + +```json +{ + "model": "insightface-opencv", + "name": "Alice", + "embedding": [0.12, -0.04, 0.31], + "registered_at": "2026-09-07T12:00:00Z", + "labels": {"client_id": "alice"} +} +``` + +The vector above is abbreviated; send the complete embedding from the same +recognizer model. Provide exactly one of `img` or `embedding`. Vectors must be +finite and nonzero. `registered_at` is optional and defaults to the current time; +replay the original timestamp to preserve it. + +The store upserts by exact vector. Registration now derives a stable ID from +that vector and the store namespace, so retries and replay after a restart +return the same ID without adding duplicate entries. Replaying updates the name +and labels. Images can produce slightly different embeddings across runs; keep +the original vector instead of embedding the photo again on each retry. + +This does not make the server store persistent. Clients must retain and restore +the records themselves. With independent stores behind a load balancer, replay +into each store or use a shared store. Do not mix different recognizer models in +one store. IDs from older versions change on their first registration replay. + ## 1:N identification workflow (register → identify → forget) This is the primary "face recognition" flow. Under the hood it uses