diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index cb6c96553..811fc1115 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=c41ffefb44c16345c0ac57a0030f4cd3b5fbc4b8 +AUDIO_CPP_VERSION?=f6277c1695a83cf388a8282c1c1a8757cf626f18 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/cpp/ds4/CMakeLists.txt b/backend/cpp/ds4/CMakeLists.txt index 0535a8a44..5783db942 100644 --- a/backend/cpp/ds4/CMakeLists.txt +++ b/backend/cpp/ds4/CMakeLists.txt @@ -84,9 +84,10 @@ elseif(DS4_GPU STREQUAL "cpu") set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o") endif() -# Upstream splits distributed inference, tensor-parallel transport, the SSD -# expert cache, and layer placement into GPU-agnostic translation units. Link -# them regardless of DS4_GPU. +# Upstream splits image preprocessing, distributed inference, tensor-parallel +# transport, the SSD expert cache, and layer placement into GPU-agnostic +# translation units. Link them regardless of DS4_GPU. +list(APPEND DS4_OBJS "${DS4_DIR}/ds4_image.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_distributed.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_tp.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_ssd.o") diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index 807a01a09..455b554d8 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 +# Upstream pin lives below as DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5 # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 +DS4_VERSION?=9ab705347c1775e7599ede7eb81a6255ec7dccb5 DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -79,21 +79,22 @@ else endif endif -# Upstream splits distributed inference, tensor-parallel transport, the SSD -# expert cache, and layer placement into GPU-agnostic translation units. They -# are shared by every GPU mode, so append them unconditionally below. +# Upstream splits image preprocessing, distributed inference, tensor-parallel +# transport, the SSD expert cache, and layer placement into GPU-agnostic +# translation units. They are shared by every GPU mode, so append them +# unconditionally below. ifeq ($(BUILD_TYPE),cublas) CMAKE_ARGS += -DDS4_GPU=cuda - DS4_OBJ_TARGET := ds4.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \ + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \ cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \ cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o else ifeq ($(UNAME_S),Darwin) CMAKE_ARGS += -DDS4_GPU=metal - DS4_OBJ_TARGET := ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else # CPU reference path (Linux only - macOS CPU path is broken by VM bug per ds4 README). CMAKE_ARGS += -DDS4_GPU=cpu - DS4_OBJ_TARGET := ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + DS4_OBJ_TARGET := ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o endif ifneq ($(NATIVE),true) @@ -120,9 +121,9 @@ ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) +$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET) else ifeq ($(UNAME_S),Darwin) - +$(MAKE) -C ds4 ds4.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + +$(MAKE) -C ds4 ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o else - +$(MAKE) -C ds4 ds4_cpu.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + +$(MAKE) -C ds4 ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o endif grpc-server: ds4/ds4.o diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index bb60a6624..3f158f49a 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=74a7c897f049c17e7080423aa2111776eff6ebbf +LLAMA_VERSION?=465e49b9cea78a68b9c244ffb48d0ee24a82873d LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 1f8e70402..e39a1d09c 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=6b3edaaf32cc19e5bb2d819c788bd557eddc8eba +STABLEDIFFUSION_GGML_VERSION?=d8fb10c02977c8ca999f3fb4e02df9ecf10f7ba6 CMAKE_ARGS+=-DGGML_MAX_NAME=128 diff --git a/backend/go/stablediffusion-ggml/cpp/gosd.cpp b/backend/go/stablediffusion-ggml/cpp/gosd.cpp index 12cc4a83e..7722e8d06 100644 --- a/backend/go/stablediffusion-ggml/cpp/gosd.cpp +++ b/backend/go/stablediffusion-ggml/cpp/gosd.cpp @@ -401,7 +401,6 @@ int load_model(const char *model, char *model_path, char* options[], int threads const char *params_backend_arg = ""; const char *rpc_servers_arg = ""; const char *max_vram_arg = ""; - bool stream_layers = false; int n_threads = threads; enum sd_type_t wtype = SD_TYPE_COUNT; @@ -510,7 +509,10 @@ int load_model(const char *model, char *model_path, char* options[], int threads if (!strcmp(optname, "params_backend")) params_backend_arg = strdup(optval); if (!strcmp(optname, "rpc_servers")) rpc_servers_arg = strdup(optval); if (!strcmp(optname, "max_vram")) max_vram_arg = strdup(optval); - if (!strcmp(optname, "stream_layers")) stream_layers = (strcmp(optval, "true") == 0 || strcmp(optval, "1") == 0); + if (!strcmp(optname, "stream_layers")) { + // Retained as a no-op for existing configurations. Upstream now + // selects segmented weight streaming automatically. + } // vae_decode_only is still accepted for backwards compatibility with // existing gallery configs, but upstream dropped the option (the model @@ -650,11 +652,9 @@ int load_model(const char *model, char *model_path, char* options[], int threads ctx_params.rpc_servers = env_rpc_servers; } } - // max_vram: GiB budget or per-backend spec for graph-cut segmented param - // offload ("0" = disabled, "-1" = auto). stream_layers only has effect when - // max_vram is set. + // max_vram is an optional GiB budget or per-backend spec for automatic + // graph-cut execution. A zero value uses the live free-VRAM budget. if (strlen(max_vram_arg) > 0) ctx_params.max_vram = max_vram_arg; - ctx_params.stream_layers = stream_layers; ctx_params.diffusion_flash_attn = diffusion_flash_attn; ctx_params.tae_preview_only = tae_preview_only; ctx_params.diffusion_conv_direct = diffusion_conv_direct; @@ -1438,4 +1438,3 @@ int unload() { free_sd_ctx(sd_c); return 0; } - diff --git a/core/http/endpoints/localai/face_register.go b/core/http/endpoints/localai/face_register.go index fbeb29e0c..9cd40b456 100644 --- a/core/http/endpoints/localai/face_register.go +++ b/core/http/endpoints/localai/face_register.go @@ -1,6 +1,7 @@ package localai import ( + "errors" "net/http" "github.com/labstack/echo/v4" @@ -33,22 +34,31 @@ func FaceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a return echo.NewHTTPError(http.StatusBadRequest, "name is required") } - img, err := decodeImageInput(input.Img) - if err != nil { - return err + if (input.Img == "") == (len(input.Embedding) == 0) { + return echo.NewHTTPError(http.StatusBadRequest, "provide exactly one of img or embedding") } - - xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) - embedding, err := backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) - if err != nil { - return mapBackendError(err) + embedding := input.Embedding + if len(embedding) == 0 { + img, err := decodeImageInput(input.Img) + if err != nil { + return err + } + xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) + embedding, err = backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) + if err != nil { + return mapBackendError(err) + } } stored, err := registry.Register(c.Request().Context(), embedding, facerecognition.Metadata{ - Name: input.Name, - Labels: input.Labels, + Name: input.Name, + RegisteredAt: input.RegisteredAt, + Labels: input.Labels, }) if err != nil { + if errors.Is(err, facerecognition.ErrInvalidEmbedding) || errors.Is(err, facerecognition.ErrDimensionMismatch) { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } return err } return c.JSON(http.StatusOK, schema.FaceRegisterResponse{ diff --git a/core/http/endpoints/localai/face_register_test.go b/core/http/endpoints/localai/face_register_test.go new file mode 100644 index 000000000..16970345a --- /dev/null +++ b/core/http/endpoints/localai/face_register_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT + +package localai_test + +import ( + "context" + "net/http" + "net/http/httptest" + "time" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/facerecognition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type registrationRecorder struct { + facerecognition.Registry + vector []float32 + meta facerecognition.Metadata + err error +} + +func (r *registrationRecorder) Register(_ context.Context, v []float32, m facerecognition.Metadata) (facerecognition.Metadata, error) { + r.vector = v + r.meta = m + m.ID = "saved-id" + return m, r.err +} + +var _ = Describe("Face registration replay", func() { + var reg *registrationRecorder + call := func(in schema.FaceRegisterRequest) (*httptest.ResponseRecorder, error) { + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/face/register", nil), rec) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &in) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{}) + // No model loader: replay must not call the embedding backend. + err := FaceRegisterEndpoint(nil, nil, nil, reg)(c) + return rec, err + } + BeforeEach(func() { reg = ®istrationRecorder{} }) + It("accepts the saved vector and timestamp without running inference", func() { + at := time.Now().UTC() + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{1, 0}, RegisteredAt: at, Labels: map[string]string{"client_id": "alice"}} + in.Model = "faces" + rec, err := call(in) + Expect(err).NotTo(HaveOccurred()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(reg.vector).To(Equal(in.Embedding)) + Expect(reg.meta.RegisteredAt).To(Equal(at)) + Expect(reg.meta.Labels).To(Equal(in.Labels)) + Expect(rec.Body.String()).To(ContainSubstring("saved-id")) + }) + It("rejects ambiguous and missing inputs before inference", func() { + for _, in := range []schema.FaceRegisterRequest{ + {Name: "Alice"}, + {Name: "Alice", Img: "image", Embedding: []float32{1, 0}}, + } { + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + Expect(reg.vector).To(BeNil()) + } + }) + It("reports invalid vectors as a client error", func() { + reg.err = facerecognition.ErrInvalidEmbedding + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{0, 0}} + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + }) +}) diff --git a/core/schema/localai.go b/core/schema/localai.go index e160badad..a40f90c8d 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -353,10 +353,12 @@ type FaceEmbedResponse struct { // FaceRegisterRequest enrolls a face into the 1:N recognition store. type FaceRegisterRequest struct { BasicModelRequest - Img string `json:"img"` - Name string `json:"name"` - Labels map[string]string `json:"labels,omitempty"` - Store string `json:"store,omitempty"` // vector store model; empty = local-store default + RegisteredAt time.Time `json:"registered_at,omitempty"` // original enrollment time when replaying a saved embedding + Embedding []float32 `json:"embedding,omitempty"` + Img string `json:"img"` + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` + Store string `json:"store,omitempty"` // vector store model; empty = local-store default } type FaceRegisterResponse struct { diff --git a/core/services/facerecognition/registry.go b/core/services/facerecognition/registry.go index adc9d9200..ae781dbb2 100644 --- a/core/services/facerecognition/registry.go +++ b/core/services/facerecognition/registry.go @@ -56,5 +56,6 @@ type Match struct { var ( ErrNotFound = errors.New("facerecognition: id not found") ErrEmptyEmbedding = errors.New("facerecognition: embedding is empty") + ErrInvalidEmbedding = errors.New("facerecognition: embedding must be finite and nonzero") ErrDimensionMismatch = errors.New("facerecognition: embedding dimension mismatch") ) diff --git a/core/services/facerecognition/replay_test.go b/core/services/facerecognition/replay_test.go new file mode 100644 index 000000000..21c965905 --- /dev/null +++ b/core/services/facerecognition/replay_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT + +package facerecognition + +import ( + "context" + "encoding/json" + "math" + "sync" + "testing" + "time" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +func TestEnrollmentReplay(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Enrollment replay") } + +type replayStore struct { + grpc.Backend + mu sync.Mutex + entries map[string][]byte +} + +func (s *replayStore) StoresSet(_ context.Context, in *pb.StoresSetOptions, _ ...ggrpc.CallOption) (*pb.Result, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i, k := range in.Keys { + b, _ := json.Marshal(k.Floats) + s.entries[string(b)] = append([]byte(nil), in.Values[i].Bytes...) + } + return &pb.Result{Success: true}, nil +} + +var _ = Describe("Enrollment replay", func() { + It("keeps the identity across registry instances and a cleared store", func(ctx SpecContext) { + storage := &replayStore{entries: map[string][]byte{}} + newRegistry := func() Registry { + return NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + } + vector := []float32{1, 0, 0, 0} + meta := Metadata{Name: "Alice", RegisteredAt: time.Now().UTC()} + first, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + again, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + storage.entries = map[string][]byte{} + restored, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(restored).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + }) + It("rejects zero and non-finite embeddings before writing", func(ctx SpecContext) { + for _, v := range [][]float32{{0, 0}, {float32(math.NaN()), 1}, {float32(math.Inf(1)), 1}} { + storage := &replayStore{entries: map[string][]byte{}} + reg := NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + _, err := reg.Register(ctx, v, Metadata{Name: "Alice"}) + Expect(err).To(HaveOccurred()) + Expect(storage.entries).To(BeEmpty()) + } + }) +}) diff --git a/core/services/facerecognition/store_registry.go b/core/services/facerecognition/store_registry.go index d4fd0d971..abf7ee1ac 100644 --- a/core/services/facerecognition/store_registry.go +++ b/core/services/facerecognition/store_registry.go @@ -2,8 +2,10 @@ package facerecognition import ( "context" + "encoding/binary" "encoding/json" "fmt" + "math" "sort" "sync" "time" @@ -57,13 +59,32 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta if r.dim != 0 && len(embedding) != r.dim { return Metadata{}, fmt.Errorf("%w: expected %d, got %d", ErrDimensionMismatch, r.dim, len(embedding)) } + var norm float64 + key := make([]byte, 4*len(embedding)) + for i, value := range embedding { + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return Metadata{}, ErrInvalidEmbedding + } + norm += float64(value) * float64(value) + // The store treats negative and positive zero as the same key. + if value == 0 { + value = 0 + } + binary.LittleEndian.PutUint32(key[i*4:], math.Float32bits(value)) + } + if norm == 0 { + return Metadata{}, ErrInvalidEmbedding + } backend, err := r.resolve(ctx, r.storeName) if err != nil { return Metadata{}, fmt.Errorf("facerecognition: resolve store: %w", err) } - meta.ID = uuid.NewString() + // The vector store upserts by the exact embedding. Derive the ID from the + // same key so replaying a saved vector preserves identity across replicas + // and after the in-memory store restarts. + meta.ID = uuid.NewSHA1(uuid.NewSHA1(uuid.NameSpaceOID, []byte(r.storeName)), key).String() if meta.RegisteredAt.IsZero() { meta.RegisteredAt = time.Now().UTC() } diff --git a/core/services/nodes/file_staging_client.go b/core/services/nodes/file_staging_client.go index 45dbe3a57..b121d5c68 100644 --- a/core/services/nodes/file_staging_client.go +++ b/core/services/nodes/file_staging_client.go @@ -16,6 +16,7 @@ import ( pb "github.com/mudler/LocalAI/pkg/grpc/proto" "github.com/mudler/xlog" ggrpc "google.golang.org/grpc" + "google.golang.org/protobuf/proto" ) // FileStagingClient wraps a grpc.Backend to transparently handle file transfer @@ -346,6 +347,19 @@ func (f *FileStagingClient) SoundGeneration(ctx context.Context, in *pb.SoundGen return result, nil } +func (f *FileStagingClient) SoundDetection(ctx context.Context, in *pb.SoundDetectionRequest, opts ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) { + if in.Src != "" && isFilePath(in.Src) { + backendPath, _, err := f.stageInputFile(ctx, requestID(), in.Src, "inputs") + if err != nil { + return nil, fmt.Errorf("staging audio for sound detection: %w", err) + } + // Keep the frontend path available if the caller retries on another node. + in = proto.Clone(in).(*pb.SoundDetectionRequest) + in.Src = backendPath + } + return f.Backend.SoundDetection(ctx, in, opts...) +} + func (f *FileStagingClient) AudioTranscription(ctx context.Context, in *pb.TranscriptRequest, opts ...ggrpc.CallOption) (*pb.TranscriptResult, error) { reqID := requestID() diff --git a/core/services/nodes/file_staging_sound_detection_test.go b/core/services/nodes/file_staging_sound_detection_test.go new file mode 100644 index 000000000..e66b18f5f --- /dev/null +++ b/core/services/nodes/file_staging_sound_detection_test.go @@ -0,0 +1,94 @@ +package nodes + +import ( + "context" + "errors" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +type soundStagingBackend struct { + grpc.Backend + request *pb.SoundDetectionRequest + err error +} + +func (b *soundStagingBackend) SoundDetection(_ context.Context, in *pb.SoundDetectionRequest, _ ...ggrpc.CallOption) (*pb.SoundDetectionResponse, error) { + b.request = in + return &pb.SoundDetectionResponse{}, b.err +} + +type soundStagingFailure struct{ FileStager } + +func (s *soundStagingFailure) EnsureRemote(context.Context, string, string, string) (string, error) { + return "", errors.New("upload failed") +} + +type soundRouteFactory struct{ client grpc.Backend } + +func (f *soundRouteFactory) NewClient(string, bool) grpc.Backend { return f.client } + +var _ = Describe("FileStagingClient sound detection", func() { + It("stages sound audio through the client returned by SmartRouter.Route", func(ctx SpecContext) { + node := &BackendNode{ID: "worker-1", Name: "worker", Address: "10.0.0.1:50051"} + reg := &fakeModelRouter{ + findAndLockNode: node, + findAndLockNM: &NodeModel{NodeID: node.ID, ModelName: "ced", Address: "10.0.0.1:9001"}, + } + backend := &soundStagingBackend{Backend: &stubBackend{healthResult: true}} + stager := &fakeFileStager{} + router := NewSmartRouter(reg, SmartRouterOptions{ + ClientFactory: &soundRouteFactory{client: backend}, + FileStager: stager, + Unloader: &fakeUnloader{}, + }) + result, err := router.Route(ctx, "ced", "ced.gguf", "ced", "", nil, false) + Expect(err).NotTo(HaveOccurred()) + Expect(result).NotTo(BeNil()) + defer result.Release() + request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf"} + _, err = result.Client.SoundDetection(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(stager.ensureCalls).To(HaveLen(1)) + Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src)) + Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key)) + Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav")) + }) + + It("stages audio on the worker without changing the caller's request", func(ctx SpecContext) { + backend := &soundStagingBackend{} + stager := &fakeFileStager{} + client := NewFileStagingClient(backend, stager, "worker-1") + request := &pb.SoundDetectionRequest{Src: "/tmp/realtime-sound-window-test.wav", ModelIdentity: "ced.gguf", TopK: 5, Threshold: 0.25} + _, err := client.SoundDetection(ctx, request) + Expect(err).NotTo(HaveOccurred()) + Expect(stager.ensureCalls).To(HaveLen(1)) + Expect(stager.ensureCalls[0].nodeID).To(Equal("worker-1")) + Expect(stager.ensureCalls[0].localPath).To(Equal(request.Src)) + Expect(backend.request.Src).To(Equal("/remote/" + stager.ensureCalls[0].key)) + Expect(backend.request.ModelIdentity).To(Equal(request.ModelIdentity)) + Expect(backend.request.TopK).To(Equal(request.TopK)) + Expect(backend.request.Threshold).To(Equal(request.Threshold)) + Expect(request.Src).To(Equal("/tmp/realtime-sound-window-test.wav")) + }) + It("does not call the backend when staging fails", func(ctx SpecContext) { + backend := &soundStagingBackend{} + client := NewFileStagingClient(backend, &soundStagingFailure{}, "worker-1") + _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{Src: "/tmp/clip.wav"}) + Expect(err).To(MatchError(ContainSubstring("upload failed"))) + Expect(backend.request).To(BeNil()) + }) + It("passes through requests without a file and preserves backend errors", func(ctx SpecContext) { + failure := errors.New("classifier failed") + backend := &soundStagingBackend{err: failure} + stager := &fakeFileStager{} + client := NewFileStagingClient(backend, stager, "worker-1") + _, err := client.SoundDetection(ctx, &pb.SoundDetectionRequest{}) + Expect(err).To(MatchError(failure)) + Expect(stager.ensureCalls).To(BeEmpty()) + }) +}) diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md index 045074340..09759ab72 100644 --- a/docs/content/features/audio-classification.md +++ b/docs/content/features/audio-classification.md @@ -11,6 +11,10 @@ LocalAI exposes this through the `/v1/audio/classification` endpoint, modelled a Because classification is exposed as a regular OpenAI-style endpoint, any HTTP client works - there is no Python dependency on the consumer side. +In distributed mode, LocalAI stages uploaded audio and realtime sound-detection +windows on the selected worker before classification. The API server and worker +do not need a shared temporary directory. + ## Endpoint ``` diff --git a/docs/content/features/face-recognition.md b/docs/content/features/face-recognition.md index 8b037eec7..33fa29fc4 100644 --- a/docs/content/features/face-recognition.md +++ b/docs/content/features/face-recognition.md @@ -73,9 +73,9 @@ Detect faces and analyze demographics (buffalo entries populate age / gender; YuNet + SFace returns regions only): ```bash -curl -sX POST http://localhost:8080/v1/face/detect \ +curl -sX POST http://localhost:8080/v1/detection \ -H "Content-Type: application/json" \ - -d '{"model": "face-detect-buffalo-l", "img": "https://example.com/group.jpg"}' + -d '{"model": "face-detect-buffalo-l", "image": "https://example.com/group.jpg"}' curl -sX POST http://localhost:8080/v1/face/analyze \ -H "Content-Type: application/json" \ @@ -141,6 +141,39 @@ Response: } ``` +## Restore enrollments after a restart + +The default identity store is in memory. Clients can keep an enrollment record +and replay it with `POST /v1/face/register` after a restart. Extract the embedding +once with `/v1/face/embed`, then save the exact returned vector, model, name, +labels, and enrollment timestamp. Submit `embedding` instead of `img`: + +```json +{ + "model": "insightface-opencv", + "name": "Alice", + "embedding": [0.12, -0.04, 0.31], + "registered_at": "2026-09-07T12:00:00Z", + "labels": {"client_id": "alice"} +} +``` + +The vector above is abbreviated; send the complete embedding from the same +recognizer model. Provide exactly one of `img` or `embedding`. Vectors must be +finite and nonzero. `registered_at` is optional and defaults to the current time; +replay the original timestamp to preserve it. + +The store upserts by exact vector. Registration now derives a stable ID from +that vector and the store namespace, so retries and replay after a restart +return the same ID without adding duplicate entries. Replaying updates the name +and labels. Images can produce slightly different embeddings across runs; keep +the original vector instead of embedding the photo again on each retry. + +This does not make the server store persistent. Clients must retain and restore +the records themselves. With independent stores behind a load balancer, replay +into each store or use a shared store. Do not mix different recognizer models in +one store. IDs from older versions change on their first registration replay. + ## 1:N identification workflow (register → identify → forget) This is the primary "face recognition" flow. Under the hood it uses diff --git a/docs/content/features/image-generation.md b/docs/content/features/image-generation.md index d94c0f9b4..1df6058b4 100644 --- a/docs/content/features/image-generation.md +++ b/docs/content/features/image-generation.md @@ -79,8 +79,8 @@ When a model does not fit entirely in VRAM, the following `options:` control whe |--------|---------|-------------| | `backend` | `backend:clip=cpu,vae=cuda0,diffusion=vulkan0` | Runtime (compute) backend assignment per component. Use `cpu` to place a component's compute on the CPU. Component keys include `te` (text encoder / CLIP), `vae`, `diffusion`, `controlnet`. | | `params_backend` | `params_backend:diffusion=disk,clip=cpu` | Where parameters (weights) are stored. Supports `cpu`, `disk` (mmap weights from disk to save RAM/VRAM), or per-component specs. | -| `max_vram` | `max_vram:8` or `max_vram:-1` | VRAM budget (in GiB) for graph-cut segmented parameter offload. `0` disables it, `-1` auto-selects (free VRAM minus ~1 GiB). Also accepts per-backend budgets. | -| `stream_layers` | `stream_layers:true` | Enable residency + prefetch streaming on top of `max_vram` (no effect unless `max_vram` is set). | +| `max_vram` | `max_vram:8` or `max_vram:-1` | Optional per-device VRAM budget (in GiB) for managed weights and automatic graph-cut execution. `0` uses live free VRAM without an explicit cap; a negative value reserves that many GiB of free VRAM. Also accepts per-backend budgets. | +| `stream_layers` | `stream_layers:true` | Deprecated compatibility option. Segmented weight streaming is now selected automatically, so this value is ignored. | | `rpc_servers` | `rpc_servers:localhost:50052,192.168.1.3:50052` | Comma-separated list of `host:port` RPC servers to offload compute to. | | `pulid_weights_path` | `pulid_weights_path:pulid.safetensors` | Path to PuLID-Flux weights for identity injection. | diff --git a/docs/content/integrations.md b/docs/content/integrations.md index 0bcf69e99..8d8bd4104 100644 --- a/docs/content/integrations.md +++ b/docs/content/integrations.md @@ -18,6 +18,15 @@ Feel free to open up a Pull request (by clicking at the "Edit page" below) to ge - [Helm chart](https://github.com/go-skynet/helm-charts) - Deploy LocalAI on Kubernetes - [GitHub Actions](https://github.com/marketplace/actions/start-localai) - Use LocalAI in CI/CD workflows +### Distribution Packages + +Community-maintained packages of LocalAI. Package versions and +availability may lag upstream releases. + +- [Homebrew](https://formulae.brew.sh/formula/localai) - `brew install localai` +- [ALT Sisyphus](https://packages.altlinux.org/en/sisyphus/srpms/localai/) - `localai` package +- [Gentoo overlay](https://git.ipnmod.org/packages/local-ai-overlay) - `sci-ml/local-ai` and per-backend packages, overlay `local-ai` in Gentoo's repository index + ### Web UIs - [localai-admin](https://github.com/Jirubizu/localai-admin)