From 88d19567b8a9dbaaff8f8e210eb01868d288354a Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 7 Sep 2026 17:40:27 +0200 Subject: [PATCH] feat(faces): replay saved face enrollments (#11908) Accept original embeddings and timestamps so clients can restore faces when the in-memory store restarts. Derive stable IDs from exact vectors to make registration retries preserve identity without duplicate entries. Assisted-by: Codex:GPT-6 golangci-lint Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- core/http/endpoints/localai/face_register.go | 30 ++++--- .../endpoints/localai/face_register_test.go | 80 +++++++++++++++++++ core/schema/localai.go | 10 ++- core/services/facerecognition/registry.go | 1 + core/services/facerecognition/replay_test.go | 67 ++++++++++++++++ .../facerecognition/store_registry.go | 23 +++++- docs/content/features/face-recognition.md | 37 ++++++++- 7 files changed, 231 insertions(+), 17 deletions(-) create mode 100644 core/http/endpoints/localai/face_register_test.go create mode 100644 core/services/facerecognition/replay_test.go diff --git a/core/http/endpoints/localai/face_register.go b/core/http/endpoints/localai/face_register.go index fbeb29e0c..9cd40b456 100644 --- a/core/http/endpoints/localai/face_register.go +++ b/core/http/endpoints/localai/face_register.go @@ -1,6 +1,7 @@ package localai import ( + "errors" "net/http" "github.com/labstack/echo/v4" @@ -33,22 +34,31 @@ func FaceRegisterEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, a return echo.NewHTTPError(http.StatusBadRequest, "name is required") } - img, err := decodeImageInput(input.Img) - if err != nil { - return err + if (input.Img == "") == (len(input.Embedding) == 0) { + return echo.NewHTTPError(http.StatusBadRequest, "provide exactly one of img or embedding") } - - xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) - embedding, err := backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) - if err != nil { - return mapBackendError(err) + embedding := input.Embedding + if len(embedding) == 0 { + img, err := decodeImageInput(input.Img) + if err != nil { + return err + } + xlog.Debug("FaceRegister", "model", cfg.Name, "name", input.Name) + embedding, err = backend.FaceEmbed(c.Request().Context(), img, ml, appConfig, *cfg) + if err != nil { + return mapBackendError(err) + } } stored, err := registry.Register(c.Request().Context(), embedding, facerecognition.Metadata{ - Name: input.Name, - Labels: input.Labels, + Name: input.Name, + RegisteredAt: input.RegisteredAt, + Labels: input.Labels, }) if err != nil { + if errors.Is(err, facerecognition.ErrInvalidEmbedding) || errors.Is(err, facerecognition.ErrDimensionMismatch) { + return echo.NewHTTPError(http.StatusBadRequest, err.Error()) + } return err } return c.JSON(http.StatusOK, schema.FaceRegisterResponse{ diff --git a/core/http/endpoints/localai/face_register_test.go b/core/http/endpoints/localai/face_register_test.go new file mode 100644 index 000000000..16970345a --- /dev/null +++ b/core/http/endpoints/localai/face_register_test.go @@ -0,0 +1,80 @@ +// SPDX-License-Identifier: MIT + +package localai_test + +import ( + "context" + "net/http" + "net/http/httptest" + "time" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/http/endpoints/localai" + "github.com/mudler/LocalAI/core/http/middleware" + "github.com/mudler/LocalAI/core/schema" + "github.com/mudler/LocalAI/core/services/facerecognition" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +type registrationRecorder struct { + facerecognition.Registry + vector []float32 + meta facerecognition.Metadata + err error +} + +func (r *registrationRecorder) Register(_ context.Context, v []float32, m facerecognition.Metadata) (facerecognition.Metadata, error) { + r.vector = v + r.meta = m + m.ID = "saved-id" + return m, r.err +} + +var _ = Describe("Face registration replay", func() { + var reg *registrationRecorder + call := func(in schema.FaceRegisterRequest) (*httptest.ResponseRecorder, error) { + e := echo.New() + rec := httptest.NewRecorder() + c := e.NewContext(httptest.NewRequest(http.MethodPost, "/v1/face/register", nil), rec) + c.Set(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST, &in) + c.Set(middleware.CONTEXT_LOCALS_KEY_MODEL_CONFIG, &config.ModelConfig{}) + // No model loader: replay must not call the embedding backend. + err := FaceRegisterEndpoint(nil, nil, nil, reg)(c) + return rec, err + } + BeforeEach(func() { reg = ®istrationRecorder{} }) + It("accepts the saved vector and timestamp without running inference", func() { + at := time.Now().UTC() + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{1, 0}, RegisteredAt: at, Labels: map[string]string{"client_id": "alice"}} + in.Model = "faces" + rec, err := call(in) + Expect(err).NotTo(HaveOccurred()) + Expect(rec.Code).To(Equal(http.StatusOK)) + Expect(reg.vector).To(Equal(in.Embedding)) + Expect(reg.meta.RegisteredAt).To(Equal(at)) + Expect(reg.meta.Labels).To(Equal(in.Labels)) + Expect(rec.Body.String()).To(ContainSubstring("saved-id")) + }) + It("rejects ambiguous and missing inputs before inference", func() { + for _, in := range []schema.FaceRegisterRequest{ + {Name: "Alice"}, + {Name: "Alice", Img: "image", Embedding: []float32{1, 0}}, + } { + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + Expect(reg.vector).To(BeNil()) + } + }) + It("reports invalid vectors as a client error", func() { + reg.err = facerecognition.ErrInvalidEmbedding + in := schema.FaceRegisterRequest{Name: "Alice", Embedding: []float32{0, 0}} + in.Model = "faces" + _, err := call(in) + Expect(err).To(HaveOccurred()) + Expect(err.(*echo.HTTPError).Code).To(Equal(http.StatusBadRequest)) + }) +}) diff --git a/core/schema/localai.go b/core/schema/localai.go index e160badad..a40f90c8d 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -353,10 +353,12 @@ type FaceEmbedResponse struct { // FaceRegisterRequest enrolls a face into the 1:N recognition store. type FaceRegisterRequest struct { BasicModelRequest - Img string `json:"img"` - Name string `json:"name"` - Labels map[string]string `json:"labels,omitempty"` - Store string `json:"store,omitempty"` // vector store model; empty = local-store default + RegisteredAt time.Time `json:"registered_at,omitempty"` // original enrollment time when replaying a saved embedding + Embedding []float32 `json:"embedding,omitempty"` + Img string `json:"img"` + Name string `json:"name"` + Labels map[string]string `json:"labels,omitempty"` + Store string `json:"store,omitempty"` // vector store model; empty = local-store default } type FaceRegisterResponse struct { diff --git a/core/services/facerecognition/registry.go b/core/services/facerecognition/registry.go index adc9d9200..ae781dbb2 100644 --- a/core/services/facerecognition/registry.go +++ b/core/services/facerecognition/registry.go @@ -56,5 +56,6 @@ type Match struct { var ( ErrNotFound = errors.New("facerecognition: id not found") ErrEmptyEmbedding = errors.New("facerecognition: embedding is empty") + ErrInvalidEmbedding = errors.New("facerecognition: embedding must be finite and nonzero") ErrDimensionMismatch = errors.New("facerecognition: embedding dimension mismatch") ) diff --git a/core/services/facerecognition/replay_test.go b/core/services/facerecognition/replay_test.go new file mode 100644 index 000000000..21c965905 --- /dev/null +++ b/core/services/facerecognition/replay_test.go @@ -0,0 +1,67 @@ +// SPDX-License-Identifier: MIT + +package facerecognition + +import ( + "context" + "encoding/json" + "math" + "sync" + "testing" + "time" + + grpc "github.com/mudler/LocalAI/pkg/grpc" + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + ggrpc "google.golang.org/grpc" +) + +func TestEnrollmentReplay(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Enrollment replay") } + +type replayStore struct { + grpc.Backend + mu sync.Mutex + entries map[string][]byte +} + +func (s *replayStore) StoresSet(_ context.Context, in *pb.StoresSetOptions, _ ...ggrpc.CallOption) (*pb.Result, error) { + s.mu.Lock() + defer s.mu.Unlock() + for i, k := range in.Keys { + b, _ := json.Marshal(k.Floats) + s.entries[string(b)] = append([]byte(nil), in.Values[i].Bytes...) + } + return &pb.Result{Success: true}, nil +} + +var _ = Describe("Enrollment replay", func() { + It("keeps the identity across registry instances and a cleared store", func(ctx SpecContext) { + storage := &replayStore{entries: map[string][]byte{}} + newRegistry := func() Registry { + return NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + } + vector := []float32{1, 0, 0, 0} + meta := Metadata{Name: "Alice", RegisteredAt: time.Now().UTC()} + first, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + again, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(again).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + storage.entries = map[string][]byte{} + restored, err := newRegistry().Register(ctx, vector, meta) + Expect(err).NotTo(HaveOccurred()) + Expect(restored).To(Equal(first)) + Expect(storage.entries).To(HaveLen(1)) + }) + It("rejects zero and non-finite embeddings before writing", func(ctx SpecContext) { + for _, v := range [][]float32{{0, 0}, {float32(math.NaN()), 1}, {float32(math.Inf(1)), 1}} { + storage := &replayStore{entries: map[string][]byte{}} + reg := NewStoreRegistry(func(context.Context, string) (grpc.Backend, error) { return storage, nil }, "faces", 0) + _, err := reg.Register(ctx, v, Metadata{Name: "Alice"}) + Expect(err).To(HaveOccurred()) + Expect(storage.entries).To(BeEmpty()) + } + }) +}) diff --git a/core/services/facerecognition/store_registry.go b/core/services/facerecognition/store_registry.go index d4fd0d971..abf7ee1ac 100644 --- a/core/services/facerecognition/store_registry.go +++ b/core/services/facerecognition/store_registry.go @@ -2,8 +2,10 @@ package facerecognition import ( "context" + "encoding/binary" "encoding/json" "fmt" + "math" "sort" "sync" "time" @@ -57,13 +59,32 @@ func (r *storeRegistry) Register(ctx context.Context, embedding []float32, meta if r.dim != 0 && len(embedding) != r.dim { return Metadata{}, fmt.Errorf("%w: expected %d, got %d", ErrDimensionMismatch, r.dim, len(embedding)) } + var norm float64 + key := make([]byte, 4*len(embedding)) + for i, value := range embedding { + if math.IsNaN(float64(value)) || math.IsInf(float64(value), 0) { + return Metadata{}, ErrInvalidEmbedding + } + norm += float64(value) * float64(value) + // The store treats negative and positive zero as the same key. + if value == 0 { + value = 0 + } + binary.LittleEndian.PutUint32(key[i*4:], math.Float32bits(value)) + } + if norm == 0 { + return Metadata{}, ErrInvalidEmbedding + } backend, err := r.resolve(ctx, r.storeName) if err != nil { return Metadata{}, fmt.Errorf("facerecognition: resolve store: %w", err) } - meta.ID = uuid.NewString() + // The vector store upserts by the exact embedding. Derive the ID from the + // same key so replaying a saved vector preserves identity across replicas + // and after the in-memory store restarts. + meta.ID = uuid.NewSHA1(uuid.NewSHA1(uuid.NameSpaceOID, []byte(r.storeName)), key).String() if meta.RegisteredAt.IsZero() { meta.RegisteredAt = time.Now().UTC() } diff --git a/docs/content/features/face-recognition.md b/docs/content/features/face-recognition.md index 8b037eec7..33fa29fc4 100644 --- a/docs/content/features/face-recognition.md +++ b/docs/content/features/face-recognition.md @@ -73,9 +73,9 @@ Detect faces and analyze demographics (buffalo entries populate age / gender; YuNet + SFace returns regions only): ```bash -curl -sX POST http://localhost:8080/v1/face/detect \ +curl -sX POST http://localhost:8080/v1/detection \ -H "Content-Type: application/json" \ - -d '{"model": "face-detect-buffalo-l", "img": "https://example.com/group.jpg"}' + -d '{"model": "face-detect-buffalo-l", "image": "https://example.com/group.jpg"}' curl -sX POST http://localhost:8080/v1/face/analyze \ -H "Content-Type: application/json" \ @@ -141,6 +141,39 @@ Response: } ``` +## Restore enrollments after a restart + +The default identity store is in memory. Clients can keep an enrollment record +and replay it with `POST /v1/face/register` after a restart. Extract the embedding +once with `/v1/face/embed`, then save the exact returned vector, model, name, +labels, and enrollment timestamp. Submit `embedding` instead of `img`: + +```json +{ + "model": "insightface-opencv", + "name": "Alice", + "embedding": [0.12, -0.04, 0.31], + "registered_at": "2026-09-07T12:00:00Z", + "labels": {"client_id": "alice"} +} +``` + +The vector above is abbreviated; send the complete embedding from the same +recognizer model. Provide exactly one of `img` or `embedding`. Vectors must be +finite and nonzero. `registered_at` is optional and defaults to the current time; +replay the original timestamp to preserve it. + +The store upserts by exact vector. Registration now derives a stable ID from +that vector and the store namespace, so retries and replay after a restart +return the same ID without adding duplicate entries. Replaying updates the name +and labels. Images can produce slightly different embeddings across runs; keep +the original vector instead of embedding the photo again on each retry. + +This does not make the server store persistent. Clients must retain and restore +the records themselves. With independent stores behind a load balancer, replay +into each store or use a shared store. Do not mix different recognizer models in +one store. IDs from older versions change on their first registration replay. + ## 1:N identification workflow (register → identify → forget) This is the primary "face recognition" flow. Under the hood it uses