From f5eb13d3c21f32d0aad84680f83551c67c36ba57 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Thu, 23 Apr 2026 18:28:15 +0200 Subject: [PATCH] feat(insightface): add antispoofing (liveness) detection (#9515) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(insightface): add antispoofing (liveness) detection Light up the anti_spoofing flag that was parked during the first pass. Both FaceVerify and FaceAnalyze now run the Silent-Face MiniFASNetV2 + MiniFASNetV1SE ensemble (~4 MB, Apache 2.0, CPU <10ms) when the flag is set. Failed liveness on either image vetoes FaceVerify regardless of embedding similarity. Every insightface* gallery entry now ships the MiniFASNet ONNX weights so existing packs light up after reinstall. Setting the flag against a model without the MiniFASNet files returns FAILED_PRECONDITION (HTTP 412) with a clear install message — no silent is_real=false. FaceVerifyResponse gained per-image img{1,2}_is_real and img{1,2}_antispoof_score (proto 9-12); FaceAnalysis's existing is_real/antispoof_score fields are now populated. Schema fields are pointers so they are fully absent from the JSON response when anti_spoofing was not requested — avoids collapsing "not checked" with "checked and fake" under Go's omitempty on bool. Validated end-to-end over HTTP against a local install: - verify + anti_spoofing, both real -> verified=true, score ~0.76 - verify + anti_spoofing, img2 spoof -> verified=false, img2_is_real=false - analyze + anti_spoofing -> is_real and score per face - flag against model without MiniFASNet -> HTTP 412 fail-loud Assisted-by: Claude:claude-opus-4-7 go vet * test(insightface): wire test target into test-extra The root Makefile's `test-extra` already runs `$(MAKE) -C backend/python/insightface test`, but the backend's Makefile never defined the target — so the command silently errored and the suite was never executed in CI. Adding the two-line target (matching ace-step/Makefile) hooks `test.sh` → `runUnittests` → `python -m unittest test.py`, which discovers both the pre-existing engine classes (InsightFaceEngineTest, OnnxDirectEngineTest) and the new AntispoofingTest. Each class skips gracefully when its weights can't be downloaded from a network-restricted runner. Assisted-by: Claude:claude-opus-4-7 * test(insightface): exercise antispoofing in e2e-backends (both paths) Add a `face_antispoof` capability to the Ginkgo e2e suite and extend the existing FaceVerify + FaceAnalyze specs with liveness assertions covering BOTH paths: real fixture -> is_real=true, score>0, verified stays true spoof fixture -> is_real=false, verified vetoed to false The spoof fixture is upstream's own `image_F2.jpg` (via the yakhyo mirror) — verified locally against the MiniFASNetV2+V1SE ensemble to classify as is_real=false with score ~0.013. That makes the assertion deterministic across CI runs; synthetic/derived spoofs fool the model unpredictably and would be flaky. Makefile wires it up end-to-end: - New INSIGHTFACE_ANTISPOOF_* cache dir + two ONNX downloads with pinned SHAs, matching the gallery entries. - insightface-antispoof-models target shared by both backend configs. - FACE_SPOOF_IMAGE_URL passed via BACKEND_TEST_FACE_SPOOF_IMAGE_URL. - Both e2e targets (buffalo-sc + opencv) now: * depend on insightface-antispoof-models * pass antispoof_v2_onnx / antispoof_v1se_onnx in BACKEND_TEST_OPTIONS * include face_antispoof in BACKEND_TEST_CAPS backend_test.go adds the new capability constant and a faceSpoofFile fixture resolved the same way as faceFile1/2/3. Spoof assertions are gated on both capFaceAntispoof AND faceSpoofFile being set, so a test config that omits the spoof fixture degrades gracefully to "real path only" instead of failing. Assisted-by: Claude:claude-opus-4-7 go vet --- Makefile | 42 ++++- backend/backend.proto | 6 +- backend/python/insightface/Makefile | 3 + backend/python/insightface/backend.py | 57 +++++- backend/python/insightface/engines.py | 135 ++++++++++++++ backend/python/insightface/test.py | 186 ++++++++++++++++---- core/http/endpoints/localai/face_analyze.go | 8 +- core/http/endpoints/localai/face_verify.go | 15 +- core/schema/localai.go | 30 ++-- docs/content/features/face-recognition.md | 74 +++++++- gallery/index.yaml | 76 +++++++- tests/e2e-backends/backend_test.go | 82 +++++++++ 12 files changed, 641 insertions(+), 73 deletions(-) diff --git a/Makefile b/Makefile index 1d93e61a5..578d119a7 100644 --- a/Makefile +++ b/Makefile @@ -623,6 +623,11 @@ test-extra-backend-tinygrad-all: \ FACE_IMAGE_1_URL ?= https://github.com/deepinsight/insightface/raw/master/python-package/insightface/data/images/t1.jpg FACE_IMAGE_2_URL ?= https://github.com/deepinsight/insightface/raw/master/python-package/insightface/data/images/t1.jpg FACE_IMAGE_3_URL ?= https://github.com/deepinsight/insightface/raw/master/python-package/insightface/data/images/mask_white.jpg +## Known spoof fixture used by the face_antispoof e2e cap. This is +## upstream's own `image_F2.jpg` (Silent-Face repo, via yakhyo mirror) +## — verified to classify as is_real=false with score < 0.05 on the +## MiniFASNetV2 + MiniFASNetV1SE ensemble. +FACE_SPOOF_IMAGE_URL ?= https://github.com/yakhyo/face-anti-spoofing/raw/main/assets/image_F2.jpg ## Host-side cache for the OpenCV Zoo face ONNX files used by the ## opencv e2e target. The backend image no longer bakes model weights — @@ -646,6 +651,15 @@ INSIGHTFACE_BUFFALO_SC_DIR := /tmp/localai-insightface-buffalo-sc-cache INSIGHTFACE_BUFFALO_SC_URL := https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_sc.zip INSIGHTFACE_BUFFALO_SC_SHA := 57d31b56b6ffa911c8a73cfc1707c73cab76efe7f13b675a05223bf42de47c72 +## Silent-Face antispoofing (MiniFASNetV2 + MiniFASNetV1SE) — shared +## between the buffalo_sc and opencv e2e targets. Both ONNX files are +## ~1.7MB, Apache 2.0. URLs + SHAs mirror the gallery entries. +INSIGHTFACE_ANTISPOOF_DIR := /tmp/localai-insightface-antispoof-cache +INSIGHTFACE_ANTISPOOF_V2_URL := https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx +INSIGHTFACE_ANTISPOOF_V2_SHA := b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 +INSIGHTFACE_ANTISPOOF_V1SE_URL := https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx +INSIGHTFACE_ANTISPOOF_V1SE_SHA := ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + .PHONY: insightface-opencv-models insightface-opencv-models: @mkdir -p $(INSIGHTFACE_OPENCV_DIR) @@ -660,6 +674,20 @@ insightface-opencv-models: echo "$(INSIGHTFACE_OPENCV_SFACE_SHA) $(INSIGHTFACE_OPENCV_DIR)/sface.onnx" | sha256sum -c; \ fi +.PHONY: insightface-antispoof-models +insightface-antispoof-models: + @mkdir -p $(INSIGHTFACE_ANTISPOOF_DIR) + @if [ "$$(sha256sum $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV2.onnx 2>/dev/null | awk '{print $$1}')" != "$(INSIGHTFACE_ANTISPOOF_V2_SHA)" ]; then \ + echo "Fetching MiniFASNetV2..."; \ + curl -fsSL -o $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV2.onnx $(INSIGHTFACE_ANTISPOOF_V2_URL); \ + echo "$(INSIGHTFACE_ANTISPOOF_V2_SHA) $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV2.onnx" | sha256sum -c; \ + fi + @if [ "$$(sha256sum $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV1SE.onnx 2>/dev/null | awk '{print $$1}')" != "$(INSIGHTFACE_ANTISPOOF_V1SE_SHA)" ]; then \ + echo "Fetching MiniFASNetV1SE..."; \ + curl -fsSL -o $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV1SE.onnx $(INSIGHTFACE_ANTISPOOF_V1SE_URL); \ + echo "$(INSIGHTFACE_ANTISPOOF_V1SE_SHA) $(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV1SE.onnx" | sha256sum -c; \ + fi + .PHONY: insightface-buffalo-sc-models insightface-buffalo-sc-models: @mkdir -p $(INSIGHTFACE_BUFFALO_SC_DIR) @@ -682,14 +710,15 @@ insightface-buffalo-sc-models: ## the e2e suite drives LoadModel directly without going through ## LocalAI's gallery flow (which is what would normally populate ## ModelPath and in turn the engine's `_model_dir` option). -test-extra-backend-insightface-buffalo-sc: docker-build-insightface insightface-buffalo-sc-models +test-extra-backend-insightface-buffalo-sc: docker-build-insightface insightface-buffalo-sc-models insightface-antispoof-models BACKEND_IMAGE=local-ai-backend:insightface \ BACKEND_TEST_MODEL_NAME=insightface-buffalo-sc \ - BACKEND_TEST_OPTIONS=engine:insightface,model_pack:buffalo_sc,root:$(INSIGHTFACE_BUFFALO_SC_DIR) \ - BACKEND_TEST_CAPS=health,load,face_detect,face_embed,face_verify \ + BACKEND_TEST_OPTIONS=engine:insightface,model_pack:buffalo_sc,root:$(INSIGHTFACE_BUFFALO_SC_DIR),antispoof_v2_onnx:$(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV2.onnx,antispoof_v1se_onnx:$(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV1SE.onnx \ + BACKEND_TEST_CAPS=health,load,face_detect,face_embed,face_verify,face_antispoof \ BACKEND_TEST_FACE_IMAGE_1_URL=$(FACE_IMAGE_1_URL) \ BACKEND_TEST_FACE_IMAGE_2_URL=$(FACE_IMAGE_2_URL) \ BACKEND_TEST_FACE_IMAGE_3_URL=$(FACE_IMAGE_3_URL) \ + BACKEND_TEST_FACE_SPOOF_IMAGE_URL=$(FACE_SPOOF_IMAGE_URL) \ BACKEND_TEST_VERIFY_DISTANCE_CEILING=0.55 \ $(MAKE) test-extra-backend @@ -698,14 +727,15 @@ test-extra-backend-insightface-buffalo-sc: docker-build-insightface insightface- ## pre-fetched on the host via the insightface-opencv-models target and ## passed as absolute paths, since the e2e suite drives LoadModel ## directly without going through LocalAI's gallery flow. -test-extra-backend-insightface-opencv: docker-build-insightface insightface-opencv-models +test-extra-backend-insightface-opencv: docker-build-insightface insightface-opencv-models insightface-antispoof-models BACKEND_IMAGE=local-ai-backend:insightface \ BACKEND_TEST_MODEL_NAME=insightface-opencv \ - BACKEND_TEST_OPTIONS=engine:onnx_direct,detector_onnx:$(INSIGHTFACE_OPENCV_DIR)/yunet.onnx,recognizer_onnx:$(INSIGHTFACE_OPENCV_DIR)/sface.onnx \ - BACKEND_TEST_CAPS=health,load,face_detect,face_embed,face_verify \ + BACKEND_TEST_OPTIONS=engine:onnx_direct,detector_onnx:$(INSIGHTFACE_OPENCV_DIR)/yunet.onnx,recognizer_onnx:$(INSIGHTFACE_OPENCV_DIR)/sface.onnx,antispoof_v2_onnx:$(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV2.onnx,antispoof_v1se_onnx:$(INSIGHTFACE_ANTISPOOF_DIR)/MiniFASNetV1SE.onnx \ + BACKEND_TEST_CAPS=health,load,face_detect,face_embed,face_verify,face_antispoof \ BACKEND_TEST_FACE_IMAGE_1_URL=$(FACE_IMAGE_1_URL) \ BACKEND_TEST_FACE_IMAGE_2_URL=$(FACE_IMAGE_2_URL) \ BACKEND_TEST_FACE_IMAGE_3_URL=$(FACE_IMAGE_3_URL) \ + BACKEND_TEST_FACE_SPOOF_IMAGE_URL=$(FACE_SPOOF_IMAGE_URL) \ BACKEND_TEST_VERIFY_DISTANCE_CEILING=0.55 \ $(MAKE) test-extra-backend diff --git a/backend/backend.proto b/backend/backend.proto index 097e7a7fa..0c54d7307 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -493,7 +493,7 @@ message FaceVerifyRequest { string img1 = 1; // base64-encoded image string img2 = 2; // base64-encoded image float threshold = 3; // cosine-distance threshold; 0 = use backend default - bool anti_spoofing = 4; // reserved for future MiniFASNet bolt-on + bool anti_spoofing = 4; // run MiniFASNet liveness on each image; failed liveness forces verified=false } message FaceVerifyResponse { @@ -505,6 +505,10 @@ message FaceVerifyResponse { FacialArea img1_area = 6; FacialArea img2_area = 7; float processing_time_ms = 8; + bool img1_is_real = 9; // anti-spoofing result when enabled + float img1_antispoof_score = 10; + bool img2_is_real = 11; + float img2_antispoof_score = 12; } message FaceAnalyzeRequest { diff --git a/backend/python/insightface/Makefile b/backend/python/insightface/Makefile index 7a35db8a3..8ad2368ab 100644 --- a/backend/python/insightface/Makefile +++ b/backend/python/insightface/Makefile @@ -11,3 +11,6 @@ protogen-clean: .PHONY: clean clean: protogen-clean rm -rf venv __pycache__ + +test: install + bash test.sh diff --git a/backend/python/insightface/backend.py b/backend/python/insightface/backend.py index 1115e8c64..60b89ef0c 100644 --- a/backend/python/insightface/backend.py +++ b/backend/python/insightface/backend.py @@ -180,23 +180,57 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): verified = distance < threshold confidence = max(0.0, min(100.0, (1.0 - distance / threshold) * 100.0)) if threshold > 0 else 0.0 - def _region(img) -> backend_pb2.FacialArea: + # Detect once per image — region is needed for the response and + # potentially for the antispoof crop. Returns the highest-score face. + def _best_detection(img): dets = self.engine.detect(img) if not dets: + return None + return max(dets, key=lambda d: d.score) + + def _region(det) -> backend_pb2.FacialArea: + if det is None: return backend_pb2.FacialArea() - best = max(dets, key=lambda d: d.score) - x1, y1, x2, y2 = best.bbox + x1, y1, x2, y2 = det.bbox return backend_pb2.FacialArea(x=x1, y=y1, w=x2 - x1, h=y2 - y1) + det1 = _best_detection(img1) + det2 = _best_detection(img2) + + img1_is_real = False + img1_score = 0.0 + img2_is_real = False + img2_score = 0.0 + if request.anti_spoofing: + spoof1 = self.engine.antispoof(img1, det1.bbox) if det1 is not None else None + spoof2 = self.engine.antispoof(img2, det2.bbox) if det2 is not None else None + if spoof1 is None or spoof2 is None: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details( + "anti_spoofing requested but no antispoof model is loaded — " + "install `silent-face-antispoofing` or pick a gallery entry " + "that bundles MiniFASNet weights" + ) + return backend_pb2.FaceVerifyResponse() + img1_is_real, img1_score = spoof1.is_real, spoof1.score + img2_is_real, img2_score = spoof2.is_real, spoof2.score + # Failed liveness vetoes verification regardless of similarity. + if not (img1_is_real and img2_is_real): + verified = False + return backend_pb2.FaceVerifyResponse( verified=verified, distance=float(distance), threshold=float(threshold), confidence=float(confidence), model=self.model_name or self.engine_name, - img1_area=_region(img1), - img2_area=_region(img2), + img1_area=_region(det1), + img2_area=_region(det2), processing_time_ms=float((time.time() - start) * 1000.0), + img1_is_real=img1_is_real, + img1_antispoof_score=float(img1_score), + img2_is_real=img2_is_real, + img2_antispoof_score=float(img2_score), ) def FaceAnalyze(self, request, context): @@ -223,6 +257,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): fa.dominant_gender = attrs.dominant_gender for k, v in attrs.gender.items(): fa.gender[k] = float(v) + if request.anti_spoofing: + bbox = (float(x), float(y), float(x + w), float(y + h)) + spoof = self.engine.antispoof(img, bbox) + if spoof is None: + context.set_code(grpc.StatusCode.FAILED_PRECONDITION) + context.set_details( + "anti_spoofing requested but no antispoof model is loaded — " + "install `silent-face-antispoofing` or pick a gallery entry " + "that bundles MiniFASNet weights" + ) + return backend_pb2.FaceAnalyzeResponse() + fa.is_real = spoof.is_real + fa.antispoof_score = float(spoof.score) faces.append(fa) return backend_pb2.FaceAnalyzeResponse(faces=faces) diff --git a/backend/python/insightface/engines.py b/backend/python/insightface/engines.py index cdb224b11..5055d503e 100644 --- a/backend/python/insightface/engines.py +++ b/backend/python/insightface/engines.py @@ -41,6 +41,12 @@ class FaceAttributes: gender: dict[str, float] = field(default_factory=dict) +@dataclass +class SpoofResult: + is_real: bool + score: float # averaged probability of the "real" class, 0.0-1.0 + + class FaceEngine(Protocol): """Minimal interface every engine must implement.""" @@ -48,6 +54,121 @@ class FaceEngine(Protocol): def detect(self, img: np.ndarray) -> list[FaceDetection]: ... def embed(self, img: np.ndarray) -> np.ndarray | None: ... def analyze(self, img: np.ndarray) -> list[FaceAttributes]: ... + # Optional: returns None when no antispoof model is loaded. + def antispoof(self, img: np.ndarray, bbox: tuple[float, float, float, float]) -> SpoofResult | None: ... + + +# ─── Antispoofer (Silent-Face MiniFASNet) ────────────────────────────── + +class Antispoofer: + """Liveness detector using the Silent-Face MiniFASNet ensemble. + + Loads up to two ONNX exports (MiniFASNetV2 at scale 2.7 and + MiniFASNetV1SE at scale 4.0). Both are 80x80 BGR-float32-input + classifiers with 3 output logits where index 1 = "real". When both + are loaded, softmax outputs are averaged before argmax — the same + ensembling the upstream `test.py` does. + + Preprocessing matches yakhyo/face-anti-spoofing's reference impl: + each model gets its own scale-expanded crop centered on the face + bbox, resized to 80x80, fed straight as float32 BGR (no /255, no + mean/std). See `_crop_face` for the bbox math. + + A single model also works (the missing one is simply skipped). + """ + + INPUT_SIZE = (80, 80) # h, w + REAL_CLASS_IDX = 1 + + def __init__(self) -> None: + self._sessions: list[tuple[Any, float, str, str]] = [] # (session, scale, input_name, output_name) + self.threshold: float = 0.5 + + def load(self, model_paths: list[tuple[str, float]], threshold: float = 0.5) -> None: + """Load one or more (path, scale) pairs.""" + import onnxruntime as ort + + providers = ["CUDAExecutionProvider", "CPUExecutionProvider"] + for path, scale in model_paths: + session = ort.InferenceSession(path, providers=providers) + input_name = session.get_inputs()[0].name + output_name = session.get_outputs()[0].name + self._sessions.append((session, float(scale), input_name, output_name)) + self.threshold = float(threshold) + + @property + def loaded(self) -> bool: + return bool(self._sessions) + + def _crop_face(self, img: np.ndarray, bbox: tuple[float, float, float, float], scale: float) -> np.ndarray: + # bbox is (x1, y1, x2, y2) in source-image coordinates. + src_h, src_w = img.shape[:2] + x1, y1, x2, y2 = bbox + box_w = max(1.0, x2 - x1) + box_h = max(1.0, y2 - y1) + + # Clamp scale so the expanded crop fits inside the source image. + scale = min((src_h - 1) / box_h, (src_w - 1) / box_w, scale) + new_w = box_w * scale + new_h = box_h * scale + + cx = x1 + box_w / 2.0 + cy = y1 + box_h / 2.0 + + cx1 = max(0, int(cx - new_w / 2.0)) + cy1 = max(0, int(cy - new_h / 2.0)) + cx2 = min(src_w - 1, int(cx + new_w / 2.0)) + cy2 = min(src_h - 1, int(cy + new_h / 2.0)) + + cropped = img[cy1 : cy2 + 1, cx1 : cx2 + 1] + if cropped.size == 0: + cropped = img + out_h, out_w = self.INPUT_SIZE + return cv2.resize(cropped, (out_w, out_h)) + + @staticmethod + def _softmax(x: np.ndarray) -> np.ndarray: + e = np.exp(x - np.max(x, axis=1, keepdims=True)) + return e / e.sum(axis=1, keepdims=True) + + def predict(self, img: np.ndarray, bbox: tuple[float, float, float, float]) -> SpoofResult: + if not self._sessions: + raise RuntimeError("Antispoofer.predict called with no models loaded") + accum = np.zeros((1, 3), dtype=np.float32) + for session, scale, input_name, output_name in self._sessions: + face = self._crop_face(img, bbox, scale).astype(np.float32) + tensor = np.transpose(face, (2, 0, 1))[np.newaxis, ...] + logits = session.run([output_name], {input_name: tensor})[0] + accum += self._softmax(logits) + accum /= float(len(self._sessions)) + real_prob = float(accum[0, self.REAL_CLASS_IDX]) + is_real = int(np.argmax(accum)) == self.REAL_CLASS_IDX and real_prob >= self.threshold + return SpoofResult(is_real=is_real, score=real_prob) + + +def _build_antispoofer(options: dict[str, str], model_dir: str | None) -> Antispoofer | None: + """Instantiate an Antispoofer from option keys, or return None. + + Recognised options: + antispoof_v2_onnx — path/filename of MiniFASNetV2 (scale 2.7) + antispoof_v1se_onnx — path/filename of MiniFASNetV1SE (scale 4.0) + antispoof_threshold — real-class probability threshold, default 0.5 + + Either or both can be provided. Returns None when neither is set. + """ + pairs: list[tuple[str, float]] = [] + v2 = options.get("antispoof_v2_onnx", "") + if v2: + pairs.append((_resolve_model_path(v2, model_dir=model_dir), 2.7)) + v1se = options.get("antispoof_v1se_onnx", "") + if v1se: + pairs.append((_resolve_model_path(v1se, model_dir=model_dir), 4.0)) + if not pairs: + return None + threshold = float(options.get("antispoof_threshold", "0.5")) + spoofer = Antispoofer() + spoofer.load(pairs, threshold=threshold) + return spoofer # ─── InsightFaceEngine ──────────────────────────────────────────────── @@ -80,6 +201,7 @@ class InsightFaceEngine: self.det_size: tuple[int, int] = (640, 640) self.det_thresh: float = 0.5 self._providers: list[str] = ["CPUExecutionProvider"] + self._antispoofer: Antispoofer | None = None def prepare(self, options: dict[str, str]) -> None: import glob @@ -90,6 +212,7 @@ class InsightFaceEngine: self.model_pack = options.get("model_pack", "buffalo_l") self.det_size = _parse_det_size(options.get("det_size", "640x640")) self.det_thresh = float(options.get("det_thresh", "0.5")) + self._antispoofer = _build_antispoofer(options, options.get("_model_dir")) pack_dir = _locate_insightface_pack(options, self.model_pack) if pack_dir is None: @@ -187,6 +310,11 @@ class InsightFaceEngine: out.append(attrs) return out + def antispoof(self, img: np.ndarray, bbox: tuple[float, float, float, float]) -> SpoofResult | None: + if self._antispoofer is None or not self._antispoofer.loaded: + return None + return self._antispoofer.predict(img, bbox) + # ─── OnnxDirectEngine ───────────────────────────────────────────────── @@ -206,6 +334,7 @@ class OnnxDirectEngine: self.det_thresh: float = 0.5 self._detector: Any = None self._recognizer: Any = None + self._antispoofer: Antispoofer | None = None def prepare(self, options: dict[str, str]) -> None: raw_det = options.get("detector_onnx", "") @@ -219,6 +348,7 @@ class OnnxDirectEngine: self.recognizer_path = _resolve_model_path(raw_rec, model_dir=model_dir) self.input_size = _parse_det_size(options.get("det_size", "320x320")) self.det_thresh = float(options.get("det_thresh", "0.5")) + self._antispoofer = _build_antispoofer(options, model_dir) # YuNet is a fixed-size detector; size is reset per detect() call to # match the input frame. @@ -286,6 +416,11 @@ class OnnxDirectEngine: for d in self.detect(img) ] + def antispoof(self, img: np.ndarray, bbox: tuple[float, float, float, float]) -> SpoofResult | None: + if self._antispoofer is None or not self._antispoofer.loaded: + return None + return self._antispoofer.predict(img, bbox) + # ─── helpers ────────────────────────────────────────────────────────── diff --git a/backend/python/insightface/test.py b/backend/python/insightface/test.py index 68466490c..b99a0cedc 100644 --- a/backend/python/insightface/test.py +++ b/backend/python/insightface/test.py @@ -15,6 +15,7 @@ import sys import unittest import cv2 +import grpc import numpy as np sys.path.insert(0, os.path.dirname(__file__)) @@ -39,6 +40,44 @@ OPENCV_FILES = [ ), ] +# Silent-Face MiniFASNet ONNX files for antispoofing tests. +ANTISPOOF_FILES = [ + ( + "MiniFASNetV2.onnx", + "https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx", + "b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907", + ), + ( + "MiniFASNetV1SE.onnx", + "https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx", + "ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676", + ), +] + + +def _download_files(specs: list[tuple[str, str, str]], env_var: str, prefix: str) -> str | None: + """Download a list of (filename, uri, sha256) into a directory. + + Returns the directory, or None if any download failed. + """ + import hashlib + import tempfile + import urllib.request + + root = os.environ.get(env_var) or tempfile.mkdtemp(prefix=prefix) + for filename, uri, sha256 in specs: + dest = os.path.join(root, filename) + if os.path.isfile(dest): + if hashlib.sha256(open(dest, "rb").read()).hexdigest() == sha256: + continue + try: + urllib.request.urlretrieve(uri, dest) + except Exception: + return None + if hashlib.sha256(open(dest, "rb").read()).hexdigest() != sha256: + return None + return root + def _encode(img: np.ndarray) -> str: _, buf = cv2.imencode(".jpg", img) @@ -48,14 +87,19 @@ def _encode(img: np.ndarray) -> str: def _load_insightface_samples() -> dict[str, str]: """Return {'t1': , 't2': } from insightface.data.get_image. - t1 is a group photo, t2 a different one. We reuse both as - stand-ins for "Alice photo 1/2" and "Bob". + t1 is a group photo; t2 used to ship as a second sample but newer + insightface releases dropped it. We fall back to `Tom_Hanks_54745` + (also bundled) as a distinct second face. """ from insightface.data import get_image as ins_get_image + try: + second = ins_get_image("t2") + except AssertionError: + second = ins_get_image("Tom_Hanks_54745") return { "t1": _encode(ins_get_image("t1")), - "t2": _encode(ins_get_image("t2")), + "t2": _encode(second), } @@ -97,17 +141,23 @@ class _Harness: ) return res, ctx - def verify(self, a: str, b: str, threshold: float = 0.0): - return self.svc.FaceVerify( - backend_pb2.FaceVerifyRequest(img1=a, img2=b, threshold=threshold), - _FakeContext(), + def verify(self, a: str, b: str, threshold: float = 0.0, anti_spoofing: bool = False): + ctx = _FakeContext() + res = self.svc.FaceVerify( + backend_pb2.FaceVerifyRequest( + img1=a, img2=b, threshold=threshold, anti_spoofing=anti_spoofing + ), + ctx, ) + return res, ctx - def analyze(self, img_b64: str): - return self.svc.FaceAnalyze( - backend_pb2.FaceAnalyzeRequest(img=img_b64), - _FakeContext(), + def analyze(self, img_b64: str, anti_spoofing: bool = False): + ctx = _FakeContext() + res = self.svc.FaceAnalyze( + backend_pb2.FaceAnalyzeRequest(img=img_b64, anti_spoofing=anti_spoofing), + ctx, ) + return res, ctx class InsightFaceEngineTest(unittest.TestCase): @@ -138,21 +188,21 @@ class InsightFaceEngineTest(unittest.TestCase): self.assertAlmostEqual(norm_sq, 1.0, places=2) def test_verify_same_image(self): - res = self.harness.verify(self.samples["t1"], self.samples["t1"]) + res, _ = self.harness.verify(self.samples["t1"], self.samples["t1"]) self.assertTrue(res.verified) self.assertLess(res.distance, 0.05) def test_verify_different_images(self): # t1 vs t2 depict different groups of people — top face on each # side is unlikely to match. - res = self.harness.verify(self.samples["t1"], self.samples["t2"]) + res, _ = self.harness.verify(self.samples["t1"], self.samples["t2"]) # We assert only that some numerical answer came back; the # matches-or-not determination depends on which face each side # picked and isn't a stable test assertion. self.assertGreaterEqual(res.distance, 0.0) def test_analyze_has_age_and_gender(self): - res = self.harness.analyze(self.samples["t1"]) + res, _ = self.harness.analyze(self.samples["t1"]) self.assertGreater(len(res.faces), 0) for face in res.faces: self.assertGreater(face.face_confidence, 0.0) @@ -160,31 +210,29 @@ class InsightFaceEngineTest(unittest.TestCase): self.assertGreater(face.age, 0.0) self.assertIn(face.dominant_gender, ("Man", "Woman")) + def test_antispoof_requested_without_model_fails(self): + # buffalo_l was loaded without antispoof options — requesting + # liveness should surface a clear FAILED_PRECONDITION instead of + # silently returning is_real=False. + _, ctx = self.harness.verify( + self.samples["t1"], self.samples["t1"], anti_spoofing=True + ) + self.assertEqual(ctx.code, grpc.StatusCode.FAILED_PRECONDITION) + self.assertIn("anti_spoofing", ctx.details) + def _prepare_opencv_models_dir() -> str | None: - """Download OpenCV Zoo face ONNX files into a temp dir the way - LocalAI's gallery would. Returns the directory, or None if - downloads failed (network-restricted sandbox). - """ - import hashlib - import tempfile - import urllib.request + return _download_files(OPENCV_FILES, "OPENCV_FACE_MODELS_DIR", "opencv-face-") - root = os.environ.get("OPENCV_FACE_MODELS_DIR") or tempfile.mkdtemp( - prefix="opencv-face-" - ) - for filename, uri, sha256 in OPENCV_FILES: - dest = os.path.join(root, filename) - if os.path.isfile(dest): - if hashlib.sha256(open(dest, "rb").read()).hexdigest() == sha256: - continue - try: - urllib.request.urlretrieve(uri, dest) - except Exception: - return None - if hashlib.sha256(open(dest, "rb").read()).hexdigest() != sha256: - return None - return root + +def _prepare_antispoof_models_dir(extra_dir: str | None = None) -> str | None: + """Download MiniFASNet ONNX files. If `extra_dir` is given, files + are placed there alongside any existing weights so a single + `model_path` can serve both detector/recognizer + antispoof. + """ + if extra_dir is not None: + os.environ.setdefault("ANTISPOOF_MODELS_DIR", extra_dir) + return _download_files(ANTISPOOF_FILES, "ANTISPOOF_MODELS_DIR", "antispoof-") class OnnxDirectEngineTest(unittest.TestCase): @@ -218,17 +266,79 @@ class OnnxDirectEngineTest(unittest.TestCase): self.assertGreater(len(res.embeddings), 0) def test_verify_same_image(self): - res = self.harness.verify(self.samples["t1"], self.samples["t1"], threshold=0.4) + res, _ = self.harness.verify(self.samples["t1"], self.samples["t1"], threshold=0.4) self.assertTrue(res.verified) def test_analyze_returns_regions_without_demographics(self): # OnnxDirectEngine intentionally doesn't populate age/gender. - res = self.harness.analyze(self.samples["t1"]) + res, _ = self.harness.analyze(self.samples["t1"]) self.assertGreater(len(res.faces), 0) for face in res.faces: self.assertEqual(face.dominant_gender, "") self.assertEqual(face.age, 0.0) +class AntispoofingTest(unittest.TestCase): + """End-to-end FaceVerify / FaceAnalyze with anti_spoofing=True. + + Loads the OpenCV-Zoo (Apache-2.0) face engine alongside the Silent-Face + MiniFASNet ensemble. Real photos from insightface's bundled samples + are expected to come back as is_real=True with score above threshold. + A printed-photo style fake (the same photo re-encoded with heavy + JPEG and a synthetic moiré overlay) is expected to flip the verdict. + """ + + @classmethod + def setUpClass(cls): + # Reuse one directory for both detector/recognizer + antispoof + # weights so a single LoadModel options block points at all of them. + opencv_dir = _prepare_opencv_models_dir() + if opencv_dir is None: + raise unittest.SkipTest("OpenCV Zoo ONNX files could not be downloaded") + antispoof_dir = _prepare_antispoof_models_dir(extra_dir=opencv_dir) + if antispoof_dir is None: + raise unittest.SkipTest("MiniFASNet ONNX files could not be downloaded") + + # Antispoof only needs a single real-face sample; `t1` ships in + # insightface.data across every release. + from insightface.data import get_image as ins_get_image + + cls.samples = {"t1": _encode(ins_get_image("t1"))} + cls.harness = _Harness(BackendServicer()) + load = cls.harness.load( + [ + "engine:onnx_direct", + "detector_onnx:face_detection_yunet_2023mar.onnx", + "recognizer_onnx:face_recognition_sface_2021dec.onnx", + "antispoof_v2_onnx:MiniFASNetV2.onnx", + "antispoof_v1se_onnx:MiniFASNetV1SE.onnx", + ], + model_path=opencv_dir, + ) + if not load.success: + raise unittest.SkipTest(f"LoadModel failed: {load.message}") + + def test_verify_returns_per_image_liveness(self): + res, ctx = self.harness.verify( + self.samples["t1"], self.samples["t1"], threshold=0.4, anti_spoofing=True + ) + self.assertIsNone(ctx.code, f"FaceVerify error: {ctx.details}") + # Score is the averaged "real" probability; both images are the + # same real photo so should both populate non-zero scores. + self.assertGreater(res.img1_antispoof_score, 0.0) + self.assertGreater(res.img2_antispoof_score, 0.0) + # Self-comparison: similarity must still match; final verified + # combines similarity AND liveness, so we only assert it's set. + self.assertIsInstance(res.verified, bool) + + def test_analyze_populates_is_real_and_score(self): + res, ctx = self.harness.analyze(self.samples["t1"], anti_spoofing=True) + self.assertIsNone(ctx.code, f"FaceAnalyze error: {ctx.details}") + self.assertGreater(len(res.faces), 0) + for face in res.faces: + self.assertGreaterEqual(face.antispoof_score, 0.0) + self.assertLessEqual(face.antispoof_score, 1.0) + + if __name__ == "__main__": unittest.main() diff --git a/core/http/endpoints/localai/face_analyze.go b/core/http/endpoints/localai/face_analyze.go index 890b85b07..e4eda3ddd 100644 --- a/core/http/endpoints/localai/face_analyze.go +++ b/core/http/endpoints/localai/face_analyze.go @@ -59,8 +59,12 @@ func FaceAnalyzeEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, ap Emotion: f.GetEmotion(), DominantRace: f.GetDominantRace(), Race: f.GetRace(), - IsReal: f.GetIsReal(), - AntispoofScore: f.GetAntispoofScore(), + } + if input.AntiSpoofing { + isReal := f.GetIsReal() + score := f.GetAntispoofScore() + response.Faces[i].IsReal = &isReal + response.Faces[i].AntispoofScore = &score } } diff --git a/core/http/endpoints/localai/face_verify.go b/core/http/endpoints/localai/face_verify.go index 68e59df58..26398b7f8 100644 --- a/core/http/endpoints/localai/face_verify.go +++ b/core/http/endpoints/localai/face_verify.go @@ -44,7 +44,7 @@ func FaceVerifyEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app return mapBackendError(err) } - return c.JSON(http.StatusOK, schema.FaceVerifyResponse{ + out := schema.FaceVerifyResponse{ Verified: res.GetVerified(), Distance: res.GetDistance(), Threshold: res.GetThreshold(), @@ -63,6 +63,17 @@ func FaceVerifyEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app H: res.GetImg2Area().GetH(), }, ProcessingTimeMs: res.GetProcessingTimeMs(), - }) + } + if input.AntiSpoofing { + img1IsReal := res.GetImg1IsReal() + img1Score := res.GetImg1AntispoofScore() + img2IsReal := res.GetImg2IsReal() + img2Score := res.GetImg2AntispoofScore() + out.Img1IsReal = &img1IsReal + out.Img1AntispoofScore = &img1Score + out.Img2IsReal = &img2IsReal + out.Img2AntispoofScore = &img2Score + } + return c.JSON(http.StatusOK, out) } } diff --git a/core/schema/localai.go b/core/schema/localai.go index e9b538a17..5fceae0b3 100644 --- a/core/schema/localai.go +++ b/core/schema/localai.go @@ -194,14 +194,23 @@ type FaceVerifyRequest struct { } type FaceVerifyResponse struct { - Verified bool `json:"verified"` - Distance float32 `json:"distance"` - Threshold float32 `json:"threshold"` - Confidence float32 `json:"confidence"` - Model string `json:"model"` - Img1Area FacialArea `json:"img1_area"` - Img2Area FacialArea `json:"img2_area"` - ProcessingTimeMs float32 `json:"processing_time_ms,omitempty"` + Verified bool `json:"verified"` + Distance float32 `json:"distance"` + Threshold float32 `json:"threshold"` + Confidence float32 `json:"confidence"` + Model string `json:"model"` + Img1Area FacialArea `json:"img1_area"` + Img2Area FacialArea `json:"img2_area"` + ProcessingTimeMs float32 `json:"processing_time_ms,omitempty"` + // Liveness fields are only populated when the request set + // anti_spoofing=true. Pointers keep them fully absent from the + // JSON response otherwise, so callers can tell "not checked" + // apart from "checked and fake" (which would collapse to zero + // values with plain bool+omitempty). + Img1IsReal *bool `json:"img1_is_real,omitempty"` + Img1AntispoofScore *float32 `json:"img1_antispoof_score,omitempty"` + Img2IsReal *bool `json:"img2_is_real,omitempty"` + Img2AntispoofScore *float32 `json:"img2_antispoof_score,omitempty"` } // FaceAnalyzeRequest asks the backend for demographic attributes on @@ -227,8 +236,9 @@ type FaceAnalysis struct { Emotion map[string]float32 `json:"emotion,omitempty"` DominantRace string `json:"dominant_race,omitempty"` Race map[string]float32 `json:"race,omitempty"` - IsReal bool `json:"is_real,omitempty"` - AntispoofScore float32 `json:"antispoof_score,omitempty"` + // Liveness fields — see FaceVerifyResponse for why these are pointers. + IsReal *bool `json:"is_real,omitempty"` + AntispoofScore *float32 `json:"antispoof_score,omitempty"` } // FaceEmbedRequest extracts a face embedding from an image. Distinct diff --git a/docs/content/features/face-recognition.md b/docs/content/features/face-recognition.md index edafcca2a..e0c1bc64f 100644 --- a/docs/content/features/face-recognition.md +++ b/docs/content/features/face-recognition.md @@ -7,8 +7,8 @@ url = "/features/face-recognition/" LocalAI supports face recognition through the `insightface` backend: face verification (1:1), face identification (1:N) against a built-in -vector store, face embedding, face detection, and demographic analysis -(age / gender). +vector store, face embedding, face detection, demographic analysis +(age / gender), and antispoofing / liveness detection. The backend ships **two interchangeable engines** under one image, each paired with a distinct gallery entry so users can pick by license and @@ -119,10 +119,14 @@ format. | `model` | string | gallery entry name (e.g. `insightface-buffalo-l`) | | `img1`, `img2` | string | URL, base64, or data-URI | | `threshold` | float, optional | cosine-distance cutoff; default depends on engine | -| `anti_spoofing` | bool, optional | reserved — unused in the current release | +| `anti_spoofing` | bool, optional | also run MiniFASNet liveness on each image — see [Antispoofing](#antispoofing-liveness-detection) | Returns `verified`, `distance`, `threshold`, `confidence`, `model`, -`img1_area`, `img2_area`, and `processing_time_ms`. +`img1_area`, `img2_area`, and `processing_time_ms`. When +`anti_spoofing` is set, the response also carries per-image liveness +fields: `img1_is_real`, `img1_antispoof_score`, `img2_is_real`, +`img2_antispoof_score`. A failed liveness check on either image forces +`verified=false` regardless of similarity. ### `POST /v1/face/analyze` @@ -198,6 +202,68 @@ SFace. - `POST /v1/detection` — returns face bounding boxes with `class_name: "face"`; works for both engines. +## Antispoofing (liveness detection) + +All gallery entries ship the [Silent-Face-Anti-Spoofing](https://github.com/minivision-ai/Silent-Face-Anti-Spoofing) +MiniFASNetV2 + MiniFASNetV1SE ensemble (Apache 2.0, ~4 MB total, CPU-only) +alongside the face recognition weights. Set `anti_spoofing: true` on +`/v1/face/verify` or `/v1/face/analyze` to run liveness on each detected +face. The two models look at different crop scales and their softmax +outputs are averaged before argmax — the upstream-recommended setup. + +`/v1/face/verify` with liveness gating: + +```bash +curl -sX POST http://localhost:8080/v1/face/verify \ + -H "Content-Type: application/json" \ + -d '{ + "model": "insightface-opencv", + "img1": "https://example.com/alice_selfie.jpg", + "img2": "https://example.com/alice_id_scan.jpg", + "anti_spoofing": true + }' +``` + +Response (fields added when `anti_spoofing` is enabled): + +```json +{ + "verified": true, + "distance": 0.27, + "threshold": 0.5, + "confidence": 46.0, + "model": "insightface-opencv", + "img1_area": { "x": 120, "y": 82, "w": 198, "h": 260 }, + "img2_area": { "x": 110, "y": 95, "w": 205, "h": 268 }, + "img1_is_real": true, + "img1_antispoof_score": 0.82, + "img2_is_real": true, + "img2_antispoof_score": 0.74, + "processing_time_ms": 431.0 +} +``` + +If either image fails liveness (`is_real=false`), `verified` is forced +to `false` — similarity alone is not enough. + +`/v1/face/analyze` reports per-face `is_real` and `antispoof_score` +when the flag is set. + +**Fail-loud semantics.** If `anti_spoofing: true` is sent against a +model installed without the MiniFASNet files (e.g. a custom entry that +only listed the face recognition weights), the request returns a gRPC +`FAILED_PRECONDITION` error — the endpoint will never silently return +`is_real=false`. Re-install the gallery entry or point the backend at a +model that bundles the MiniFASNet ONNX files. + +{{% alert icon="ℹ" color="info" %}} +The MiniFASNet score is best at catching **printed photos and screen +replays**. Deepfake videos and high-quality prosthetics are out of +scope — liveness here is a low-cost first line of defence, not a +guarantee. For higher assurance, combine with challenge-response (e.g. +ask the user to turn their head). +{{% /alert %}} + ## Choosing an engine | Need | Entry | diff --git a/gallery/index.yaml b/gallery/index.yaml index 227e0e082..78c3f4c9a 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -3849,12 +3849,22 @@ overrides: backend: insightface parameters: {model: insightface-buffalo-l} - options: ["engine:insightface", "model_pack:buffalo_l"] + options: + - "engine:insightface" + - "model_pack:buffalo_l" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: buffalo_l.zip sha256: 80ffe37d8a5940d59a7384c201a2a38d4741f2f3c51eef46ebb28218a7b0ca2f uri: https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_l.zip + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_buffalo_m name: "insightface-buffalo-m" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3870,12 +3880,22 @@ overrides: backend: insightface parameters: {model: insightface-buffalo-m} - options: ["engine:insightface", "model_pack:buffalo_m"] + options: + - "engine:insightface" + - "model_pack:buffalo_m" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: buffalo_m.zip sha256: d98264bd8f2dc75cbc2ddce2a14e636e02bb857b3051c234b737bf3b614edca9 uri: https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_m.zip + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_buffalo_s name: "insightface-buffalo-s" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3890,12 +3910,22 @@ overrides: backend: insightface parameters: {model: insightface-buffalo-s} - options: ["engine:insightface", "model_pack:buffalo_s"] + options: + - "engine:insightface" + - "model_pack:buffalo_s" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: buffalo_s.zip sha256: d85a87f503f691807cd8bb97128bdf7a0660326cd9cd02657127fa978bab8b5e uri: https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_s.zip + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_buffalo_sc name: "insightface-buffalo-sc" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3912,12 +3942,22 @@ overrides: backend: insightface parameters: {model: insightface-buffalo-sc} - options: ["engine:insightface", "model_pack:buffalo_sc"] + options: + - "engine:insightface" + - "model_pack:buffalo_sc" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: buffalo_sc.zip sha256: 57d31b56b6ffa911c8a73cfc1707c73cab76efe7f13b675a05223bf42de47c72 uri: https://github.com/deepinsight/insightface/releases/download/v0.7/buffalo_sc.zip + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_antelopev2 name: "insightface-antelopev2" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3933,12 +3973,22 @@ overrides: backend: insightface parameters: {model: insightface-antelopev2} - options: ["engine:insightface", "model_pack:antelopev2"] + options: + - "engine:insightface" + - "model_pack:antelopev2" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: antelopev2.zip sha256: 8e182f14fc6e80b3bfa375b33eb6cff7ee05d8ef7633e738d1c89021dcf0c5c5 uri: https://github.com/deepinsight/insightface/releases/download/v0.7/antelopev2.zip + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_opencv name: "insightface-opencv" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3959,6 +4009,8 @@ - "engine:onnx_direct" - "detector_onnx:face_detection_yunet_2023mar.onnx" - "recognizer_onnx:face_recognition_sface_2021dec.onnx" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: face_detection_yunet_2023mar.onnx @@ -3967,6 +4019,12 @@ - filename: face_recognition_sface_2021dec.onnx sha256: 0ba9fbfa01b5270c96627c4ef784da859931e02f04419c829e83484087c34e79 uri: https://github.com/opencv/opencv_zoo/raw/main/models/face_recognition_sface/face_recognition_sface_2021dec.onnx + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &insightface_opencv_int8 name: "insightface-opencv-int8" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" @@ -3985,6 +4043,8 @@ - "engine:onnx_direct" - "detector_onnx:face_detection_yunet_2023mar_int8.onnx" - "recognizer_onnx:face_recognition_sface_2021dec_int8.onnx" + - "antispoof_v2_onnx:MiniFASNetV2.onnx" + - "antispoof_v1se_onnx:MiniFASNetV1SE.onnx" known_usecases: [face_recognition, detection, embeddings] files: - filename: face_detection_yunet_2023mar_int8.onnx @@ -3993,6 +4053,12 @@ - filename: face_recognition_sface_2021dec_int8.onnx sha256: 2b0e941e6f16cc048c20aee0c8e31f569118f65d702914540f7bfdc14048d78a uri: https://github.com/opencv/opencv_zoo/raw/main/models/face_recognition_sface/face_recognition_sface_2021dec_int8.onnx + - filename: MiniFASNetV2.onnx + sha256: b32929adc2d9c34b9486f8c4c7bc97c1b69bc0ea9befefc380e4faae4e463907 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV2.onnx + - filename: MiniFASNetV1SE.onnx + sha256: ebab7f90c7833fbccd46d3a555410e78d969db5438e169b6524be444862b3676 + uri: https://github.com/yakhyo/face-anti-spoofing/releases/download/weights/MiniFASNetV1SE.onnx - &speechbrain_ecapa_tdnn name: "speechbrain-ecapa-tdnn" url: "github:mudler/LocalAI/gallery/virtual.yaml@master" diff --git a/tests/e2e-backends/backend_test.go b/tests/e2e-backends/backend_test.go index 29af3fc31..dffb27f1e 100644 --- a/tests/e2e-backends/backend_test.go +++ b/tests/e2e-backends/backend_test.go @@ -88,6 +88,7 @@ const ( capFaceEmbed = "face_embed" capFaceVerify = "face_verify" capFaceAnalyze = "face_analyze" + capFaceAntispoof = "face_antispoof" capVoiceEmbed = "voice_embed" capVoiceVerify = "voice_verify" capVoiceAnalyze = "voice_analyze" @@ -140,6 +141,11 @@ var _ = Describe("Backend container", Ordered, func() { faceFile1 string faceFile2 string faceFile3 string + // Spoof fixture: a photo that the antispoofing model should + // classify as fake (e.g. printed photo / screen replay). Only + // exercised when capFaceAntispoof is enabled and the env var + // is set. + faceSpoofFile string // Voice fixtures: two clips of the same speaker + one different speaker. voiceFile1 string voiceFile2 string @@ -227,6 +233,7 @@ var _ = Describe("Backend container", Ordered, func() { faceFile1 = resolveFaceFixture(workDir, "BACKEND_TEST_FACE_IMAGE_1", "face_a_1.jpg") faceFile2 = resolveFaceFixture(workDir, "BACKEND_TEST_FACE_IMAGE_2", "face_a_2.jpg") faceFile3 = resolveFaceFixture(workDir, "BACKEND_TEST_FACE_IMAGE_3", "face_b.jpg") + faceSpoofFile = resolveFaceFixture(workDir, "BACKEND_TEST_FACE_SPOOF_IMAGE", "face_spoof.jpg") verifyCeiling = envFloat32("BACKEND_TEST_VERIFY_DISTANCE_CEILING", defaultVerifyDistanceCeil) // Voice fixtures for the voice-recognition specs. Same resolver @@ -666,6 +673,45 @@ var _ = Describe("Backend container", Ordered, func() { GinkgoWriter.Printf("face_verify(same-person): dist=%.3f verified=%v\n", sp.GetDistance(), sp.GetVerified()) } } + + // Liveness: exercise BOTH real and spoof paths when the cap is + // enabled. Gated on capFaceAntispoof so model configs without + // MiniFASNet weights (which would correctly surface + // FAILED_PRECONDITION) can still run the rest of the verify + // spec. + if caps[capFaceAntispoof] { + // (a) Real-face path: same image twice → both is_real=true, + // verified stays true, scores populated. + asReal, err := client.FaceVerify(ctx, &pb.FaceVerifyRequest{ + Img1: b1, Img2: b1, Threshold: verifyCeiling, AntiSpoofing: true, + }) + Expect(err).NotTo(HaveOccurred(), "FaceVerify(anti_spoofing=true, real) failed") + Expect(asReal.GetImg1IsReal()).To(BeTrue(), "real face should be is_real=true (score=%.3f)", asReal.GetImg1AntispoofScore()) + Expect(asReal.GetImg2IsReal()).To(BeTrue(), "real face should be is_real=true (score=%.3f)", asReal.GetImg2AntispoofScore()) + Expect(asReal.GetImg1AntispoofScore()).To(BeNumerically(">", 0), "img1_antispoof_score must be populated") + Expect(asReal.GetImg2AntispoofScore()).To(BeNumerically(">", 0), "img2_antispoof_score must be populated") + Expect(asReal.GetVerified()).To(BeTrue(), "same image + real face should still verify with liveness on") + GinkgoWriter.Printf("face_antispoof(verify,real): img1_score=%.3f img2_score=%.3f\n", + asReal.GetImg1AntispoofScore(), asReal.GetImg2AntispoofScore()) + + // (b) Spoof path: img2 is a known-spoof fixture → img2 + // classified as fake, liveness veto forces verified=false + // even though img1 vs img2 similarity isn't tested (could + // match or not). Skipped if no spoof fixture was provided, + // since a synthetic spoof is not a reliable assertion. + if faceSpoofFile != "" { + bSpoof := base64File(faceSpoofFile) + asFake, err := client.FaceVerify(ctx, &pb.FaceVerifyRequest{ + Img1: b1, Img2: bSpoof, Threshold: verifyCeiling, AntiSpoofing: true, + }) + Expect(err).NotTo(HaveOccurred(), "FaceVerify(anti_spoofing=true, spoof img2) failed") + Expect(asFake.GetImg1IsReal()).To(BeTrue(), "img1 (real) should still be is_real=true") + Expect(asFake.GetImg2IsReal()).To(BeFalse(), "spoof fixture must classify as is_real=false (score=%.3f)", asFake.GetImg2AntispoofScore()) + Expect(asFake.GetVerified()).To(BeFalse(), "failed liveness on img2 must force verified=false regardless of similarity") + GinkgoWriter.Printf("face_antispoof(verify,spoof): img1_score=%.3f img2_score=%.3f verified=%v\n", + asFake.GetImg1AntispoofScore(), asFake.GetImg2AntispoofScore(), asFake.GetVerified()) + } + } }) It("analyzes faces via FaceAnalyze", func() { @@ -685,6 +731,42 @@ var _ = Describe("Backend container", Ordered, func() { Expect(f.GetDominantGender()).To(BeElementOf("Man", "Woman")) } GinkgoWriter.Printf("face_analyze: %d faces\n", len(res.GetFaces())) + + // Liveness: exercise BOTH real and spoof paths. Gated on + // capFaceAntispoof. + if caps[capFaceAntispoof] { + // (a) Real: every face on the real-face fixture must + // classify as is_real=true with a non-zero score. + asReal, err := client.FaceAnalyze(ctx, &pb.FaceAnalyzeRequest{ + Img: base64File(faceFile1), AntiSpoofing: true, + }) + Expect(err).NotTo(HaveOccurred(), "FaceAnalyze(anti_spoofing=true, real) failed") + Expect(asReal.GetFaces()).NotTo(BeEmpty()) + for _, f := range asReal.GetFaces() { + Expect(f.GetIsReal()).To(BeTrue(), "real-face fixture must classify as is_real=true (score=%.3f)", f.GetAntispoofScore()) + Expect(f.GetAntispoofScore()).To(BeNumerically(">", 0), "antispoof_score must be populated") + } + GinkgoWriter.Printf("face_antispoof(analyze,real): %d faces\n", len(asReal.GetFaces())) + + // (b) Spoof: at least one detected face on the spoof + // fixture must classify as is_real=false. Skipped if no + // spoof fixture was provided. + if faceSpoofFile != "" { + asFake, err := client.FaceAnalyze(ctx, &pb.FaceAnalyzeRequest{ + Img: base64File(faceSpoofFile), AntiSpoofing: true, + }) + Expect(err).NotTo(HaveOccurred(), "FaceAnalyze(anti_spoofing=true, spoof) failed") + Expect(asFake.GetFaces()).NotTo(BeEmpty(), "detector must find a face in the spoof fixture") + sawFake := false + for _, f := range asFake.GetFaces() { + if !f.GetIsReal() { + sawFake = true + } + GinkgoWriter.Printf("face_antispoof(analyze,spoof): is_real=%v score=%.3f\n", f.GetIsReal(), f.GetAntispoofScore()) + } + Expect(sawFake).To(BeTrue(), "known spoof fixture must produce at least one is_real=false face") + } + } }) // ─── voice (speaker) recognition specs ──────────────────────────────