feat(parakeet-cpp): serve VoiceEmbed and VoiceVerify from the speaker encoder

Bind parakeet_capi_speaker_embed_pcm and parakeet_capi_free_floats with a
Dlsym probe and implement VoiceEmbed on the speaker context, so a model
with a speaker_model or a bundle speaker_component can serve the
realtime voice_recognition stage and /v1/voice/*. The response carries
the sha256 identity of the encoder weights.

VoiceVerify embeds both clips and compares them by cosine distance. It
refuses anti_spoofing because there is no such head.

A libparakeet.so without the symbols answers Unimplemented, and a model
without a speaker encoder answers FailedPrecondition.

Assisted-by: Claude Code:claude-sonnet-5-5

backend/go/parakeet-cpp/goparakeetcpp.go

@@ -147,6 +147,11 @@ var (
 	// CppTranscribeAndDiarizeNamedJSON are ABI v9; CppSpeakerRegistryAddEmbedding and
 	// CppDiarizeNamedPCMJSON are ABI v10. All are nil on an older libparakeet.so, and
 	// Load refuses speaker_model: unless the v10 ones are present.
+	// CppSpeakerEmbedPCM embeds 16 kHz mono PCM with the speaker context. It returns 0 on
+	// success; *out is a malloc'd float vector of *dim values that the caller releases with
+	// CppFreeFloats. Additive, nil on a library from before it (VoiceEmbed then refuses).
+	CppSpeakerEmbedPCM             func(speaker uintptr, pcm *float32, n, sampleRate int32, out, dim unsafe.Pointer) int32
+	CppFreeFloats                  func(p uintptr)
 	CppSpeakerIdentity             func(ctx uintptr) uintptr
 	CppDiarizeProfilesPCMJSON      func(diar, speaker, reg uintptr, samples *float32, n, sampleRate int32, acceptThreshold, margin float32) uintptr
 	CppSpeakerDim                  func(ctx uintptr) int32

backend/go/parakeet-cpp/goparakeetcpp_test.go

@@ -93,6 +93,13 @@ func ensureLibLoaded() {
 			purego.RegisterLibFunc(&CppSceneStreamBeginSpeaker, lib, "parakeet_capi_scene_stream_begin_speaker")
 			purego.RegisterLibFunc(&CppTranscribeAndDiarizeNamedJSON, lib, "parakeet_capi_transcribe_and_diarize_named_json")
 		}
+		if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_embed_pcm"); err == nil && sym != 0 {
+			purego.RegisterLibFunc(&CppSpeakerEmbedPCM, lib, "parakeet_capi_speaker_embed_pcm")
+			purego.RegisterLibFunc(&CppFreeFloats, lib, "parakeet_capi_free_floats")
+		}
+		if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_identity"); err == nil && sym != 0 {
+			purego.RegisterLibFunc(&CppSpeakerIdentity, lib, "parakeet_capi_speaker_identity")
+		}
 		if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_named_pcm_json"); err == nil && sym != 0 {
 			purego.RegisterLibFunc(&CppSpeakerRegistryAddEmbedding, lib, "parakeet_capi_speaker_registry_add_embedding")
 			purego.RegisterLibFunc(&CppDiarizeNamedPCMJSON, lib, "parakeet_capi_diarize_named_pcm_json")

backend/go/parakeet-cpp/main.go

@@ -169,6 +169,13 @@ func main() {
 		purego.RegisterLibFunc(&CppSpeakerRegistrySetStrict, lib, "parakeet_capi_speaker_registry_set_strict")
 		purego.RegisterLibFunc(&CppSpeakerEncoderFamily, lib, "parakeet_capi_speaker_encoder_family")
 	}
+	// Speaker embedding from PCM (additive, no ABI bump). Both symbols come together.
+	if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_embed_pcm"); err == nil && sym != 0 {
+		if sym2, err2 := purego.Dlsym(lib, "parakeet_capi_free_floats"); err2 == nil && sym2 != 0 {
+			purego.RegisterLibFunc(&CppSpeakerEmbedPCM, lib, "parakeet_capi_speaker_embed_pcm")
+			purego.RegisterLibFunc(&CppFreeFloats, lib, "parakeet_capi_free_floats")
+		}
+	}
 	// Word filter on transcription (additive, no ABI bump).
 	if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_path_json_with"); err == nil && sym != 0 {
 		purego.RegisterLibFunc(&CppTranscribePathJSONWith, lib, "parakeet_capi_transcribe_path_json_with")

backend/go/parakeet-cpp/voice_embed.go

@@ -0,0 +1,146 @@
+// SPDX-License-Identifier: MIT
+package main
+
+import (
+	"math"
+	"unsafe"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// speakerEmbedRate is the sample rate the speaker encoder takes; decodeWavMono16k
+// already resamples to it.
+const speakerEmbedRate = 16000
+
+// speakerEmbedReady reports why the loaded model cannot embed a voice, or nil.
+// An older libparakeet.so without parakeet_capi_speaker_embed_pcm gets Unimplemented;
+// a model without a speaker encoder (no speaker_component/speaker_model) gets
+// FailedPrecondition. Caller holds engineMu.
+func (p *ParakeetCpp) speakerEmbedReady() error {
+	if CppSpeakerEmbedPCM == nil || CppFreeFloats == nil {
+		return status.Error(codes.Unimplemented,
+			"parakeet-cpp: voice embedding needs a libparakeet.so with parakeet_capi_speaker_embed_pcm; rebuild the backend against a newer parakeet.cpp")
+	}
+	if p.spkCtx == 0 {
+		return status.Error(codes.FailedPrecondition,
+			"parakeet-cpp: no speaker encoder loaded; use a bundle with a voice component (speaker_component) or set speaker_model")
+	}
+	return nil
+}
+
+// embedPCMLocked embeds mono 16 kHz samples and copies the vector out of the C buffer.
+// Caller holds engineMu.
+func (p *ParakeetCpp) embedPCMLocked(pcm []float32) ([]float32, error) {
+	var vec uintptr
+	var dim int32
+	rc := CppSpeakerEmbedPCM(p.spkCtx, &pcm[0], int32(len(pcm)), speakerEmbedRate, unsafe.Pointer(&vec), unsafe.Pointer(&dim))
+	if rc != 0 || vec == 0 || dim <= 0 {
+		if vec != 0 {
+			CppFreeFloats(vec)
+		}
+		return nil, status.Errorf(codes.Internal, "parakeet-cpp: speaker embedding failed: %s", CppLastError(p.spkCtx))
+	}
+	defer CppFreeFloats(vec)
+	src := unsafe.Slice((*float32)(unsafe.Pointer(vec)), int(dim)) //nolint:govet // C-owned malloc'd vector, copied out before free
+	out := make([]float32, int(dim))
+	copy(out, src)
+	return out, nil
+}
+
+// embedFile decodes an audio file to 16 kHz mono and embeds it under engineMu.
+func (p *ParakeetCpp) embedFile(path string) ([]float32, error) {
+	if path == "" {
+		return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: audio path is required")
+	}
+	pcm, _, err := decodeWavMono16k(path)
+	if err != nil {
+		return nil, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err)
+	}
+	if len(pcm) == 0 {
+		return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio")
+	}
+	p.engineMu.Lock()
+	defer p.engineMu.Unlock()
+	if err := p.speakerEmbedReady(); err != nil {
+		return nil, err
+	}
+	return p.embedPCMLocked(pcm)
+}
+
+// speakerIdentity is the "sha256:" identity of the loaded speaker encoder weights, "" when
+// the library cannot report one. Caller holds engineMu.
+func (p *ParakeetCpp) speakerIdentityLocked() string {
+	if p.spkCtx == 0 || CppSpeakerIdentity == nil {
+		return ""
+	}
+	return goStringFromCPtr(CppSpeakerIdentity(p.spkCtx))
+}
+
+// VoiceEmbed returns the speaker embedding of the audio file in req.Audio. The embedding
+// space is the one of the loaded speaker encoder, so a bundle's voice component gives the
+// same vectors as the standalone model with the same weights.
+func (p *ParakeetCpp) VoiceEmbed(req *pb.VoiceEmbedRequest) (pb.VoiceEmbedResponse, error) {
+	emb, err := p.embedFile(req.GetAudio())
+	if err != nil {
+		return pb.VoiceEmbedResponse{}, err
+	}
+	p.engineMu.Lock()
+	model := p.speakerIdentityLocked()
+	p.engineMu.Unlock()
+	return pb.VoiceEmbedResponse{Embedding: emb, Model: model}, nil
+}
+
+// VoiceVerify embeds both clips and compares them by cosine distance. A request threshold
+// of 0 or less uses the default speaker distance. There is no anti-spoofing head, so a
+// request for it is refused rather than answered without the check.
+func (p *ParakeetCpp) VoiceVerify(req *pb.VoiceVerifyRequest) (pb.VoiceVerifyResponse, error) {
+	if req.GetAntiSpoofing() {
+		return pb.VoiceVerifyResponse{}, status.Error(codes.Unimplemented, "parakeet-cpp: anti-spoofing is not supported")
+	}
+	if req.GetAudio1() == "" || req.GetAudio2() == "" {
+		return pb.VoiceVerifyResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: audio1 and audio2 are required")
+	}
+	a, err := p.embedFile(req.GetAudio1())
+	if err != nil {
+		return pb.VoiceVerifyResponse{}, err
+	}
+	b, err := p.embedFile(req.GetAudio2())
+	if err != nil {
+		return pb.VoiceVerifyResponse{}, err
+	}
+	if len(a) != len(b) {
+		return pb.VoiceVerifyResponse{}, status.Errorf(codes.Internal, "parakeet-cpp: embedding sizes differ (%d and %d)", len(a), len(b))
+	}
+	threshold := req.GetThreshold()
+	if threshold <= 0 {
+		threshold = defaultSpeakerDistance
+	}
+	distance := cosineDistance(a, b)
+	confidence := float32(math.Max(0, math.Min(100, (1-float64(distance)/float64(threshold))*100)))
+	p.engineMu.Lock()
+	model := p.speakerIdentityLocked()
+	p.engineMu.Unlock()
+	return pb.VoiceVerifyResponse{
+		Verified:   distance <= threshold,
+		Distance:   distance,
+		Threshold:  threshold,
+		Confidence: confidence,
+		Model:      model,
+	}, nil
+}
+
+// cosineDistance is 1 minus the cosine similarity; a zero vector counts as maximally far.
+func cosineDistance(a, b []float32) float32 {
+	var dot, na, nb float64
+	for i := range a {
+		dot += float64(a[i]) * float64(b[i])
+		na += float64(a[i]) * float64(a[i])
+		nb += float64(b[i]) * float64(b[i])
+	}
+	if na == 0 || nb == 0 {
+		return 1
+	}
+	return float32(1 - dot/(math.Sqrt(na)*math.Sqrt(nb)))
+}

backend/go/parakeet-cpp/voice_embed_test.go

@@ -0,0 +1,165 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"unsafe"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+var _ = Describe("ParakeetCpp.VoiceEmbed", func() {
+	var (
+		restore func()
+		pool    *diarizeCstrPool
+		keep    [][]float32 // the fake "C" vectors, alive until freed
+		freed   []uintptr
+		got     struct {
+			ctx  uintptr
+			n    int32
+			rate int32
+		}
+		vector []float32
+		rc     int32
+	)
+	BeforeEach(func() {
+		sEmbed, sFree, sID, sErr := CppSpeakerEmbedPCM, CppFreeFloats, CppSpeakerIdentity, CppLastError
+		restore = func() { CppSpeakerEmbedPCM, CppFreeFloats, CppSpeakerIdentity, CppLastError = sEmbed, sFree, sID, sErr }
+		pool = &diarizeCstrPool{}
+		keep, freed, vector, rc = nil, nil, []float32{0.5, 0.25, 0.125}, 0
+		got.ctx, got.n, got.rate = 0, 0, 0
+		CppSpeakerEmbedPCM = func(spk uintptr, pcm *float32, n, rate int32, out, dim unsafe.Pointer) int32 {
+			got.ctx, got.n, got.rate = spk, n, rate
+			if rc != 0 {
+				return rc
+			}
+			v := append([]float32(nil), vector...)
+			keep = append(keep, v)
+			*(*uintptr)(out) = uintptr(unsafe.Pointer(&v[0]))
+			*(*int32)(dim) = int32(len(v))
+			return 0
+		}
+		CppFreeFloats = func(p uintptr) { freed = append(freed, p) }
+		CppSpeakerIdentity = func(uintptr) uintptr { return pool.cstr("sha256:abcd") }
+		CppLastError = func(uintptr) string { return "stub failure" }
+	})
+	AfterEach(func() { restore() })
+
+	It("embeds the decoded clip with the speaker context and frees the C vector", func() {
+		p := &ParakeetCpp{spkCtx: 9}
+		res, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(res.Embedding).To(Equal([]float32{0.5, 0.25, 0.125}))
+		Expect(res.Model).To(Equal("sha256:abcd"))
+		Expect(got.ctx).To(Equal(uintptr(9)))
+		Expect(got.n).To(Equal(int32(16000)))
+		Expect(got.rate).To(Equal(int32(16000)))
+		Expect(freed).To(HaveLen(1))
+		Expect(freed[0]).To(Equal(uintptr(unsafe.Pointer(&keep[0][0]))))
+	})
+
+	It("fails with FailedPrecondition when the model has no speaker encoder", func() {
+		_, err := (&ParakeetCpp{}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)})
+		Expect(status.Code(err)).To(Equal(codes.FailedPrecondition))
+		Expect(err.Error()).To(ContainSubstring("speaker"))
+	})
+
+	It("fails with Unimplemented on a library without the embed symbols", func() {
+		CppSpeakerEmbedPCM, CppFreeFloats = nil, nil
+		_, err := (&ParakeetCpp{spkCtx: 9}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)})
+		Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+		Expect(err.Error()).To(ContainSubstring("parakeet_capi_speaker_embed_pcm"))
+	})
+
+	It("rejects a missing path, an undecodable file and empty audio", func() {
+		p := &ParakeetCpp{spkCtx: 9}
+		_, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{})
+		Expect(status.Code(err)).To(Equal(codes.InvalidArgument))
+
+		bad := filepath.Join(GinkgoT().TempDir(), "bad.wav")
+		Expect(os.WriteFile(bad, []byte("not audio"), 0o600)).To(Succeed())
+		_, err = p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: bad})
+		Expect(status.Code(err)).To(Equal(codes.InvalidArgument))
+
+		empty := filepath.Join(GinkgoT().TempDir(), "empty.wav")
+		writeMono16kWav(empty, 0)
+		_, err = p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: empty})
+		Expect(status.Code(err)).To(Equal(codes.InvalidArgument))
+		Expect(got.n).To(BeZero(), "the encoder must not run on invalid audio")
+	})
+
+	It("returns the library error when the encoder fails", func() {
+		rc = 1
+		_, err := (&ParakeetCpp{spkCtx: 9}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)})
+		Expect(status.Code(err)).To(Equal(codes.Internal))
+		Expect(err.Error()).To(ContainSubstring("stub failure"))
+	})
+
+	Describe("VoiceVerify", func() {
+		It("accepts the same voice and rejects a different one", func() {
+			p := &ParakeetCpp{spkCtx: 9}
+			res, err := p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: diarizeWav(1), Audio2: diarizeWav(1)})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Verified).To(BeTrue())
+			Expect(res.Distance).To(BeNumerically("~", 0, 1e-6))
+			Expect(res.Threshold).To(BeNumerically("~", 0.5, 1e-6))
+			Expect(res.Confidence).To(BeNumerically("~", 100, 1e-3))
+
+			vectors := [][]float32{{1, 0, 0}, {0, 1, 0}}
+			calls := 0
+			CppSpeakerEmbedPCM = func(_ uintptr, _ *float32, _, _ int32, out, dim unsafe.Pointer) int32 {
+				v := vectors[calls%2]
+				calls++
+				keep = append(keep, v)
+				*(*uintptr)(out) = uintptr(unsafe.Pointer(&v[0]))
+				*(*int32)(dim) = 3
+				return 0
+			}
+			res, err = p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: diarizeWav(1), Audio2: diarizeWav(1), Threshold: 0.3})
+			Expect(err).ToNot(HaveOccurred())
+			Expect(res.Verified).To(BeFalse())
+			Expect(res.Distance).To(BeNumerically("~", 1, 1e-6))
+			Expect(res.Threshold).To(BeNumerically("~", 0.3, 1e-6))
+		})
+
+		It("refuses anti-spoofing and missing clips", func() {
+			p := &ParakeetCpp{spkCtx: 9}
+			_, err := p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: "a", Audio2: "b", AntiSpoofing: true})
+			Expect(status.Code(err)).To(Equal(codes.Unimplemented))
+			_, err = p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: "a"})
+			Expect(status.Code(err)).To(Equal(codes.InvalidArgument))
+		})
+	})
+})
+
+// PARAKEET_BACKEND_TEST_BUNDLE (or ..._SPEAKER_MODEL) points at a GGUF with a speaker
+// encoder and PARAKEET_BACKEND_TEST_WAV at any speech WAV; the spec needs a libparakeet.so
+// that exports parakeet_capi_speaker_embed_pcm (PARAKEET_LIBRARY).
+var _ = Describe("ParakeetCpp.VoiceEmbed (real libparakeet.so)", func() {
+	It("embeds a clip, repeats exactly and reports the encoder identity", func() {
+		model := os.Getenv("PARAKEET_BACKEND_TEST_SPEAKER_BUNDLE")
+		wav := os.Getenv("PARAKEET_BACKEND_TEST_WAV")
+		if model == "" || wav == "" {
+			Skip("set PARAKEET_BACKEND_TEST_SPEAKER_BUNDLE (a bundle GGUF with a voice component) and PARAKEET_BACKEND_TEST_WAV")
+		}
+		ensureLibLoaded()
+		if CppSpeakerEmbedPCM == nil {
+			Skip("libparakeet.so has no parakeet_capi_speaker_embed_pcm")
+		}
+		p := &ParakeetCpp{}
+		Expect(p.Load(&pb.ModelOptions{ModelFile: model, Options: []string{"speaker_component:voice"}})).To(Succeed())
+		defer func() { _ = p.Free() }()
+
+		first, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: wav})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(first.Embedding).ToNot(BeEmpty())
+		Expect(first.Model).To(HavePrefix("sha256:"))
+		again, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: wav})
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cosineDistance(first.Embedding, again.Embedding)).To(BeNumerically("<", 1e-4))
+	})
+})