diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index c8d8b9b68..343db03ee 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -147,6 +147,11 @@ var ( // CppTranscribeAndDiarizeNamedJSON are ABI v9; CppSpeakerRegistryAddEmbedding and // CppDiarizeNamedPCMJSON are ABI v10. All are nil on an older libparakeet.so, and // Load refuses speaker_model: unless the v10 ones are present. + // CppSpeakerEmbedPCM embeds 16 kHz mono PCM with the speaker context. It returns 0 on + // success; *out is a malloc'd float vector of *dim values that the caller releases with + // CppFreeFloats. Additive, nil on a library from before it (VoiceEmbed then refuses). + CppSpeakerEmbedPCM func(speaker uintptr, pcm *float32, n, sampleRate int32, out, dim unsafe.Pointer) int32 + CppFreeFloats func(p uintptr) CppSpeakerIdentity func(ctx uintptr) uintptr CppDiarizeProfilesPCMJSON func(diar, speaker, reg uintptr, samples *float32, n, sampleRate int32, acceptThreshold, margin float32) uintptr CppSpeakerDim func(ctx uintptr) int32 diff --git a/backend/go/parakeet-cpp/goparakeetcpp_test.go b/backend/go/parakeet-cpp/goparakeetcpp_test.go index 7c1469179..11cd144ad 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp_test.go +++ b/backend/go/parakeet-cpp/goparakeetcpp_test.go @@ -93,6 +93,13 @@ func ensureLibLoaded() { purego.RegisterLibFunc(&CppSceneStreamBeginSpeaker, lib, "parakeet_capi_scene_stream_begin_speaker") purego.RegisterLibFunc(&CppTranscribeAndDiarizeNamedJSON, lib, "parakeet_capi_transcribe_and_diarize_named_json") } + if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_embed_pcm"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerEmbedPCM, lib, "parakeet_capi_speaker_embed_pcm") + purego.RegisterLibFunc(&CppFreeFloats, lib, "parakeet_capi_free_floats") + } + if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_identity"); err == nil && sym != 0 { + purego.RegisterLibFunc(&CppSpeakerIdentity, lib, "parakeet_capi_speaker_identity") + } if sym, err := purego.Dlsym(lib, "parakeet_capi_diarize_named_pcm_json"); err == nil && sym != 0 { purego.RegisterLibFunc(&CppSpeakerRegistryAddEmbedding, lib, "parakeet_capi_speaker_registry_add_embedding") purego.RegisterLibFunc(&CppDiarizeNamedPCMJSON, lib, "parakeet_capi_diarize_named_pcm_json") diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index fbc6cb6ee..0364f19f4 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -169,6 +169,13 @@ func main() { purego.RegisterLibFunc(&CppSpeakerRegistrySetStrict, lib, "parakeet_capi_speaker_registry_set_strict") purego.RegisterLibFunc(&CppSpeakerEncoderFamily, lib, "parakeet_capi_speaker_encoder_family") } + // Speaker embedding from PCM (additive, no ABI bump). Both symbols come together. + if sym, err := purego.Dlsym(lib, "parakeet_capi_speaker_embed_pcm"); err == nil && sym != 0 { + if sym2, err2 := purego.Dlsym(lib, "parakeet_capi_free_floats"); err2 == nil && sym2 != 0 { + purego.RegisterLibFunc(&CppSpeakerEmbedPCM, lib, "parakeet_capi_speaker_embed_pcm") + purego.RegisterLibFunc(&CppFreeFloats, lib, "parakeet_capi_free_floats") + } + } // Word filter on transcription (additive, no ABI bump). if sym, err := purego.Dlsym(lib, "parakeet_capi_transcribe_path_json_with"); err == nil && sym != 0 { purego.RegisterLibFunc(&CppTranscribePathJSONWith, lib, "parakeet_capi_transcribe_path_json_with") diff --git a/backend/go/parakeet-cpp/voice_embed.go b/backend/go/parakeet-cpp/voice_embed.go new file mode 100644 index 000000000..2d00beb7f --- /dev/null +++ b/backend/go/parakeet-cpp/voice_embed.go @@ -0,0 +1,146 @@ +// SPDX-License-Identifier: MIT +package main + +import ( + "math" + "unsafe" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// speakerEmbedRate is the sample rate the speaker encoder takes; decodeWavMono16k +// already resamples to it. +const speakerEmbedRate = 16000 + +// speakerEmbedReady reports why the loaded model cannot embed a voice, or nil. +// An older libparakeet.so without parakeet_capi_speaker_embed_pcm gets Unimplemented; +// a model without a speaker encoder (no speaker_component/speaker_model) gets +// FailedPrecondition. Caller holds engineMu. +func (p *ParakeetCpp) speakerEmbedReady() error { + if CppSpeakerEmbedPCM == nil || CppFreeFloats == nil { + return status.Error(codes.Unimplemented, + "parakeet-cpp: voice embedding needs a libparakeet.so with parakeet_capi_speaker_embed_pcm; rebuild the backend against a newer parakeet.cpp") + } + if p.spkCtx == 0 { + return status.Error(codes.FailedPrecondition, + "parakeet-cpp: no speaker encoder loaded; use a bundle with a voice component (speaker_component) or set speaker_model") + } + return nil +} + +// embedPCMLocked embeds mono 16 kHz samples and copies the vector out of the C buffer. +// Caller holds engineMu. +func (p *ParakeetCpp) embedPCMLocked(pcm []float32) ([]float32, error) { + var vec uintptr + var dim int32 + rc := CppSpeakerEmbedPCM(p.spkCtx, &pcm[0], int32(len(pcm)), speakerEmbedRate, unsafe.Pointer(&vec), unsafe.Pointer(&dim)) + if rc != 0 || vec == 0 || dim <= 0 { + if vec != 0 { + CppFreeFloats(vec) + } + return nil, status.Errorf(codes.Internal, "parakeet-cpp: speaker embedding failed: %s", CppLastError(p.spkCtx)) + } + defer CppFreeFloats(vec) + src := unsafe.Slice((*float32)(unsafe.Pointer(vec)), int(dim)) //nolint:govet // C-owned malloc'd vector, copied out before free + out := make([]float32, int(dim)) + copy(out, src) + return out, nil +} + +// embedFile decodes an audio file to 16 kHz mono and embeds it under engineMu. +func (p *ParakeetCpp) embedFile(path string) ([]float32, error) { + if path == "" { + return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: audio path is required") + } + pcm, _, err := decodeWavMono16k(path) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err) + } + if len(pcm) == 0 { + return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio") + } + p.engineMu.Lock() + defer p.engineMu.Unlock() + if err := p.speakerEmbedReady(); err != nil { + return nil, err + } + return p.embedPCMLocked(pcm) +} + +// speakerIdentity is the "sha256:" identity of the loaded speaker encoder weights, "" when +// the library cannot report one. Caller holds engineMu. +func (p *ParakeetCpp) speakerIdentityLocked() string { + if p.spkCtx == 0 || CppSpeakerIdentity == nil { + return "" + } + return goStringFromCPtr(CppSpeakerIdentity(p.spkCtx)) +} + +// VoiceEmbed returns the speaker embedding of the audio file in req.Audio. The embedding +// space is the one of the loaded speaker encoder, so a bundle's voice component gives the +// same vectors as the standalone model with the same weights. +func (p *ParakeetCpp) VoiceEmbed(req *pb.VoiceEmbedRequest) (pb.VoiceEmbedResponse, error) { + emb, err := p.embedFile(req.GetAudio()) + if err != nil { + return pb.VoiceEmbedResponse{}, err + } + p.engineMu.Lock() + model := p.speakerIdentityLocked() + p.engineMu.Unlock() + return pb.VoiceEmbedResponse{Embedding: emb, Model: model}, nil +} + +// VoiceVerify embeds both clips and compares them by cosine distance. A request threshold +// of 0 or less uses the default speaker distance. There is no anti-spoofing head, so a +// request for it is refused rather than answered without the check. +func (p *ParakeetCpp) VoiceVerify(req *pb.VoiceVerifyRequest) (pb.VoiceVerifyResponse, error) { + if req.GetAntiSpoofing() { + return pb.VoiceVerifyResponse{}, status.Error(codes.Unimplemented, "parakeet-cpp: anti-spoofing is not supported") + } + if req.GetAudio1() == "" || req.GetAudio2() == "" { + return pb.VoiceVerifyResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: audio1 and audio2 are required") + } + a, err := p.embedFile(req.GetAudio1()) + if err != nil { + return pb.VoiceVerifyResponse{}, err + } + b, err := p.embedFile(req.GetAudio2()) + if err != nil { + return pb.VoiceVerifyResponse{}, err + } + if len(a) != len(b) { + return pb.VoiceVerifyResponse{}, status.Errorf(codes.Internal, "parakeet-cpp: embedding sizes differ (%d and %d)", len(a), len(b)) + } + threshold := req.GetThreshold() + if threshold <= 0 { + threshold = defaultSpeakerDistance + } + distance := cosineDistance(a, b) + confidence := float32(math.Max(0, math.Min(100, (1-float64(distance)/float64(threshold))*100))) + p.engineMu.Lock() + model := p.speakerIdentityLocked() + p.engineMu.Unlock() + return pb.VoiceVerifyResponse{ + Verified: distance <= threshold, + Distance: distance, + Threshold: threshold, + Confidence: confidence, + Model: model, + }, nil +} + +// cosineDistance is 1 minus the cosine similarity; a zero vector counts as maximally far. +func cosineDistance(a, b []float32) float32 { + var dot, na, nb float64 + for i := range a { + dot += float64(a[i]) * float64(b[i]) + na += float64(a[i]) * float64(a[i]) + nb += float64(b[i]) * float64(b[i]) + } + if na == 0 || nb == 0 { + return 1 + } + return float32(1 - dot/(math.Sqrt(na)*math.Sqrt(nb))) +} diff --git a/backend/go/parakeet-cpp/voice_embed_test.go b/backend/go/parakeet-cpp/voice_embed_test.go new file mode 100644 index 000000000..68796c8cf --- /dev/null +++ b/backend/go/parakeet-cpp/voice_embed_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "os" + "path/filepath" + "unsafe" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +var _ = Describe("ParakeetCpp.VoiceEmbed", func() { + var ( + restore func() + pool *diarizeCstrPool + keep [][]float32 // the fake "C" vectors, alive until freed + freed []uintptr + got struct { + ctx uintptr + n int32 + rate int32 + } + vector []float32 + rc int32 + ) + BeforeEach(func() { + sEmbed, sFree, sID, sErr := CppSpeakerEmbedPCM, CppFreeFloats, CppSpeakerIdentity, CppLastError + restore = func() { CppSpeakerEmbedPCM, CppFreeFloats, CppSpeakerIdentity, CppLastError = sEmbed, sFree, sID, sErr } + pool = &diarizeCstrPool{} + keep, freed, vector, rc = nil, nil, []float32{0.5, 0.25, 0.125}, 0 + got.ctx, got.n, got.rate = 0, 0, 0 + CppSpeakerEmbedPCM = func(spk uintptr, pcm *float32, n, rate int32, out, dim unsafe.Pointer) int32 { + got.ctx, got.n, got.rate = spk, n, rate + if rc != 0 { + return rc + } + v := append([]float32(nil), vector...) + keep = append(keep, v) + *(*uintptr)(out) = uintptr(unsafe.Pointer(&v[0])) + *(*int32)(dim) = int32(len(v)) + return 0 + } + CppFreeFloats = func(p uintptr) { freed = append(freed, p) } + CppSpeakerIdentity = func(uintptr) uintptr { return pool.cstr("sha256:abcd") } + CppLastError = func(uintptr) string { return "stub failure" } + }) + AfterEach(func() { restore() }) + + It("embeds the decoded clip with the speaker context and frees the C vector", func() { + p := &ParakeetCpp{spkCtx: 9} + res, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Embedding).To(Equal([]float32{0.5, 0.25, 0.125})) + Expect(res.Model).To(Equal("sha256:abcd")) + Expect(got.ctx).To(Equal(uintptr(9))) + Expect(got.n).To(Equal(int32(16000))) + Expect(got.rate).To(Equal(int32(16000))) + Expect(freed).To(HaveLen(1)) + Expect(freed[0]).To(Equal(uintptr(unsafe.Pointer(&keep[0][0])))) + }) + + It("fails with FailedPrecondition when the model has no speaker encoder", func() { + _, err := (&ParakeetCpp{}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)}) + Expect(status.Code(err)).To(Equal(codes.FailedPrecondition)) + Expect(err.Error()).To(ContainSubstring("speaker")) + }) + + It("fails with Unimplemented on a library without the embed symbols", func() { + CppSpeakerEmbedPCM, CppFreeFloats = nil, nil + _, err := (&ParakeetCpp{spkCtx: 9}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)}) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + Expect(err.Error()).To(ContainSubstring("parakeet_capi_speaker_embed_pcm")) + }) + + It("rejects a missing path, an undecodable file and empty audio", func() { + p := &ParakeetCpp{spkCtx: 9} + _, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{}) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + + bad := filepath.Join(GinkgoT().TempDir(), "bad.wav") + Expect(os.WriteFile(bad, []byte("not audio"), 0o600)).To(Succeed()) + _, err = p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: bad}) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + + empty := filepath.Join(GinkgoT().TempDir(), "empty.wav") + writeMono16kWav(empty, 0) + _, err = p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: empty}) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + Expect(got.n).To(BeZero(), "the encoder must not run on invalid audio") + }) + + It("returns the library error when the encoder fails", func() { + rc = 1 + _, err := (&ParakeetCpp{spkCtx: 9}).VoiceEmbed(&pb.VoiceEmbedRequest{Audio: diarizeWav(1)}) + Expect(status.Code(err)).To(Equal(codes.Internal)) + Expect(err.Error()).To(ContainSubstring("stub failure")) + }) + + Describe("VoiceVerify", func() { + It("accepts the same voice and rejects a different one", func() { + p := &ParakeetCpp{spkCtx: 9} + res, err := p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: diarizeWav(1), Audio2: diarizeWav(1)}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Verified).To(BeTrue()) + Expect(res.Distance).To(BeNumerically("~", 0, 1e-6)) + Expect(res.Threshold).To(BeNumerically("~", 0.5, 1e-6)) + Expect(res.Confidence).To(BeNumerically("~", 100, 1e-3)) + + vectors := [][]float32{{1, 0, 0}, {0, 1, 0}} + calls := 0 + CppSpeakerEmbedPCM = func(_ uintptr, _ *float32, _, _ int32, out, dim unsafe.Pointer) int32 { + v := vectors[calls%2] + calls++ + keep = append(keep, v) + *(*uintptr)(out) = uintptr(unsafe.Pointer(&v[0])) + *(*int32)(dim) = 3 + return 0 + } + res, err = p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: diarizeWav(1), Audio2: diarizeWav(1), Threshold: 0.3}) + Expect(err).ToNot(HaveOccurred()) + Expect(res.Verified).To(BeFalse()) + Expect(res.Distance).To(BeNumerically("~", 1, 1e-6)) + Expect(res.Threshold).To(BeNumerically("~", 0.3, 1e-6)) + }) + + It("refuses anti-spoofing and missing clips", func() { + p := &ParakeetCpp{spkCtx: 9} + _, err := p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: "a", Audio2: "b", AntiSpoofing: true}) + Expect(status.Code(err)).To(Equal(codes.Unimplemented)) + _, err = p.VoiceVerify(&pb.VoiceVerifyRequest{Audio1: "a"}) + Expect(status.Code(err)).To(Equal(codes.InvalidArgument)) + }) + }) +}) + +// PARAKEET_BACKEND_TEST_BUNDLE (or ..._SPEAKER_MODEL) points at a GGUF with a speaker +// encoder and PARAKEET_BACKEND_TEST_WAV at any speech WAV; the spec needs a libparakeet.so +// that exports parakeet_capi_speaker_embed_pcm (PARAKEET_LIBRARY). +var _ = Describe("ParakeetCpp.VoiceEmbed (real libparakeet.so)", func() { + It("embeds a clip, repeats exactly and reports the encoder identity", func() { + model := os.Getenv("PARAKEET_BACKEND_TEST_SPEAKER_BUNDLE") + wav := os.Getenv("PARAKEET_BACKEND_TEST_WAV") + if model == "" || wav == "" { + Skip("set PARAKEET_BACKEND_TEST_SPEAKER_BUNDLE (a bundle GGUF with a voice component) and PARAKEET_BACKEND_TEST_WAV") + } + ensureLibLoaded() + if CppSpeakerEmbedPCM == nil { + Skip("libparakeet.so has no parakeet_capi_speaker_embed_pcm") + } + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: model, Options: []string{"speaker_component:voice"}})).To(Succeed()) + defer func() { _ = p.Free() }() + + first, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: wav}) + Expect(err).ToNot(HaveOccurred()) + Expect(first.Embedding).ToNot(BeEmpty()) + Expect(first.Model).To(HavePrefix("sha256:")) + again, err := p.VoiceEmbed(&pb.VoiceEmbedRequest{Audio: wav}) + Expect(err).ToNot(HaveOccurred()) + Expect(cosineDistance(first.Embedding, again.Embedding)).To(BeNumerically("<", 1e-4)) + }) +})