LocalAI/backend/go/parakeet-cpp/voice_embed.go

// SPDX-License-Identifier: MIT
package main

import (
	"fmt"
	"math"
	"strconv"
	"strings"
	"unsafe"

	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

// speakerEmbedRate is the sample rate the speaker encoder takes; decodeWavMono16k
// already resamples to it.
const speakerEmbedRate = 16000

// speakerEmbedReady reports why the loaded model cannot embed a voice, or nil.
// An older libparakeet.so without parakeet_capi_speaker_embed_pcm gets Unimplemented;
// a model without a speaker encoder (no speaker_component/speaker_model) gets
// FailedPrecondition. Caller holds engineMu.
func (p *ParakeetCpp) speakerEmbedReady() error {
	if CppSpeakerEmbedPCM == nil || CppFreeFloats == nil {
		return status.Error(codes.Unimplemented,
			"parakeet-cpp: voice embedding needs a libparakeet.so with parakeet_capi_speaker_embed_pcm; rebuild the backend against a newer parakeet.cpp")
	}
	if p.spkCtx == 0 {
		return status.Error(codes.FailedPrecondition,
			"parakeet-cpp: no speaker encoder loaded; use a bundle with a voice component (speaker_component) or set speaker_model")
	}
	return nil
}

// embedPCMLocked embeds mono 16 kHz samples and copies the vector out of the C buffer.
// Caller holds engineMu.
func (p *ParakeetCpp) embedPCMLocked(pcm []float32) ([]float32, error) {
	var vec uintptr
	var dim int32
	rc := CppSpeakerEmbedPCM(p.spkCtx, &pcm[0], int32(len(pcm)), speakerEmbedRate, unsafe.Pointer(&vec), unsafe.Pointer(&dim))
	if rc != 0 || vec == 0 || dim <= 0 {
		if vec != 0 {
			CppFreeFloats(vec)
		}
		return nil, status.Errorf(codes.Internal, "parakeet-cpp: speaker embedding failed: %s", CppLastError(p.spkCtx))
	}
	defer CppFreeFloats(vec)
	src := unsafe.Slice((*float32)(unsafe.Pointer(vec)), int(dim)) //nolint:govet // C-owned malloc'd vector, copied out before free
	out := make([]float32, int(dim))
	copy(out, src)
	return out, nil
}

// embedFile decodes an audio file to 16 kHz mono and embeds it under engineMu.
func (p *ParakeetCpp) embedFile(path string) ([]float32, error) {
	if path == "" {
		return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: audio path is required")
	}
	pcm, _, err := decodeWavMono16k(path)
	if err != nil {
		return nil, status.Errorf(codes.InvalidArgument, "parakeet-cpp: decode audio: %s", err)
	}
	if len(pcm) == 0 {
		return nil, status.Error(codes.InvalidArgument, "parakeet-cpp: empty audio")
	}
	p.engineMu.Lock()
	defer p.engineMu.Unlock()
	if err := p.speakerEmbedReady(); err != nil {
		return nil, err
	}
	return p.embedPCMLocked(pcm)
}

// speakerIdentity is the "sha256:" identity of the loaded speaker encoder weights, "" when
// the library cannot report one. Caller holds engineMu.
func (p *ParakeetCpp) speakerIdentityLocked() string {
	if p.spkCtx == 0 || CppSpeakerIdentity == nil {
		return ""
	}
	return goStringFromCPtr(CppSpeakerIdentity(p.spkCtx))
}

// VoiceEmbed returns the speaker embedding of the audio file in req.Audio. The embedding
// space is the one of the loaded speaker encoder, so a bundle's voice component gives the
// same vectors as the standalone model with the same weights.
func (p *ParakeetCpp) VoiceEmbed(req *pb.VoiceEmbedRequest) (pb.VoiceEmbedResponse, error) {
	emb, err := p.embedFile(req.GetAudio())
	if err != nil {
		return pb.VoiceEmbedResponse{}, err
	}
	p.engineMu.Lock()
	model := p.speakerIdentityLocked()
	p.engineMu.Unlock()
	return pb.VoiceEmbedResponse{Embedding: emb, Model: model}, nil
}

// VoiceVerify embeds both clips and compares them by cosine distance. A request threshold
// of 0 or less uses the default speaker distance. There is no anti-spoofing head, so a
// request for it is refused rather than answered without the check.
func (p *ParakeetCpp) VoiceVerify(req *pb.VoiceVerifyRequest) (pb.VoiceVerifyResponse, error) {
	if req.GetAntiSpoofing() {
		return pb.VoiceVerifyResponse{}, status.Error(codes.Unimplemented, "parakeet-cpp: anti-spoofing is not supported")
	}
	if req.GetAudio1() == "" || req.GetAudio2() == "" {
		return pb.VoiceVerifyResponse{}, status.Error(codes.InvalidArgument, "parakeet-cpp: audio1 and audio2 are required")
	}
	a, err := p.embedFile(req.GetAudio1())
	if err != nil {
		return pb.VoiceVerifyResponse{}, err
	}
	b, err := p.embedFile(req.GetAudio2())
	if err != nil {
		return pb.VoiceVerifyResponse{}, err
	}
	if len(a) != len(b) {
		return pb.VoiceVerifyResponse{}, status.Errorf(codes.Internal, "parakeet-cpp: embedding sizes differ (%d and %d)", len(a), len(b))
	}
	threshold := req.GetThreshold()
	if threshold <= 0 {
		threshold = p.verifyDistance
	}
	if threshold <= 0 {
		threshold = defaultSpeakerDistance
	}
	distance := cosineDistance(a, b)
	confidence := float32(math.Max(0, math.Min(100, (1-float64(distance)/float64(threshold))*100)))
	p.engineMu.Lock()
	model := p.speakerIdentityLocked()
	p.engineMu.Unlock()
	return pb.VoiceVerifyResponse{
		Verified:   distance <= threshold,
		Distance:   distance,
		Threshold:  threshold,
		Confidence: confidence,
		Model:      model,
	}, nil
}

// parseVerifyThreshold reads voice_verify_threshold, a distance in (0, 2). Empty means 0,
// which VoiceVerify turns into the default.
func parseVerifyThreshold(s string) (float32, error) {
	if strings.TrimSpace(s) == "" {
		return 0, nil
	}
	v, err := strconv.ParseFloat(strings.TrimSpace(s), 64)
	if err != nil || math.IsNaN(v) || v <= 0 || v >= 2 {
		return 0, fmt.Errorf("parakeet-cpp: voice_verify_threshold %q must be a distance in (0, 2) (1 minus cosine similarity)", s)
	}
	return float32(v), nil
}

// cosineDistance is 1 minus the cosine similarity; a zero vector counts as maximally far.
func cosineDistance(a, b []float32) float32 {
	var dot, na, nb float64
	for i := range a {
		dot += float64(a[i]) * float64(b[i])
		na += float64(a[i]) * float64(a[i])
		nb += float64(b[i]) * float64(b[i])
	}
	if na == 0 || nb == 0 {
		return 1
	}
	return float32(1 - dot/(math.Sqrt(na)*math.Sqrt(nb)))
}