diff --git a/core/http/endpoints/openai/realtime.go b/core/http/endpoints/openai/realtime.go index 8b17a8238..b387805cf 100644 --- a/core/http/endpoints/openai/realtime.go +++ b/core/http/endpoints/openai/realtime.go @@ -634,6 +634,17 @@ func runRealtimeSession(application *application.Application, t Transport, model sendError(t, "model_load_error", "Failed to load model", "", "") return } + if wrapped, ok := m.(*wrappedModel); ok { + resolvedVoice, params, release, resolveErr := resolveRealtimeVoice(context.Background(), session.Voice, wrapped.TTSConfig, application.VoiceProfileStore()) + if resolveErr != nil { + xlog.Error("failed to resolve realtime voice", "error", resolveErr) + sendError(t, "voice_profile_error", resolveErr.Error(), "", "") + return + } + defer release() + session.Voice = resolvedVoice + wrapped.ttsParams = params + } session.ModelInterface = m // A pipeline-seeded option list gets its scoring prompt prewarmed // alongside the model warm-up below, so the session's first turn diff --git a/core/http/endpoints/openai/realtime_model.go b/core/http/endpoints/openai/realtime_model.go index 68dd61997..33ca2c762 100644 --- a/core/http/endpoints/openai/realtime_model.go +++ b/core/http/endpoints/openai/realtime_model.go @@ -6,7 +6,9 @@ import ( "encoding/binary" "encoding/hex" "encoding/json" + "errors" "fmt" + "maps" "strings" "sync" "time" @@ -18,6 +20,7 @@ import ( "github.com/mudler/LocalAI/core/http/middleware" "github.com/mudler/LocalAI/core/schema" "github.com/mudler/LocalAI/core/services/routing/router" + "github.com/mudler/LocalAI/core/services/voiceprofile" "github.com/mudler/LocalAI/core/templates" "github.com/mudler/LocalAI/pkg/functions" "github.com/mudler/LocalAI/pkg/grpc/proto" @@ -35,6 +38,7 @@ var ( // which are for Any-To-Any models, but instead we will call a pipeline (for e.g STT->LLM->TTS) type wrappedModel struct { TTSConfig *config.ModelConfig + ttsParams map[string]string TranscriptionConfig *config.ModelConfig LLMConfig *config.ModelConfig VADConfig *config.ModelConfig @@ -391,11 +395,35 @@ func newRealtimeDecisionID() string { } func (m *wrappedModel) TTS(ctx context.Context, text, voice, language string) (string, *proto.Result, error) { - return backend.ModelTTS(ctx, text, voice, language, "", nil, m.modelLoader, m.appConfig, *m.TTSConfig) + return backend.ModelTTS(ctx, text, voice, language, "", maps.Clone(m.ttsParams), m.modelLoader, m.appConfig, *m.TTSConfig) } func (m *wrappedModel) TTSStream(ctx context.Context, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error { - return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, onAudio) + return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, maps.Clone(m.ttsParams), onAudio) +} + +func resolveRealtimeVoice(ctx context.Context, configuredVoice string, ttsConfig *config.ModelConfig, profiles *voiceprofile.Store) (string, map[string]string, func(), error) { + if !voiceprofile.IsReference(configuredVoice) { + return configuredVoice, nil, func() {}, nil + } + profileID, valid := voiceprofile.ParseReference(configuredVoice) + if !valid { + return "", nil, nil, fmt.Errorf("invalid voice profile reference %q", configuredVoice) + } + if config.VoiceCloningForModel(ttsConfig) == nil { + return "", nil, nil, fmt.Errorf("selected TTS model does not support reference-audio voice cloning") + } + if profiles == nil { + return "", nil, nil, fmt.Errorf("voice profile store is unavailable") + } + profile, referencePath, release, err := profiles.LeaseAudio(ctx, profileID) + if err != nil { + if errors.Is(err, voiceprofile.ErrNotFound) { + return "", nil, nil, fmt.Errorf("voice profile not found: %w", err) + } + return "", nil, nil, fmt.Errorf("resolve voice profile: %w", err) + } + return referencePath, map[string]string{"ref_text": profile.Transcript}, release, nil } func (m *wrappedModel) TranscribeStream(ctx context.Context, audio, language string, translate, diarize bool, prompt string, onDelta func(text string)) (*schema.TranscriptionResult, error) { @@ -674,11 +702,11 @@ const wavStreamHeaderBytes = 44 // callback, which wants raw PCM plus the sample rate. The header is buffered // until complete, the sample rate is read from it, and subsequent bytes are // forwarded as PCM. -func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error { +func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, params map[string]string, onAudio func(pcm []byte, sampleRate int) error) error { var header []byte headerDone := false sampleRate := 0 - return backend.ModelTTSStream(ctx, text, voice, language, "", nil, ml, appConfig, ttsConfig, func(b []byte) error { + return backend.ModelTTSStream(ctx, text, voice, language, "", params, ml, appConfig, ttsConfig, func(b []byte) error { if headerDone { if len(b) == 0 { return nil diff --git a/core/http/endpoints/openai/realtime_voice_profile_test.go b/core/http/endpoints/openai/realtime_voice_profile_test.go new file mode 100644 index 000000000..801d37bbc --- /dev/null +++ b/core/http/endpoints/openai/realtime_voice_profile_test.go @@ -0,0 +1,180 @@ +package openai + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "time" + + grpcPkg "github.com/mudler/LocalAI/pkg/grpc" + "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + "google.golang.org/grpc" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/services/voiceprofile" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func realtimeProfileWAV(duration time.Duration) []byte { + const ( + sampleRate = 16000 + channels = 1 + bitsPerSample = 16 + ) + dataSize := int(duration.Seconds() * sampleRate * channels * bitsPerSample / 8) + buf := bytes.NewBuffer(nil) + buf.WriteString("RIFF") + _ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize)) + buf.WriteString("WAVEfmt ") + _ = binary.Write(buf, binary.LittleEndian, uint32(16)) + _ = binary.Write(buf, binary.LittleEndian, uint16(1)) + _ = binary.Write(buf, binary.LittleEndian, uint16(channels)) + _ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate)) + _ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*channels*bitsPerSample/8)) + _ = binary.Write(buf, binary.LittleEndian, uint16(channels*bitsPerSample/8)) + _ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample)) + buf.WriteString("data") + _ = binary.Write(buf, binary.LittleEndian, uint32(dataSize)) + buf.Write(make([]byte, dataSize)) + return buf.Bytes() +} + +var _ = Describe("realtime pipeline voice profiles", func() { + It("resolves a saved profile to an immutable lease and transcript", func(ctx SpecContext) { + store := voiceprofile.NewStore(GinkgoT().TempDir()) + DeferCleanup(func() { Expect(store.Close()).To(Succeed()) }) + profile, err := store.Create(ctx, voiceprofile.CreateInput{ + Name: "Narrator", + Language: "en-US", + Transcript: "The reference transcript.", + ConsentConfirmed: true, + }, bytes.NewReader(realtimeProfileWAV(time.Second))) + Expect(err).NotTo(HaveOccurred()) + + voice, params, release, err := resolveRealtimeVoice(ctx, profile.Voice, &config.ModelConfig{ + Name: "clone-base", + Backend: "qwen3-tts-cpp", + TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)}, + }, store) + + Expect(err).NotTo(HaveOccurred()) + Expect(voice).To(BeAnExistingFile()) + Expect(params).To(Equal(map[string]string{"ref_text": "The reference transcript."})) + release() + release() + Expect(voice).NotTo(BeAnExistingFile()) + }) + + It("leaves an ordinary backend voice unchanged with no parameters", func() { + voice, params, release, err := resolveRealtimeVoice(context.Background(), "speaker-7", &config.ModelConfig{}, nil) + + Expect(err).NotTo(HaveOccurred()) + Expect(voice).To(Equal("speaker-7")) + Expect(params).To(BeNil()) + Expect(release).NotTo(BeNil()) + Expect(func() { release(); release() }).NotTo(Panic()) + }) + + DescribeTable("returns actionable reference errors", + func(configuredVoice string, cfg *config.ModelConfig, store *voiceprofile.Store, expected string) { + _, _, release, err := resolveRealtimeVoice(context.Background(), configuredVoice, cfg, store) + Expect(err).To(MatchError(ContainSubstring(expected))) + Expect(release).To(BeNil()) + }, + Entry("malformed reference", "localai://voice-profiles/not-a-uuid", &config.ModelConfig{}, nil, "invalid voice profile reference"), + Entry("unsupported model", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Backend: "piper"}, nil, "does not support reference-audio voice cloning"), + Entry("unavailable store", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)}}, nil, "voice profile store is unavailable"), + ) + + It("reports a missing profile", func() { + store := voiceprofile.NewStore(GinkgoT().TempDir()) + DeferCleanup(func() { Expect(store.Close()).To(Succeed()) }) + _, _, release, err := resolveRealtimeVoice(context.Background(), "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{ + Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)}, + }, store) + Expect(errors.Is(err, voiceprofile.ErrNotFound)).To(BeTrue()) + Expect(err.Error()).To(ContainSubstring("voice profile not found")) + Expect(release).To(BeNil()) + }) +}) + +type recordingTTSBackend struct { + grpcPkg.Backend + requests []*proto.TTSRequest +} + +func (b *recordingTTSBackend) HealthCheck(context.Context) (bool, error) { return true, nil } +func (b *recordingTTSBackend) IsBusy() bool { return false } + +func (b *recordingTTSBackend) record(req *proto.TTSRequest) { + b.requests = append(b.requests, req) + req.Params["ref_text"] = "backend mutation" +} + +func (b *recordingTTSBackend) TTS(_ context.Context, req *proto.TTSRequest, _ ...grpc.CallOption) (*proto.Result, error) { + b.record(req) + return &proto.Result{Success: true}, nil +} + +func (b *recordingTTSBackend) TTSStream(_ context.Context, req *proto.TTSRequest, callback func(*proto.Reply), _ ...grpc.CallOption) error { + b.record(req) + header := make([]byte, wavStreamHeaderBytes) + binary.LittleEndian.PutUint32(header[24:28], 24000) + callback(&proto.Reply{Audio: header}) + return nil +} + +var _ = Describe("wrappedModel voice profile parameters", func() { + var ( + wrapped *wrappedModel + backendRecorder *recordingTTSBackend + ) + + BeforeEach(func() { + state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir())) + Expect(err).NotTo(HaveOccurred()) + appConfig := config.NewApplicationConfig(config.WithSystemState(state)) + appConfig.GeneratedContentDir = GinkgoT().TempDir() + loader := model.NewModelLoader(state) + backendRecorder = &recordingTTSBackend{} + cfg := &config.ModelConfig{Name: "tts-test", Backend: "test"} + cfg.Model = "weights" + loaded := model.NewModelWithClient(cfg.ModelID(), "in-process", backendRecorder) + loaded.MarkHealthy() + _, err = loader.LoadModel(cfg.ModelID(), cfg.Model, func(_, _, _ string) (*model.Model, error) { return loaded, nil }) + Expect(err).NotTo(HaveOccurred()) + wrapped = &wrappedModel{ + TTSConfig: cfg, + ttsParams: map[string]string{"ref_text": "Original transcript"}, + modelLoader: loader, + appConfig: appConfig, + } + }) + + It("forwards a fresh transcript parameter map to every unary request", func() { + _, _, err := wrapped.TTS(context.Background(), "one", "voice.wav", "en") + Expect(err).NotTo(HaveOccurred()) + _, _, err = wrapped.TTS(context.Background(), "two", "voice.wav", "en") + Expect(err).NotTo(HaveOccurred()) + + Expect(backendRecorder.requests).To(HaveLen(2)) + Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation")) + Expect(backendRecorder.requests[1].Params).To(HaveKeyWithValue("ref_text", "backend mutation")) + Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript")) + }) + + It("forwards a copied transcript parameter map to streaming requests", func() { + err := wrapped.TTSStream(context.Background(), "one", "voice.wav", "en", func([]byte, int) error { return nil }) + Expect(err).NotTo(HaveOccurred()) + + Expect(backendRecorder.requests).To(HaveLen(1)) + Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation")) + Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript")) + }) +}) + +func ptrTo[T any](value T) *T { return &value } diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index 8ff355a73..877bd4796 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -124,6 +124,25 @@ Reference selection follows this order: When a saved profile is selected, LocalAI supplies both its private WAV and exact transcript for that request. It does not rewrite the model YAML or copy the recording into the model directory. +### Realtime pipeline default + +Set `tts.voice` on a realtime pipeline model to use a saved Voice Library profile as the session default: + +```yaml +name: gpt-realtime +tts: + voice: localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000 +pipeline: + vad: silero-vad-ggml + transcription: whisper-large-turbo + llm: qwen3-4b + tts: qwen3-tts-base +``` + +LocalAI resolves this profile when the realtime session starts. The selected TTS model must support Voice Library cloning. + +This feature does not resolve Voice Library URIs sent later through realtime `session.update`. You can still use `session.update` with ordinary backend voice names or IDs. + #### Supported backend and model variants | Backend | Automatically compatible variants |