fix(realtime): resolve pipeline voice profiles (#11942)

* fix(realtime): resolve saved voice profiles

Realtime pipelines now validate saved voices against the selected TTS model and retain leased audio until session teardown. Each synthesis request receives its own transcript parameter map.

Assisted-by: Codex:GPT-5

* docs(tts): document realtime voice defaults

Show how a realtime pipeline selects a saved Voice Library profile at session start. Clarify which session voice updates remain supported.

Assisted-by: Codex:GPT-5

---------

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

core/http/endpoints/openai/realtime.go

@@ -634,6 +634,17 @@ func runRealtimeSession(application *application.Application, t Transport, model
 		sendError(t, "model_load_error", "Failed to load model", "", "")
 		return
 	}
+	if wrapped, ok := m.(*wrappedModel); ok {
+		resolvedVoice, params, release, resolveErr := resolveRealtimeVoice(context.Background(), session.Voice, wrapped.TTSConfig, application.VoiceProfileStore())
+		if resolveErr != nil {
+			xlog.Error("failed to resolve realtime voice", "error", resolveErr)
+			sendError(t, "voice_profile_error", resolveErr.Error(), "", "")
+			return
+		}
+		defer release()
+		session.Voice = resolvedVoice
+		wrapped.ttsParams = params
+	}
 	session.ModelInterface = m
 	// A pipeline-seeded option list gets its scoring prompt prewarmed
 	// alongside the model warm-up below, so the session's first turn

core/http/endpoints/openai/realtime_model.go

@@ -6,7 +6,9 @@ import (
 	"encoding/binary"
 	"encoding/hex"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"maps"
 	"strings"
 	"sync"
 	"time"
@@ -18,6 +20,7 @@ import (
 	"github.com/mudler/LocalAI/core/http/middleware"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/services/routing/router"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
 	"github.com/mudler/LocalAI/core/templates"
 	"github.com/mudler/LocalAI/pkg/functions"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
@@ -35,6 +38,7 @@ var (
 // which are for Any-To-Any models, but instead we will call a pipeline (for e.g STT->LLM->TTS)
 type wrappedModel struct {
 	TTSConfig            *config.ModelConfig
+	ttsParams            map[string]string
 	TranscriptionConfig  *config.ModelConfig
 	LLMConfig            *config.ModelConfig
 	VADConfig            *config.ModelConfig
@@ -391,11 +395,35 @@ func newRealtimeDecisionID() string {
 }
 
 func (m *wrappedModel) TTS(ctx context.Context, text, voice, language string) (string, *proto.Result, error) {
-	return backend.ModelTTS(ctx, text, voice, language, "", nil, m.modelLoader, m.appConfig, *m.TTSConfig)
+	return backend.ModelTTS(ctx, text, voice, language, "", maps.Clone(m.ttsParams), m.modelLoader, m.appConfig, *m.TTSConfig)
 }
 
 func (m *wrappedModel) TTSStream(ctx context.Context, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error {
-	return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, onAudio)
+	return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, maps.Clone(m.ttsParams), onAudio)
+}
+
+func resolveRealtimeVoice(ctx context.Context, configuredVoice string, ttsConfig *config.ModelConfig, profiles *voiceprofile.Store) (string, map[string]string, func(), error) {
+	if !voiceprofile.IsReference(configuredVoice) {
+		return configuredVoice, nil, func() {}, nil
+	}
+	profileID, valid := voiceprofile.ParseReference(configuredVoice)
+	if !valid {
+		return "", nil, nil, fmt.Errorf("invalid voice profile reference %q", configuredVoice)
+	}
+	if config.VoiceCloningForModel(ttsConfig) == nil {
+		return "", nil, nil, fmt.Errorf("selected TTS model does not support reference-audio voice cloning")
+	}
+	if profiles == nil {
+		return "", nil, nil, fmt.Errorf("voice profile store is unavailable")
+	}
+	profile, referencePath, release, err := profiles.LeaseAudio(ctx, profileID)
+	if err != nil {
+		if errors.Is(err, voiceprofile.ErrNotFound) {
+			return "", nil, nil, fmt.Errorf("voice profile not found: %w", err)
+		}
+		return "", nil, nil, fmt.Errorf("resolve voice profile: %w", err)
+	}
+	return referencePath, map[string]string{"ref_text": profile.Transcript}, release, nil
 }
 
 func (m *wrappedModel) TranscribeStream(ctx context.Context, audio, language string, translate, diarize bool, prompt string, onDelta func(text string)) (*schema.TranscriptionResult, error) {
@@ -674,11 +702,11 @@ const wavStreamHeaderBytes = 44
 // callback, which wants raw PCM plus the sample rate. The header is buffered
 // until complete, the sample rate is read from it, and subsequent bytes are
 // forwarded as PCM.
-func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error {
+func ttsStream(ctx context.Context, ml *model.ModelLoader, appConfig *config.ApplicationConfig, ttsConfig config.ModelConfig, text, voice, language string, params map[string]string, onAudio func(pcm []byte, sampleRate int) error) error {
 	var header []byte
 	headerDone := false
 	sampleRate := 0
-	return backend.ModelTTSStream(ctx, text, voice, language, "", nil, ml, appConfig, ttsConfig, func(b []byte) error {
+	return backend.ModelTTSStream(ctx, text, voice, language, "", params, ml, appConfig, ttsConfig, func(b []byte) error {
 		if headerDone {
 			if len(b) == 0 {
 				return nil

core/http/endpoints/openai/realtime_voice_profile_test.go

@@ -0,0 +1,180 @@
+package openai
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"time"
+
+	grpcPkg "github.com/mudler/LocalAI/pkg/grpc"
+	"github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/model"
+	"github.com/mudler/LocalAI/pkg/system"
+	"google.golang.org/grpc"
+
+	"github.com/mudler/LocalAI/core/config"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func realtimeProfileWAV(duration time.Duration) []byte {
+	const (
+		sampleRate    = 16000
+		channels      = 1
+		bitsPerSample = 16
+	)
+	dataSize := int(duration.Seconds() * sampleRate * channels * bitsPerSample / 8)
+	buf := bytes.NewBuffer(nil)
+	buf.WriteString("RIFF")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
+	buf.WriteString("WAVEfmt ")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(16))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(1))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(channels))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
+	_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*channels*bitsPerSample/8))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(channels*bitsPerSample/8))
+	_ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample))
+	buf.WriteString("data")
+	_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
+	buf.Write(make([]byte, dataSize))
+	return buf.Bytes()
+}
+
+var _ = Describe("realtime pipeline voice profiles", func() {
+	It("resolves a saved profile to an immutable lease and transcript", func(ctx SpecContext) {
+		store := voiceprofile.NewStore(GinkgoT().TempDir())
+		DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
+		profile, err := store.Create(ctx, voiceprofile.CreateInput{
+			Name:             "Narrator",
+			Language:         "en-US",
+			Transcript:       "The reference transcript.",
+			ConsentConfirmed: true,
+		}, bytes.NewReader(realtimeProfileWAV(time.Second)))
+		Expect(err).NotTo(HaveOccurred())
+
+		voice, params, release, err := resolveRealtimeVoice(ctx, profile.Voice, &config.ModelConfig{
+			Name:      "clone-base",
+			Backend:   "qwen3-tts-cpp",
+			TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)},
+		}, store)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(voice).To(BeAnExistingFile())
+		Expect(params).To(Equal(map[string]string{"ref_text": "The reference transcript."}))
+		release()
+		release()
+		Expect(voice).NotTo(BeAnExistingFile())
+	})
+
+	It("leaves an ordinary backend voice unchanged with no parameters", func() {
+		voice, params, release, err := resolveRealtimeVoice(context.Background(), "speaker-7", &config.ModelConfig{}, nil)
+
+		Expect(err).NotTo(HaveOccurred())
+		Expect(voice).To(Equal("speaker-7"))
+		Expect(params).To(BeNil())
+		Expect(release).NotTo(BeNil())
+		Expect(func() { release(); release() }).NotTo(Panic())
+	})
+
+	DescribeTable("returns actionable reference errors",
+		func(configuredVoice string, cfg *config.ModelConfig, store *voiceprofile.Store, expected string) {
+			_, _, release, err := resolveRealtimeVoice(context.Background(), configuredVoice, cfg, store)
+			Expect(err).To(MatchError(ContainSubstring(expected)))
+			Expect(release).To(BeNil())
+		},
+		Entry("malformed reference", "localai://voice-profiles/not-a-uuid", &config.ModelConfig{}, nil, "invalid voice profile reference"),
+		Entry("unsupported model", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Backend: "piper"}, nil, "does not support reference-audio voice cloning"),
+		Entry("unavailable store", "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)}}, nil, "voice profile store is unavailable"),
+	)
+
+	It("reports a missing profile", func() {
+		store := voiceprofile.NewStore(GinkgoT().TempDir())
+		DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
+		_, _, release, err := resolveRealtimeVoice(context.Background(), "localai://voice-profiles/00000000-0000-0000-0000-000000000001", &config.ModelConfig{
+			Name: "clone-base", Backend: "qwen3-tts-cpp", TTSConfig: config.TTSConfig{VoiceCloning: ptrTo(true)},
+		}, store)
+		Expect(errors.Is(err, voiceprofile.ErrNotFound)).To(BeTrue())
+		Expect(err.Error()).To(ContainSubstring("voice profile not found"))
+		Expect(release).To(BeNil())
+	})
+})
+
+type recordingTTSBackend struct {
+	grpcPkg.Backend
+	requests []*proto.TTSRequest
+}
+
+func (b *recordingTTSBackend) HealthCheck(context.Context) (bool, error) { return true, nil }
+func (b *recordingTTSBackend) IsBusy() bool                              { return false }
+
+func (b *recordingTTSBackend) record(req *proto.TTSRequest) {
+	b.requests = append(b.requests, req)
+	req.Params["ref_text"] = "backend mutation"
+}
+
+func (b *recordingTTSBackend) TTS(_ context.Context, req *proto.TTSRequest, _ ...grpc.CallOption) (*proto.Result, error) {
+	b.record(req)
+	return &proto.Result{Success: true}, nil
+}
+
+func (b *recordingTTSBackend) TTSStream(_ context.Context, req *proto.TTSRequest, callback func(*proto.Reply), _ ...grpc.CallOption) error {
+	b.record(req)
+	header := make([]byte, wavStreamHeaderBytes)
+	binary.LittleEndian.PutUint32(header[24:28], 24000)
+	callback(&proto.Reply{Audio: header})
+	return nil
+}
+
+var _ = Describe("wrappedModel voice profile parameters", func() {
+	var (
+		wrapped         *wrappedModel
+		backendRecorder *recordingTTSBackend
+	)
+
+	BeforeEach(func() {
+		state, err := system.GetSystemState(system.WithModelPath(GinkgoT().TempDir()))
+		Expect(err).NotTo(HaveOccurred())
+		appConfig := config.NewApplicationConfig(config.WithSystemState(state))
+		appConfig.GeneratedContentDir = GinkgoT().TempDir()
+		loader := model.NewModelLoader(state)
+		backendRecorder = &recordingTTSBackend{}
+		cfg := &config.ModelConfig{Name: "tts-test", Backend: "test"}
+		cfg.Model = "weights"
+		loaded := model.NewModelWithClient(cfg.ModelID(), "in-process", backendRecorder)
+		loaded.MarkHealthy()
+		_, err = loader.LoadModel(cfg.ModelID(), cfg.Model, func(_, _, _ string) (*model.Model, error) { return loaded, nil })
+		Expect(err).NotTo(HaveOccurred())
+		wrapped = &wrappedModel{
+			TTSConfig:   cfg,
+			ttsParams:   map[string]string{"ref_text": "Original transcript"},
+			modelLoader: loader,
+			appConfig:   appConfig,
+		}
+	})
+
+	It("forwards a fresh transcript parameter map to every unary request", func() {
+		_, _, err := wrapped.TTS(context.Background(), "one", "voice.wav", "en")
+		Expect(err).NotTo(HaveOccurred())
+		_, _, err = wrapped.TTS(context.Background(), "two", "voice.wav", "en")
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(backendRecorder.requests).To(HaveLen(2))
+		Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(backendRecorder.requests[1].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript"))
+	})
+
+	It("forwards a copied transcript parameter map to streaming requests", func() {
+		err := wrapped.TTSStream(context.Background(), "one", "voice.wav", "en", func([]byte, int) error { return nil })
+		Expect(err).NotTo(HaveOccurred())
+
+		Expect(backendRecorder.requests).To(HaveLen(1))
+		Expect(backendRecorder.requests[0].Params).To(HaveKeyWithValue("ref_text", "backend mutation"))
+		Expect(wrapped.ttsParams).To(HaveKeyWithValue("ref_text", "Original transcript"))
+	})
+})
+
+func ptrTo[T any](value T) *T { return &value }

docs/content/features/text-to-audio.md

@@ -124,6 +124,25 @@ Reference selection follows this order:
 
 When a saved profile is selected, LocalAI supplies both its private WAV and exact transcript for that request. It does not rewrite the model YAML or copy the recording into the model directory.
 
+### Realtime pipeline default
+
+Set `tts.voice` on a realtime pipeline model to use a saved Voice Library profile as the session default:
+
+```yaml
+name: gpt-realtime
+tts:
+  voice: localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000
+pipeline:
+  vad: silero-vad-ggml
+  transcription: whisper-large-turbo
+  llm: qwen3-4b
+  tts: qwen3-tts-base
+```
+
+LocalAI resolves this profile when the realtime session starts. The selected TTS model must support Voice Library cloning.
+
+This feature does not resolve Voice Library URIs sent later through realtime `session.update`. You can still use `session.update` with ordinary backend voice names or IDs.
+
 #### Supported backend and model variants
 
 | Backend | Automatically compatible variants |