chore: ⬆️ Update ggml-org/llama.cpp to f3f1a8f2760f28325a5ec20c05b171e5b7c83a29 (#11934)

⬆️ Update ggml-org/llama.cpp

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>

backend/cpp/audio-cpp/Makefile

@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.
 
-AUDIO_CPP_VERSION?=9c6a282337cc83f227cc10428867a478947706ad
+AUDIO_CPP_VERSION?=05e508a70e3600b01454c647cdb122133ba8e64c
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ds4/Makefile

@@ -1,10 +1,10 @@
 # ds4 backend Makefile.
 #
-# Upstream pin lives below as DS4_VERSION?=f62ca29a308724cde5bc99134ede19104b2a3260
+# Upstream pin lives below as DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804
 # (.github/bump_deps.sh) can find and update it - matches the
 # llama-cpp / ik-llama-cpp / turboquant convention.
 
-DS4_VERSION?=f62ca29a308724cde5bc99134ede19104b2a3260
+DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804
 DS4_REPO?=https://github.com/antirez/ds4
 
 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

backend/cpp/ik-llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-IK_LLAMA_VERSION?=fe215a8ccdce6b844d2a3a3bbde08ae76a6284bf
+IK_LLAMA_VERSION?=1a2a8604a6c6c6413c06bf9adfc2f64329af4366
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=67672dc5b76f8bc17785a19d3dc6d1463fc2902c
+LLAMA_VERSION?=f3f1a8f2760f28325a5ec20c05b171e5b7c83a29
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/go/nemo-speech-cpp/Makefile

@@ -12,7 +12,7 @@
 # runs 'make -C backend/go/$(BACKEND) build' and then copies package/), so it
 # has to produce the binary and the package, not just the shared libraries.
 
-NEMO_SPEECH_VERSION?=ffa38cb2408f1e832a36d46fef5e3e1e80d07e6c
+NEMO_SPEECH_VERSION?=a5b6953c4a579a2bbd1c0913ad8a85c2a4d99953
 NEMO_SPEECH_REPO?=https://github.com/NVIDIA/NeMo-Speech.cpp
 
 GOCMD?=go

backend/go/whisper/Makefile

@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
 
 # whisper.cpp version
 WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
-WHISPER_CPP_VERSION?=52a939a2a762224e255d366c1182b2af4dd1a032
+WHISPER_CPP_VERSION?=c44b60b8053bbf2a5c1e014f11323fb3f2485177
 SO_TARGET?=libgowhisper.so
 
 CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF

core/http/endpoints/openai/realtime.go

@@ -9,6 +9,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"maps"
 	"math"
 	"os"
 	"strconv"
@@ -32,6 +33,7 @@ import (
 	"github.com/mudler/LocalAI/core/http/endpoints/openai/types"
 	"github.com/mudler/LocalAI/core/schema"
 	"github.com/mudler/LocalAI/core/services/routing/router"
+	"github.com/mudler/LocalAI/core/services/voiceprofile"
 	"github.com/mudler/LocalAI/core/templates"
 	laudio "github.com/mudler/LocalAI/pkg/audio"
 	"github.com/mudler/LocalAI/pkg/functions"
@@ -136,6 +138,8 @@ type Session struct {
 	Instructions           string
 	DefaultConversationID  string
 	ModelInterface         Model
+	ttsParams              map[string]string
+	voiceRelease           func()
 	// The pipeline model config or the config for an any-to-any model
 	ModelConfig      *config.ModelConfig
 	InputSampleRate  int
@@ -199,6 +203,22 @@ type Session struct {
 	respSink *responseSink
 }
 
+func (s *Session) installVoiceBinding(voice string, params map[string]string, release func()) {
+	if release == nil {
+		release = func() {}
+	}
+	var once sync.Once
+	s.Voice = voice
+	s.ttsParams = maps.Clone(params)
+	s.voiceRelease = func() { once.Do(release) }
+}
+
+func (s *Session) releaseVoiceBinding() {
+	if s.voiceRelease != nil {
+		s.voiceRelease()
+	}
+}
+
 func (s *Session) FromClient(session *types.SessionUnion) {
 }
 
@@ -635,15 +655,15 @@ func runRealtimeSession(application *application.Application, t Transport, model
 		return
 	}
 	if wrapped, ok := m.(*wrappedModel); ok {
-		resolvedVoice, params, release, resolveErr := resolveRealtimeVoice(context.Background(), session.Voice, wrapped.TTSConfig, application.VoiceProfileStore())
+		resolvedVoice, params, release, resolveErr := resolveRealtimeVoice(context.Background(), wrapped.TTSConfig.TTSConfig.Voice, wrapped.TTSConfig, application.VoiceProfileStore())
 		if resolveErr != nil {
 			xlog.Error("failed to resolve realtime voice", "error", resolveErr)
 			sendError(t, "voice_profile_error", resolveErr.Error(), "", "")
 			return
 		}
-		defer release()
-		session.Voice = resolvedVoice
-		wrapped.ttsParams = params
+		session.installVoiceBinding(resolvedVoice, params, release)
+		defer session.releaseVoiceBinding()
+		wrapped.setTTSParams(params)
 	}
 	session.ModelInterface = m
 	// A pipeline-seeded option list gets its scoring prompt prewarmed
@@ -838,6 +858,7 @@ func runRealtimeSession(application *application.Application, t Transport, model
 					application.ApplicationConfig(),
 					evaluator,
 					buildRealtimeRoutingContext(application, session.ID),
+					application.VoiceProfileStore(),
 				); err != nil {
 					xlog.Error("failed to update session", "error", err)
 					sendError(t, "session_update_error", fmt.Sprintf("Failed to update session: %v", err), "", "")
@@ -1176,7 +1197,7 @@ func updateTransSession(session *Session, update *types.SessionUnion, cl *config
 	return nil
 }
 
-func updateSession(session *Session, update *types.SessionUnion, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, evaluator *templates.Evaluator, routing *RealtimeRoutingContext) error {
+func updateSession(session *Session, update *types.SessionUnion, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, evaluator *templates.Evaluator, routing *RealtimeRoutingContext, profiles *voiceprofile.Store) error {
 	sessionLock.Lock()
 	defer sessionLock.Unlock()
 
@@ -1184,8 +1205,13 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 		return nil
 	}
 
-	session.TranscriptionOnly = false
 	rt := update.Realtime
+	explicitVoice := rt.Audio != nil && rt.Audio.Output != nil && rt.Audio.Output.Voice != ""
+	rebuild := rt.Model != "" || explicitVoice || (rt.Audio != nil && rt.Audio.Input != nil && rt.Audio.Input.Transcription != nil)
+
+	candidateModelName := session.Model
+	candidateConfig := session.ModelConfig
+	candidateTranscription := session.InputAudioTranscription
 
 	if rt.Model != "" {
 		cfg, err := cl.LoadModelConfigFileByNameDefaultOptions(rt.Model, appConfig)
@@ -1196,40 +1222,78 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 			return fmt.Errorf("model is not a valid pipeline model: %s", rt.Model)
 		}
 
-		if session.InputAudioTranscription == nil {
-			session.InputAudioTranscription = &types.AudioTranscription{}
-		}
-		session.InputAudioTranscription.Model = cfg.Pipeline.Transcription
-		session.Voice = cfg.TTSConfig.Voice
-		session.Model = rt.Model
-		session.ModelConfig = cfg
-	}
-
-	if rt.Audio != nil && rt.Audio.Output != nil && rt.Audio.Output.Voice != "" {
-		session.Voice = string(rt.Audio.Output.Voice)
+		candidateModelName = rt.Model
+		candidateConfig = cfg
+		candidateTranscription = &types.AudioTranscription{Model: cfg.Pipeline.Transcription}
 	}
 
 	if rt.Audio != nil && rt.Audio.Input != nil && rt.Audio.Input.Transcription != nil {
-		trUpd := rt.Audio.Input.Transcription
+		trUpd := *rt.Audio.Input.Transcription
 		// A language-only update (e.g. a client forcing the STT language) carries
 		// an empty Model. Preserve the pipeline's configured transcription backend
 		// instead of blanking it — otherwise the next utterance transcribes against
 		// an empty model and the backend RPC fails with "unimplemented".
-		if trUpd.Model == "" && session.InputAudioTranscription != nil {
-			trUpd.Model = session.InputAudioTranscription.Model
+		if trUpd.Model == "" && candidateTranscription != nil {
+			trUpd.Model = candidateTranscription.Model
 		}
-		session.InputAudioTranscription = trUpd
+		candidateTranscription = &trUpd
 		if trUpd.Model != "" {
-			session.ModelConfig.Pipeline.Transcription = trUpd.Model
+			cfgCopy := *candidateConfig
+			candidateConfig = &cfgCopy
+			candidateConfig.Pipeline.Transcription = trUpd.Model
 		}
 	}
 
-	if rt.Model != "" || (rt.Audio != nil && rt.Audio.Output != nil && rt.Audio.Output.Voice != "") || (rt.Audio != nil && rt.Audio.Input != nil && rt.Audio.Input.Transcription != nil) {
-		m, err := newModel(&session.ModelConfig.Pipeline, cl, ml, appConfig, evaluator, routing)
+	candidateModel := session.ModelInterface
+	candidateVoice := session.Voice
+	candidateParams := maps.Clone(session.ttsParams)
+	var candidateRelease func()
+	selectVoice := rt.Model != "" || explicitVoice
+	if rebuild {
+		m, err := newModel(&candidateConfig.Pipeline, cl, ml, appConfig, evaluator, routing)
 		if err != nil {
 			return err
 		}
-		session.ModelInterface = m
+		candidateModel = m
+		wrapped := m.(*wrappedModel)
+		if selectVoice {
+			configuredVoice := wrapped.TTSConfig.TTSConfig.Voice
+			if explicitVoice {
+				configuredVoice = string(rt.Audio.Output.Voice)
+			}
+			candidateVoice, candidateParams, candidateRelease, err = resolveRealtimeVoice(
+				context.Background(), configuredVoice, wrapped.TTSConfig, profiles,
+			)
+			if err != nil {
+				return err
+			}
+		}
+		wrapped.setTTSParams(candidateParams)
+	}
+
+	if rt.LocalAIClassifier != nil {
+		if err := validateClassifierActivation(candidateModel, rt.LocalAIClassifier); err != nil {
+			if candidateRelease != nil {
+				candidateRelease()
+			}
+			return err
+		}
+	}
+
+	oldRelease := session.voiceRelease
+	session.TranscriptionOnly = false
+	session.Model = candidateModelName
+	session.ModelConfig = candidateConfig
+	session.ModelInterface = candidateModel
+	session.InputAudioTranscription = candidateTranscription
+	if selectVoice {
+		session.installVoiceBinding(candidateVoice, candidateParams, candidateRelease)
+		if oldRelease != nil {
+			oldRelease()
+		}
+	}
+
+	if rebuild {
 		// A session.update that swaps the model/voice rebuilds the pipeline, so
 		// warm the new backends too (unless opted out) — otherwise the next turn
 		// pays the cold-start load the original session warm-up already avoided.
@@ -1238,9 +1302,9 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 		// stall every other session. Load errors are logged (and still surface on
 		// first use); per-stage failures are already warned inside
 		// backend.PreloadStages.
-		if !session.ModelConfig.Pipeline.DisableWarmup {
+		if !candidateConfig.Pipeline.DisableWarmup {
 			go func() {
-				if err := m.Warmup(context.Background()); err != nil {
+				if err := candidateModel.Warmup(context.Background()); err != nil {
 					xlog.Error("realtime warmup failed after session.update", "error", err)
 				}
 			}()
@@ -1299,9 +1363,6 @@ func updateSession(session *Session, update *types.SessionUnion, cl *config.Mode
 		// Replace-not-merge, like tools: the client owns the whole option
 		// list. Invalid configs reject the update without touching the
 		// session's current classifier.
-		if err := validateClassifierActivation(session.ModelInterface, rt.LocalAIClassifier); err != nil {
-			return err
-		}
 		session.Classifier = rt.LocalAIClassifier
 		prewarmClassifier(session)
 	}

core/http/endpoints/openai/realtime_model.go

@@ -398,6 +398,10 @@ func (m *wrappedModel) TTS(ctx context.Context, text, voice, language string) (s
 	return backend.ModelTTS(ctx, text, voice, language, "", maps.Clone(m.ttsParams), m.modelLoader, m.appConfig, *m.TTSConfig)
 }
 
+func (m *wrappedModel) setTTSParams(params map[string]string) {
+	m.ttsParams = maps.Clone(params)
+}
+
 func (m *wrappedModel) TTSStream(ctx context.Context, text, voice, language string, onAudio func(pcm []byte, sampleRate int) error) error {
 	return ttsStream(ctx, m.modelLoader, m.appConfig, *m.TTSConfig, text, voice, language, maps.Clone(m.ttsParams), onAudio)
 }

core/http/endpoints/openai/realtime_voice_profile_test.go

@@ -5,8 +5,12 @@ import (
 	"context"
 	"encoding/binary"
 	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
 	"time"
 
+	"github.com/mudler/LocalAI/core/http/endpoints/openai/types"
 	grpcPkg "github.com/mudler/LocalAI/pkg/grpc"
 	"github.com/mudler/LocalAI/pkg/grpc/proto"
 	"github.com/mudler/LocalAI/pkg/model"
@@ -177,4 +181,174 @@ var _ = Describe("wrappedModel voice profile parameters", func() {
 	})
 })
 
+var _ = Describe("realtime session voice switching", func() {
+	type fixture struct {
+		store     *voiceprofile.Store
+		voiceDir  string
+		loader    *config.ModelConfigLoader
+		models    *model.ModelLoader
+		appConfig *config.ApplicationConfig
+		profileA  voiceprofile.Profile
+		profileB  voiceprofile.Profile
+	}
+
+	newFixture := func(ctx SpecContext) *fixture {
+		modelDir := GinkgoT().TempDir()
+		voiceDir := GinkgoT().TempDir()
+		store := voiceprofile.NewStore(voiceDir)
+		DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
+		profileA, err := store.Create(ctx, voiceprofile.CreateInput{
+			Name: "Alpha", Language: "en", Transcript: "Alpha transcript", ConsentConfirmed: true,
+		}, bytes.NewReader(realtimeProfileWAV(time.Second)))
+		Expect(err).NotTo(HaveOccurred())
+		profileB, err := store.Create(ctx, voiceprofile.CreateInput{
+			Name: "Beta", Language: "it", Transcript: "Beta transcript", ConsentConfirmed: true,
+		}, bytes.NewReader(realtimeProfileWAV(time.Second)))
+		Expect(err).NotTo(HaveOccurred())
+
+		configs := map[string]string{
+			"vad":    "name: vad\nbackend: test\nparameters:\n  model: vad.bin\n",
+			"stt":    "name: stt\nbackend: test\nparameters:\n  model: stt.bin\n",
+			"llm":    "name: llm\nbackend: test\nparameters:\n  model: llm.bin\n",
+			"tts-a":  fmt.Sprintf("name: tts-a\nbackend: qwen3-tts-cpp\nparameters:\n  model: tts-a.bin\ntts:\n  voice: %s\n  voice_cloning: true\n", profileA.Voice),
+			"tts-b":  fmt.Sprintf("name: tts-b\nbackend: qwen3-tts-cpp\nparameters:\n  model: tts-b.bin\ntts:\n  voice: %s\n  voice_cloning: true\n", profileB.Voice),
+			"pipe-a": "name: pipe-a\npipeline:\n  vad: vad\n  transcription: stt\n  llm: llm\n  tts: tts-a\n  disable_warmup: true\n",
+			"pipe-b": "name: pipe-b\npipeline:\n  vad: vad\n  transcription: stt\n  llm: llm\n  tts: tts-b\n  disable_warmup: true\n",
+		}
+		for name, body := range configs {
+			Expect(os.WriteFile(filepath.Join(modelDir, name+".yaml"), []byte(body), 0o644)).To(Succeed())
+		}
+		loader := config.NewModelConfigLoader(modelDir)
+		Expect(loader.LoadModelConfigsFromPath(modelDir)).To(Succeed())
+		state, err := system.GetSystemState(system.WithModelPath(modelDir))
+		Expect(err).NotTo(HaveOccurred())
+		return &fixture{
+			store: store, voiceDir: voiceDir, loader: loader, models: model.NewModelLoader(state),
+			appConfig: config.NewApplicationConfig(config.WithSystemState(state)),
+			profileA:  profileA, profileB: profileB,
+		}
+	}
+
+	newSession := func(f *fixture, voice string) *Session {
+		cfg, err := f.loader.LoadModelConfigFileByNameDefaultOptions("pipe-a", f.appConfig)
+		Expect(err).NotTo(HaveOccurred())
+		m, err := newModel(&cfg.Pipeline, f.loader, f.models, f.appConfig, nil, nil)
+		Expect(err).NotTo(HaveOccurred())
+		session := &Session{
+			Model: "pipe-a", Voice: voice, ModelConfig: cfg, ModelInterface: m,
+			InputAudioTranscription: &types.AudioTranscription{Model: "stt"},
+		}
+		resolved, params, release, err := resolveRealtimeVoice(context.Background(), voice, m.(*wrappedModel).TTSConfig, f.store)
+		Expect(err).NotTo(HaveOccurred())
+		session.installVoiceBinding(resolved, params, release)
+		return session
+	}
+
+	update := func(f *fixture, session *Session, rt *types.RealtimeSession) error {
+		return updateSession(session, &types.SessionUnion{Realtime: rt}, f.loader, f.models, f.appConfig, nil, nil, f.store)
+	}
+
+	It("switches ordinary voices to profiles and clears the lease at final cleanup", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, "speaker-1")
+		Expect(update(f, session, &types.RealtimeSession{Audio: &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: types.Voice(f.profileA.Voice)}}})).To(Succeed())
+
+		Expect(session.Voice).To(BeAnExistingFile())
+		Expect(session.ttsParams).To(Equal(map[string]string{"ref_text": "Alpha transcript"}))
+		leased := session.Voice
+		session.releaseVoiceBinding()
+		session.releaseVoiceBinding()
+		Expect(leased).NotTo(BeAnExistingFile())
+	})
+
+	It("replaces one profile lease with another and then an ordinary voice", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, f.profileA.Voice)
+		firstLease := session.Voice
+
+		Expect(update(f, session, &types.RealtimeSession{Audio: &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: types.Voice(f.profileB.Voice)}}})).To(Succeed())
+		Expect(firstLease).NotTo(BeAnExistingFile())
+		secondLease := session.Voice
+		Expect(secondLease).To(BeAnExistingFile())
+		Expect(session.ttsParams).To(HaveKeyWithValue("ref_text", "Beta transcript"))
+
+		Expect(update(f, session, &types.RealtimeSession{Audio: &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: "speaker-2"}}})).To(Succeed())
+		Expect(secondLease).NotTo(BeAnExistingFile())
+		Expect(session.Voice).To(Equal("speaker-2"))
+		Expect(session.ttsParams).To(BeNil())
+		Expect(session.ModelInterface.(*wrappedModel).ttsParams).To(BeNil())
+	})
+
+	It("uses a new model default profile unless an explicit voice takes precedence", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, "speaker-1")
+		Expect(update(f, session, &types.RealtimeSession{Model: "pipe-b"})).To(Succeed())
+		Expect(session.ttsParams).To(HaveKeyWithValue("ref_text", "Beta transcript"))
+		defaultLease := session.Voice
+
+		Expect(update(f, session, &types.RealtimeSession{
+			Model: "pipe-a",
+			Audio: &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: "speaker-explicit"}},
+		})).To(Succeed())
+		Expect(defaultLease).NotTo(BeAnExistingFile())
+		Expect(session.Voice).To(Equal("speaker-explicit"))
+		Expect(session.ttsParams).To(BeNil())
+	})
+
+	It("preserves a profile binding across a language-only rebuild", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, f.profileA.Voice)
+		lease := session.Voice
+		Expect(update(f, session, &types.RealtimeSession{Audio: &types.RealtimeSessionAudio{Input: &types.SessionAudioInput{
+			Transcription: &types.AudioTranscription{Language: "fr"},
+		}}})).To(Succeed())
+
+		Expect(session.Voice).To(Equal(lease))
+		Expect(session.InputAudioTranscription.Model).To(Equal("stt"))
+		Expect(session.InputAudioTranscription.Language).To(Equal("fr"))
+		wrapped := session.ModelInterface.(*wrappedModel)
+		Expect(wrapped.ttsParams).To(Equal(map[string]string{"ref_text": "Alpha transcript"}))
+		wrapped.ttsParams["ref_text"] = "wrapper mutation"
+		Expect(session.ttsParams).To(HaveKeyWithValue("ref_text", "Alpha transcript"))
+	})
+
+	It("rolls back the model, wrapper, voice, and lease when preparation fails", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, f.profileA.Voice)
+		oldModel, oldConfig, oldVoice := session.ModelInterface, session.ModelConfig, session.Voice
+		err := update(f, session, &types.RealtimeSession{
+			Model: "pipe-b",
+			Audio: &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: "localai://voice-profiles/00000000-0000-0000-0000-000000000001"}},
+		})
+
+		Expect(err).To(HaveOccurred())
+		Expect(session.Model).To(Equal("pipe-a"))
+		Expect(session.ModelConfig).To(BeIdenticalTo(oldConfig))
+		Expect(session.ModelInterface).To(BeIdenticalTo(oldModel))
+		Expect(session.Voice).To(Equal(oldVoice))
+		Expect(oldVoice).To(BeAnExistingFile())
+		Expect(session.ttsParams).To(HaveKeyWithValue("ref_text", "Alpha transcript"))
+	})
+
+	It("releases a candidate profile lease when later validation fails", func(ctx SpecContext) {
+		f := newFixture(ctx)
+		session := newSession(f, "speaker-1")
+		leases := func() []string {
+			matches, err := filepath.Glob(filepath.Join(f.voiceDir, voiceprofile.DirectoryName, ".leases", "*", "*.wav"))
+			Expect(err).NotTo(HaveOccurred())
+			return matches
+		}
+		Expect(leases()).To(BeEmpty())
+
+		err := update(f, session, &types.RealtimeSession{
+			Audio:             &types.RealtimeSessionAudio{Output: &types.SessionAudioOutput{Voice: types.Voice(f.profileA.Voice)}},
+			LocalAIClassifier: classifierTestConfig(0, nil),
+		})
+
+		Expect(err).To(HaveOccurred())
+		Expect(session.Voice).To(Equal("speaker-1"))
+		Expect(leases()).To(BeEmpty())
+	})
+})
+
 func ptrTo[T any](value T) *T { return &value }

core/services/worker/ephemeral_capacity_test.go

@@ -43,7 +43,7 @@ func (capacityShortWriter) Write(p []byte) (int, error) {
 
 var _ = Describe("EphemeralCapacityGuard", func() {
 	It("derives bounded defaults and preserves positive overrides", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		limit, headroom, err := effectiveEphemeralCapacity([]string{root}, 0, -1)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(limit).To(BeNumerically(">", 0))
@@ -57,8 +57,8 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("accounts existing regular files without following symlinks", func() {
-		root := GinkgoT().TempDir()
-		outside := filepath.Join(GinkgoT().TempDir(), "outside.bin")
+		root := canonicalWorkerTempDir()
+		outside := filepath.Join(canonicalWorkerTempDir(), "outside.bin")
 		Expect(os.WriteFile(filepath.Join(root, "existing.bin"), make([]byte, 6), 0o600)).To(Succeed())
 		Expect(os.WriteFile(outside, make([]byte, 100), 0o600)).To(Succeed())
 		Expect(os.Symlink(outside, filepath.Join(root, "outside-link"))).To(Succeed())
@@ -77,7 +77,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("serializes competing reservations", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 1, 0)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -109,7 +109,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("makes only an equal active reservation idempotent", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "nested", "payload.bin")
@@ -128,7 +128,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("retains committed bytes when the same path starts another reservation", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -147,7 +147,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("retains startup-accounted bytes when the path is reserved", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		path := filepath.Join(root, "payload.bin")
 		Expect(os.WriteFile(path, make([]byte, 4), 0o600)).To(Succeed())
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
@@ -163,7 +163,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("commits the regular file's actual size", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -176,7 +176,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("preserves configured filesystem headroom", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 1<<30, 1<<62)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -189,7 +189,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("reserves bounded chunks before forwarding unknown-length input", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, ephemeralCapacityWriteChunk+1, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -210,7 +210,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("waits for an open bounded writer before committing", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -254,7 +254,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("does not share pending capacity between concurrent writers", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "payload.bin")
@@ -290,7 +290,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rolls back bytes the destination writer does not accept", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 5, 0)
 		Expect(err).NotTo(HaveOccurred())
 		writer, err := guard.NewWriter(filepath.Join(root, "payload.bin"), capacityShortWriter{})
@@ -304,8 +304,8 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects paths outside roots and through symlinks", func() {
-		root := GinkgoT().TempDir()
-		outside := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
+		outside := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 100, 0)
 		Expect(err).NotTo(HaveOccurred())
 
@@ -319,7 +319,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("supports recovery tree accounting without dropping active reservations", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		active := filepath.Join(root, "active", "payload.bin")
@@ -335,7 +335,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("waits for pre-release reservations before request cleanup scans", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-1", "input.wav")
@@ -357,7 +357,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects staging after request cleanup begins", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(guard.BeginRequestRelease(context.Background(), "request-1")).To(Succeed())
@@ -370,7 +370,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("leaves a late commit recoverable when release times out", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-1", "late.wav")
@@ -386,7 +386,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("bounds release markers without reopening registered work", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		Expect(guard.BeginRequestOperation("request-pinned")).To(Succeed())
@@ -411,7 +411,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("applies backpressure at the release-pin cap and clears ownership", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
@@ -438,7 +438,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("makes committed files recoverable when pin backpressure expires", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		guard, err := NewEphemeralCapacityGuard([]string{root}, 10, 0)
 		Expect(err).NotTo(HaveOccurred())
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
@@ -459,7 +459,7 @@ var _ = Describe("EphemeralCapacityGuard", func() {
 	})
 
 	It("rejects a registered cache-hit claim after pin backpressure expires", func() {
-		root := GinkgoT().TempDir()
+		root := canonicalWorkerTempDir()
 		path := filepath.Join(root, "audio", "request-target", "input.wav")
 		Expect(os.MkdirAll(filepath.Dir(path), 0o750)).To(Succeed())
 		Expect(os.WriteFile(path, []byte("data"), 0o600)).To(Succeed())

core/services/worker/ephemeral_cleanup_test.go

@@ -24,7 +24,7 @@ var _ = Describe("Worker ephemeral staging cleanup", func() {
 		return dir
 	}
 
-	BeforeEach(func() { stagingDir = GinkgoT().TempDir() })
+	BeforeEach(func() { stagingDir = canonicalWorkerTempDir() })
 
 	It("removes staged request directories older than the TTL", func() {
 		old := mkEphemeral("aaaa1111", 48*time.Hour)
@@ -57,7 +57,7 @@ var _ = Describe("Worker ephemeral staging cleanup", func() {
 	})
 
 	It("sweeps both transport roots by newest descendant and skips active requests", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		httpRoot := filepath.Join(stagingDir, "ephemeral")
 		s3Root := filepath.Join(cacheDir, "ephemeral")
 		guard, err := NewEphemeralCapacityGuard([]string{httpRoot, s3Root}, 8, 0)

core/services/worker/file_staging_release_test.go

@@ -87,7 +87,7 @@ func (m *releaseMessagingClient) Close()            {}
 
 var _ = Describe("Worker exact-key staging release", func() {
 	It("protects a startup-accounted HTTP cache hit through authenticated repeated probes", func() {
-		stagingDir := GinkgoT().TempDir()
+		stagingDir := canonicalWorkerTempDir()
 		root := filepath.Join(stagingDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		remotePath := filepath.Join(stagingDir, filepath.FromSlash(key))
@@ -107,11 +107,11 @@ var _ = Describe("Worker exact-key staging release", func() {
 		Expect(err).NotTo(HaveOccurred())
 		addr := listener.Addr().String()
 		Expect(listener.Close()).To(Succeed())
-		server, err := nodes.StartFileTransferServerWithCapacity(addr, stagingDir, GinkgoT().TempDir(), GinkgoT().TempDir(), "secret", 0, nil, guard)
+		server, err := nodes.StartFileTransferServerWithCapacity(addr, stagingDir, canonicalWorkerTempDir(), canonicalWorkerTempDir(), "secret", 0, nil, guard)
 		Expect(err).NotTo(HaveOccurred())
 		DeferCleanup(nodes.ShutdownFileTransferServer, server)
 
-		localPath := filepath.Join(GinkgoT().TempDir(), "input.wav")
+		localPath := filepath.Join(canonicalWorkerTempDir(), "input.wav")
 		Expect(os.WriteFile(localPath, content, 0o600)).To(Succeed())
 		stager := nodes.NewHTTPFileStager(func(string) (string, error) { return addr, nil }, "secret")
 		for range 2 {
@@ -129,7 +129,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("claims a startup-scanned cache hit against stale recovery until release", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -158,7 +158,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("downloads again when a cache file disappears while being claimed", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
 		Expect(os.MkdirAll(filepath.Dir(cachePath), 0o750)).To(Succeed())
@@ -176,7 +176,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("makes repeated cache-hit claims idempotent", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -199,7 +199,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("capacity-checks growth of a startup-scanned cache file", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		key := "ephemeral/audio/request-id/input.wav"
 		cachePath := filepath.Join(cacheDir, filepath.FromSlash(key))
@@ -220,7 +220,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("reserves S3 object size before download and releases it with the exact key", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		store := &stagingObjectStore{payload: []byte("data")}
 		fm, err := storage.NewFileManager(store, cacheDir)
@@ -240,7 +240,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rejects an oversized S3 object before starting its download", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		store := &stagingObjectStore{payload: []byte("oversized")}
 		fm, err := storage.NewFileManager(store, cacheDir)
 		Expect(err).NotTo(HaveOccurred())
@@ -253,7 +253,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rolls back an S3 reservation when the download fails", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		root := filepath.Join(cacheDir, "ephemeral")
 		store := &stagingObjectStore{payload: []byte("data"), getErr: errors.New("download failed")}
 		fm, err := storage.NewFileManager(store, cacheDir)
@@ -267,7 +267,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("removes only the exact cache file and upload sidecars", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 		Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 		target := filepath.Join(categoryDir, "input.wav")
@@ -285,7 +285,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("succeeds for a missing file and prunes empty category and request directories", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 		Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 
@@ -298,8 +298,8 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("rejects traversal and symlink escapes", func() {
-		cacheDir := GinkgoT().TempDir()
-		outsideDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
+		outsideDir := canonicalWorkerTempDir()
 		outsidePath := filepath.Join(outsideDir, "input.wav")
 		Expect(os.WriteFile(outsidePath, []byte("keep"), 0640)).To(Succeed())
 		requestDir := filepath.Join(cacheDir, "ephemeral", "request-id")
@@ -319,7 +319,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 
 	It("rejects symlinked files and sidecars without deleting their targets", func() {
 		for _, linkedName := range []string{"input.wav", "input.wav.sha256", "input.wav.sha256.target"} {
-			cacheDir := GinkgoT().TempDir()
+			cacheDir := canonicalWorkerTempDir()
 			categoryDir := filepath.Join(cacheDir, "ephemeral", "request-id", "audio")
 			Expect(os.MkdirAll(categoryDir, 0750)).To(Succeed())
 			target := filepath.Join(categoryDir, "input.wav")
@@ -336,7 +336,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("registers an exact release handler", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		path := filepath.Join(cacheDir, "ephemeral", "request-id", "audio", "input.wav")
 		Expect(os.MkdirAll(filepath.Dir(path), 0750)).To(Succeed())
 		Expect(os.WriteFile(path, []byte("data"), 0640)).To(Succeed())
@@ -358,7 +358,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("releases a request batch through one worker message", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		keys := []string{
 			"ephemeral/audio/request-id/input.wav",
 			"ephemeral/images/request-id/frame.jpg",
@@ -387,7 +387,7 @@ var _ = Describe("Worker exact-key staging release", func() {
 	})
 
 	It("returns validation errors through the release handler", func() {
-		cacheDir := GinkgoT().TempDir()
+		cacheDir := canonicalWorkerTempDir()
 		fm, err := storage.NewFileManager(nil, cacheDir)
 		Expect(err).NotTo(HaveOccurred())
 		client := &releaseMessagingClient{}

core/services/worker/worker_suite_test.go

@@ -1,6 +1,7 @@
 package worker
 
 import (
+	"path/filepath"
 	"testing"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -11,3 +12,11 @@ func TestWorker(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Worker Suite")
 }
+
+// Capacity guards reject symlink components, including macOS /var -> /private/var.
+func canonicalWorkerTempDir() string {
+	GinkgoHelper()
+	dir, err := filepath.EvalSymlinks(GinkgoT().TempDir())
+	Expect(err).NotTo(HaveOccurred())
+	return dir
+}

docs/content/features/text-to-audio.md

@@ -141,7 +141,22 @@ pipeline:
 
 LocalAI resolves this profile when the realtime session starts. The selected TTS model must support Voice Library cloning.
 
-This feature does not resolve Voice Library URIs sent later through realtime `session.update`. You can still use `session.update` with ordinary backend voice names or IDs.
+You can also change the profile during a realtime session. Set `audio.output.voice` to a Voice Library URI in a `session.update` event:
+
+```json
+{
+  "type": "session.update",
+  "session": {
+    "audio": {
+      "output": {
+        "voice": "localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000"
+      }
+    }
+  }
+}
+```
+
+If the same update changes `model`, the explicit `audio.output.voice` value takes precedence over the new model's `tts.voice` default. The selected model must support Voice Library cloning.
 
 #### Supported backend and model variants