[voice] feat: add managed voice cloning profiles (#10799)

* feat(ui): add voice library workflow

Give administrators a production-ready flow to record or upload consented reference audio, manage reusable profiles, inspect API usage, discover compatible models, and hand a saved voice directly to text-to-speech.

Assisted-by: Codex:gpt-5

* feat(voice): add managed voice cloning profiles

Make reusable reference voices manageable through the admin API instead of requiring model-directory and YAML edits. Discover compatible installed and gallery models from server-side backend capabilities, retain explicit model configuration controls, and stage saved references for supported backends.

Expose profile management through REST and MCP, document backend-specific behavior, and cover the workflow from profile creation through real Qwen3-TTS synthesis. Harden the agent-job HTTP test against completion racing cancellation.

Assisted-by: Codex:gpt-5

---------

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/python/chatterbox/backend.py

@@ -199,7 +199,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
             if "language" in self.options:
                 kwargs["language_id"] = self.options["language"]
-            if self.AudioPath is not None:
+            if request.voice and os.path.isfile(request.voice):
+                kwargs["audio_prompt_path"] = request.voice
+            elif self.AudioPath is not None:
                 kwargs["audio_prompt_path"] = self.AudioPath
 
             # add options to kwargs
@@ -211,6 +213,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # strings on the wire and are coerced to float/int/bool.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     kwargs[key] = coerce_param_value(value)
 
             # Check if text exceeds 250 characters

backend/python/coqui/backend.py

@@ -79,14 +79,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             if self.tts.is_multi_lingual and lang is None:
                return backend_pb2.Result(success=False, message=f"Model is multi-lingual, but no language was provided")
 
-            # if model is multi-speaker, use speaker_wav or the speaker_id from request.voice
-            if self.tts.is_multi_speaker and self.AudioPath is None and request.voice is None:
+            # A path-shaped per-request voice is a cloning reference; otherwise
+            # preserve Coqui's named-speaker behavior.
+            request_voice = request.voice if request.voice else ""
+            speaker_wav = request_voice if os.path.isfile(request_voice) else self.AudioPath
+            if self.tts.is_multi_speaker and speaker_wav is None and not request_voice:
                 return backend_pb2.Result(success=False, message=f"Model is multi-speaker, but no speaker was provided")
 
-            if self.tts.is_multi_speaker and request.voice is not None:
-               self.tts.tts_to_file(text=request.text, speaker=request.voice, language=lang, file_path=request.dst)
+            if speaker_wav is not None:
+                self.tts.tts_to_file(text=request.text, speaker_wav=speaker_wav, language=lang, file_path=request.dst)
+            elif self.tts.is_multi_speaker and request_voice:
+               self.tts.tts_to_file(text=request.text, speaker=request_voice, language=lang, file_path=request.dst)
             else:
-                self.tts.tts_to_file(text=request.text, speaker_wav=self.AudioPath, language=lang, file_path=request.dst)
+                self.tts.tts_to_file(text=request.text, language=lang, file_path=request.dst)
         except Exception as err:
             return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
         return backend_pb2.Result(success=True)

backend/python/faster-qwen3-tts/backend.py

@@ -83,20 +83,23 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         return backend_pb2.Result(message="Model loaded successfully", success=True)
 
     def _get_ref_audio_path(self, request):
-        if not self.audio_path:
+        # A per-request voice path is the canonical LocalAI voice-profile
+        # contract. Keep AudioPath as the backwards-compatible YAML fallback.
+        audio_path = request.voice if hasattr(request, "voice") and request.voice else self.audio_path
+        if not audio_path:
             return None
-        if os.path.isabs(self.audio_path):
-            return self.audio_path
+        if os.path.isabs(audio_path):
+            return audio_path
         if self.model_file:
             model_file_base = os.path.dirname(self.model_file)
-            ref_path = os.path.join(model_file_base, self.audio_path)
+            ref_path = os.path.join(model_file_base, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
         if self.model_path:
-            ref_path = os.path.join(self.model_path, self.audio_path)
+            ref_path = os.path.join(self.model_path, audio_path)
             if os.path.exists(ref_path):
                 return ref_path
-        return self.audio_path
+        return audio_path
 
     def TTS(self, request, context):
         try:
@@ -122,13 +125,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     success=False,
                     message="AudioPath is required for voice clone (set in LoadModel)"
                 )
-            ref_text = self.options.get("ref_text")
-            if not ref_text and hasattr(request, 'ref_text') and request.ref_text:
-                ref_text = request.ref_text
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.options.get("ref_text")
             if not ref_text:
                 return backend_pb2.Result(
                     success=False,
-                    message="ref_text is required for voice clone (set via LoadModel Options, e.g. ref_text:Your reference transcript)"
+                    message="ref_text is required for voice clone (set in request.params or LoadModel options)"
                 )
 
             chunk_size = self.options.get("chunk_size")

backend/python/fish-speech/backend.py

@@ -267,6 +267,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def _get_ref_audio_path(self, voice_name=None):
         """Get reference audio path from voices dict or stored AudioPath."""
+        if voice_name and os.path.isfile(voice_name):
+            return voice_name
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
 
@@ -332,7 +334,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             references = []
             voice_name = request.voice if request.voice else None
 
-            if voice_name and voice_name in self.voices:
+            if voice_name and os.path.isfile(voice_name):
+                ref_audio_path = self._get_ref_audio_path(voice_name)
+                with open(ref_audio_path, "rb") as f:
+                    audio_bytes = f.read()
+                ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                references.append(
+                    ServeReferenceAudio(audio=audio_bytes, text=ref_text)
+                )
+                print(
+                    f"[INFO] Using per-request reference audio: {ref_audio_path}",
+                    file=sys.stderr,
+                )
+            elif voice_name and voice_name in self.voices:
                 ref_audio_path = self._get_ref_audio_path(voice_name)
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
@@ -350,7 +364,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if ref_audio_path and os.path.exists(ref_audio_path):
                     with open(ref_audio_path, "rb") as f:
                         audio_bytes = f.read()
-                    ref_text = self.options.get("ref_text", "")
+                    ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
+                    if not ref_text:
+                        ref_text = self.options.get("ref_text", "")
                     references.append(
                         ServeReferenceAudio(audio=audio_bytes, text=ref_text)
                     )

backend/python/neutts/backend.py

@@ -119,9 +119,18 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # add options to kwargs
             kwargs.update(self.options)
 
-            ref_codes = self.model.encode_reference(self.AudioPath)
+            ref_audio = request.voice if request.voice else self.AudioPath
+            if not ref_audio:
+                return backend_pb2.Result(success=False, message="reference audio is required")
+            ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+            if not ref_text:
+                ref_text = self.ref_text
+            if not ref_text:
+                return backend_pb2.Result(success=False, message="ref_text is required")
 
-            wav = self.model.infer(request.text, ref_codes, self.ref_text)
+            ref_codes = self.model.encode_reference(ref_audio)
+
+            wav = self.model.infer(request.text, ref_codes, ref_text)
 
             sf.write(request.dst, wav, 24000)            
         except Exception as err:

backend/python/qwen-tts/backend.py

@@ -362,7 +362,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
         # model_type explicitly set
         if self.model_type == "CustomVoice":
             return "CustomVoice"
-        if self.model_type == "VoiceClone":
+        if self.model_type in ("VoiceClone", "Base"):
             return "VoiceClone"
         if self.model_type == "VoiceDesign":
             return "VoiceDesign"
@@ -380,6 +380,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
     def _get_ref_audio_path(self, request, voice_name=None):
         """Get reference audio path from stored AudioPath or from voices dict."""
+        if hasattr(request, "voice") and request.voice and os.path.isfile(request.voice):
+            return request.voice
         # If voice_name is provided and exists in voices dict, use that
         if voice_name and voice_name in self.voices:
             audio_path = self.voices[voice_name]["audio"]
@@ -735,6 +737,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
             # model receives the types it expects. These override YAML-derived kwargs.
             if hasattr(request, "params") and request.params:
                 for key, value in request.params.items():
+                    if key == "ref_text":
+                        continue
                     generation_kwargs[key] = coerce_param_value(value)
 
             # Generate audio based on mode
@@ -743,7 +747,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
 
                 # Check if multi-voice mode is active (voices dict is populated)
                 voice_name = None
-                if self.voices:
+                request_voice_path = request.voice if request.voice and os.path.isfile(request.voice) else None
+                if self.voices and request_voice_path is None:
                     # Get voice from request (priority) or options
                     voice_name = request.voice if request.voice else None
                     if not voice_name:
@@ -775,11 +780,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 if voice_name and voice_name in self.voices:
                     ref_text_source = self.voices[voice_name]["ref_text"]
                 else:
-                    ref_text_source = self.options.get("ref_text", None)
+                    ref_text_source = request.params.get("ref_text") if hasattr(request, "params") else None
                     if not ref_text_source:
-                        # Try to get from request if available
-                        if hasattr(request, "ref_text") and request.ref_text:
-                            ref_text_source = request.ref_text
+                        ref_text_source = self.options.get("ref_text", None)
 
                 if not ref_text_source:
                     # x_vector_only_mode doesn't require ref_text

backend/python/vllm-omni/backend.py

@@ -700,10 +700,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 inputs["additional_information"]["non_streaming_mode"] = [True]
             elif task_type == "Base":
                 # Voice cloning requires ref_audio and ref_text
-                if "ref_audio" in self.options:
-                    inputs["additional_information"]["ref_audio"] = [self.options["ref_audio"]]
-                if "ref_text" in self.options:
-                    inputs["additional_information"]["ref_text"] = [self.options["ref_text"]]
+                ref_audio = voice if voice and os.path.isfile(voice) else self.options.get("ref_audio")
+                ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
+                if not ref_text:
+                    ref_text = self.options.get("ref_text")
+                if not ref_audio or not ref_text:
+                    return backend_pb2.Result(
+                        success=False,
+                        message="Base TTS voice cloning requires request.voice/ref_audio and params.ref_text",
+                    )
+                inputs["additional_information"]["ref_audio"] = [ref_audio]
+                inputs["additional_information"]["ref_text"] = [ref_text]
                 if "x_vector_only_mode" in self.options:
                     inputs["additional_information"]["x_vector_only_mode"] = [self.options["x_vector_only_mode"]]
 

backend/python/voxcpm/backend.py

@@ -144,7 +144,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     if os.path.exists(potential_path):
                         prompt_wav_path = potential_path
 
-            if hasattr(request, 'AudioPath') and request.AudioPath:
+            if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
                 if os.path.isabs(request.AudioPath):
                     prompt_wav_path = request.AudioPath
                 elif hasattr(request, 'ModelFile') and request.ModelFile:
@@ -155,8 +155,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 else:
                     prompt_wav_path = request.AudioPath
 
-            # Get prompt_text from options if available
-            if "prompt_text" in self.options:
+            # Per-request profile transcript takes precedence over YAML.
+            if hasattr(request, "params") and request.params.get("ref_text"):
+                prompt_text = request.params["ref_text"]
+            elif "prompt_text" in self.options:
                 prompt_text = self.options["prompt_text"]
 
             # Prepare text
@@ -241,7 +243,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                     if os.path.exists(potential_path):
                         prompt_wav_path = potential_path
 
-            if hasattr(request, 'AudioPath') and request.AudioPath:
+            if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
                 if os.path.isabs(request.AudioPath):
                     prompt_wav_path = request.AudioPath
                 elif hasattr(request, 'ModelFile') and request.ModelFile:
@@ -252,8 +254,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
                 else:
                     prompt_wav_path = request.AudioPath
 
-            # Get prompt_text from options if available
-            if "prompt_text" in self.options:
+            # Per-request profile transcript takes precedence over YAML.
+            if hasattr(request, "params") and request.params.get("ref_text"):
+                prompt_text = request.params["ref_text"]
+            elif "prompt_text" in self.options:
                 prompt_text = self.options["prompt_text"]
 
             # Prepare text