mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
[voice] feat: add managed voice cloning profiles (#10799)
* feat(ui): add voice library workflow Give administrators a production-ready flow to record or upload consented reference audio, manage reusable profiles, inspect API usage, discover compatible models, and hand a saved voice directly to text-to-speech. Assisted-by: Codex:gpt-5 * feat(voice): add managed voice cloning profiles Make reusable reference voices manageable through the admin API instead of requiring model-directory and YAML edits. Discover compatible installed and gallery models from server-side backend capabilities, retain explicit model configuration controls, and stage saved references for supported backends. Expose profile management through REST and MCP, document backend-specific behavior, and cover the workflow from profile creation through real Qwen3-TTS synthesis. Harden the agent-job HTTP test against completion racing cancellation. Assisted-by: Codex:gpt-5 --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
@@ -530,11 +530,51 @@ func setVoice(voice string) {
|
||||
}
|
||||
}
|
||||
|
||||
// applyRequestVoice distinguishes named speakers from per-request reference
|
||||
// WAVs. The latter are used by F5-TTS and require the exact transcript under
|
||||
// the cross-backend params.ref_text contract.
|
||||
func applyRequestVoice(req *pb.TTSRequest) error {
|
||||
voice := strings.TrimSpace(req.Voice)
|
||||
if voice == "" {
|
||||
return nil
|
||||
}
|
||||
|
||||
info, statErr := os.Stat(voice)
|
||||
looksLikeFile := filepath.IsAbs(voice) || strings.EqualFold(filepath.Ext(voice), ".wav")
|
||||
if statErr == nil && info.Mode().IsRegular() {
|
||||
refText := ""
|
||||
if req.Params != nil {
|
||||
refText = strings.TrimSpace(req.Params["ref_text"])
|
||||
if refText == "" {
|
||||
refText = strings.TrimSpace(req.Params["voice_text"])
|
||||
}
|
||||
}
|
||||
if refText == "" {
|
||||
return fmt.Errorf("crispasr: params.ref_text is required with a reference voice WAV")
|
||||
}
|
||||
if rc := CppTTSSetVoiceFile(voice, refText); rc < 0 {
|
||||
return fmt.Errorf("crispasr: failed to apply reference voice %q (rc=%d)", voice, rc)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if looksLikeFile {
|
||||
if statErr != nil {
|
||||
return fmt.Errorf("crispasr: reference voice %q: %w", voice, statErr)
|
||||
}
|
||||
return fmt.Errorf("crispasr: reference voice %q is not a regular file", voice)
|
||||
}
|
||||
|
||||
setVoice(voice)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *CrispASR) TTS(req *pb.TTSRequest) error {
|
||||
if req.Dst == "" {
|
||||
return fmt.Errorf("crispasr: TTS requires a destination path")
|
||||
}
|
||||
setVoice(req.Voice)
|
||||
if err := applyRequestVoice(req); err != nil {
|
||||
return err
|
||||
}
|
||||
pcm, err := w.synthesize(req.Text)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -553,7 +593,9 @@ func (w *CrispASR) TTSStream(req *pb.TTSRequest, results chan []byte) error {
|
||||
if req.Text == "" {
|
||||
return fmt.Errorf("crispasr: TTSStream requires text")
|
||||
}
|
||||
setVoice(req.Voice)
|
||||
if err := applyRequestVoice(req); err != nil {
|
||||
return err
|
||||
}
|
||||
pcm, err := w.synthesize(req.Text)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -172,6 +172,32 @@ var _ = Describe("CrispASR", func() {
|
||||
})
|
||||
|
||||
Context("TTS", func() {
|
||||
It("applies a per-request reference WAV and transcript", func() {
|
||||
refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
|
||||
Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
|
||||
|
||||
original := CppTTSSetVoiceFile
|
||||
DeferCleanup(func() { CppTTSSetVoiceFile = original })
|
||||
var gotPath, gotText string
|
||||
CppTTSSetVoiceFile = func(path, refText string) int {
|
||||
gotPath, gotText = path, refText
|
||||
return 0
|
||||
}
|
||||
|
||||
Expect(applyRequestVoice(&pb.TTSRequest{
|
||||
Voice: refWAV,
|
||||
Params: map[string]string{"ref_text": "The exact words in the clip."},
|
||||
})).To(Succeed())
|
||||
Expect(gotPath).To(Equal(refWAV))
|
||||
Expect(gotText).To(Equal("The exact words in the clip."))
|
||||
})
|
||||
|
||||
It("rejects a reference WAV without a transcript", func() {
|
||||
refWAV := filepath.Join(GinkgoT().TempDir(), "reference.wav")
|
||||
Expect(os.WriteFile(refWAV, []byte("fixture"), 0o600)).To(Succeed())
|
||||
Expect(applyRequestVoice(&pb.TTSRequest{Voice: refWAV})).To(MatchError(ContainSubstring("params.ref_text")))
|
||||
})
|
||||
|
||||
It("synthesizes a non-empty WAV", func() {
|
||||
ttsModel := ttsModelOrSkip()
|
||||
ensureLibLoaded()
|
||||
|
||||
85
backend/go/qwen3-tts-cpp/README.md
Normal file
85
backend/go/qwen3-tts-cpp/README.md
Normal file
@@ -0,0 +1,85 @@
|
||||
# Qwen3-TTS C++ backend
|
||||
|
||||
This backend runs Qwen3-TTS GGUF models through
|
||||
[qwentts.cpp](https://github.com/ServeurpersoCom/qwentts.cpp). It supports
|
||||
24 kHz speech generation, streaming, named speakers, voice design, and
|
||||
reference-audio cloning depending on the model variant.
|
||||
|
||||
## Gallery models
|
||||
|
||||
The following Base models accept LocalAI Voice Library profiles:
|
||||
|
||||
- `qwen3-tts-cpp`
|
||||
- `qwen3-tts-cpp-0.6b-base-q4`
|
||||
- `qwen3-tts-cpp-1.7b-base`
|
||||
- `qwen3-tts-cpp-1.7b-base-q4`
|
||||
|
||||
Gallery models containing `customvoice` or `voicedesign` implement those Qwen
|
||||
modes instead and are not advertised as raw reference-audio models.
|
||||
|
||||
Install a Base model with:
|
||||
|
||||
```bash
|
||||
local-ai models install qwen3-tts-cpp
|
||||
```
|
||||
|
||||
## Model configuration
|
||||
|
||||
Base filenames are detected automatically. Set `tts.voice_cloning` only when a
|
||||
verified private conversion has a name that does not identify it as a Base or
|
||||
VoiceClone model:
|
||||
|
||||
```yaml
|
||||
name: private-qwen-voice
|
||||
backend: qwen3-tts-cpp
|
||||
parameters:
|
||||
model: qwen-private/talker.gguf
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
audio_path: voices/default-reference.wav # optional model-wide fallback
|
||||
```
|
||||
|
||||
The tokenizer GGUF is auto-discovered when its filename contains `tokenizer`
|
||||
and it is stored beside the talker. Otherwise set
|
||||
`options: ["tokenizer:qwen-private/tokenizer.gguf"]`.
|
||||
|
||||
`tts.voice_cloning: false` removes a model from Voice Library compatibility
|
||||
results and rejects saved `localai://voice-profiles/...` references. It does
|
||||
not disable Qwen's named-speaker or VoiceDesign modes. Setting it to `true`
|
||||
cannot add cloning to a backend that lacks LocalAI's reference-audio contract.
|
||||
|
||||
Request precedence is: a request `voice`, then `tts.voice`, then
|
||||
`tts.audio_path`. A saved profile supplies its private WAV and exact transcript
|
||||
for that request without changing the model YAML.
|
||||
|
||||
## API example
|
||||
|
||||
Create or select a profile in **Operate → Voice Library**, then pass its stable
|
||||
URI to either speech endpoint:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/v1/audio/speech \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "qwen3-tts-cpp",
|
||||
"input": "This request uses a saved reference voice.",
|
||||
"voice": "localai://voice-profiles/PROFILE_ID"
|
||||
}' \
|
||||
--output speech.wav
|
||||
```
|
||||
|
||||
## Native end-to-end test
|
||||
|
||||
The labeled test loads real GGUFs, synthesizes speech, streams audio, and
|
||||
exercises cloning with a generated 24 kHz reference WAV:
|
||||
|
||||
```bash
|
||||
make -C backend/go/qwen3-tts-cpp qwen3-tts-cpp
|
||||
|
||||
QWEN3TTS_MODEL=/path/to/qwen-talker-0.6b-base-Q8_0.gguf \
|
||||
QWEN3TTS_CODEC=/path/to/qwen-tokenizer-12hz-Q8_0.gguf \
|
||||
QWEN3TTS_LIBRARY=backend/go/qwen3-tts-cpp/libgoqwen3ttscpp-fallback.so \
|
||||
go test ./backend/go/qwen3-tts-cpp -ginkgo.label-filter=e2e
|
||||
```
|
||||
@@ -199,7 +199,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
|
||||
if "language" in self.options:
|
||||
kwargs["language_id"] = self.options["language"]
|
||||
if self.AudioPath is not None:
|
||||
if request.voice and os.path.isfile(request.voice):
|
||||
kwargs["audio_prompt_path"] = request.voice
|
||||
elif self.AudioPath is not None:
|
||||
kwargs["audio_prompt_path"] = self.AudioPath
|
||||
|
||||
# add options to kwargs
|
||||
@@ -211,6 +213,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
# strings on the wire and are coerced to float/int/bool.
|
||||
if hasattr(request, "params") and request.params:
|
||||
for key, value in request.params.items():
|
||||
if key == "ref_text":
|
||||
continue
|
||||
kwargs[key] = coerce_param_value(value)
|
||||
|
||||
# Check if text exceeds 250 characters
|
||||
|
||||
@@ -79,14 +79,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
if self.tts.is_multi_lingual and lang is None:
|
||||
return backend_pb2.Result(success=False, message=f"Model is multi-lingual, but no language was provided")
|
||||
|
||||
# if model is multi-speaker, use speaker_wav or the speaker_id from request.voice
|
||||
if self.tts.is_multi_speaker and self.AudioPath is None and request.voice is None:
|
||||
# A path-shaped per-request voice is a cloning reference; otherwise
|
||||
# preserve Coqui's named-speaker behavior.
|
||||
request_voice = request.voice if request.voice else ""
|
||||
speaker_wav = request_voice if os.path.isfile(request_voice) else self.AudioPath
|
||||
if self.tts.is_multi_speaker and speaker_wav is None and not request_voice:
|
||||
return backend_pb2.Result(success=False, message=f"Model is multi-speaker, but no speaker was provided")
|
||||
|
||||
if self.tts.is_multi_speaker and request.voice is not None:
|
||||
self.tts.tts_to_file(text=request.text, speaker=request.voice, language=lang, file_path=request.dst)
|
||||
if speaker_wav is not None:
|
||||
self.tts.tts_to_file(text=request.text, speaker_wav=speaker_wav, language=lang, file_path=request.dst)
|
||||
elif self.tts.is_multi_speaker and request_voice:
|
||||
self.tts.tts_to_file(text=request.text, speaker=request_voice, language=lang, file_path=request.dst)
|
||||
else:
|
||||
self.tts.tts_to_file(text=request.text, speaker_wav=self.AudioPath, language=lang, file_path=request.dst)
|
||||
self.tts.tts_to_file(text=request.text, language=lang, file_path=request.dst)
|
||||
except Exception as err:
|
||||
return backend_pb2.Result(success=False, message=f"Unexpected {err=}, {type(err)=}")
|
||||
return backend_pb2.Result(success=True)
|
||||
|
||||
@@ -83,20 +83,23 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
return backend_pb2.Result(message="Model loaded successfully", success=True)
|
||||
|
||||
def _get_ref_audio_path(self, request):
|
||||
if not self.audio_path:
|
||||
# A per-request voice path is the canonical LocalAI voice-profile
|
||||
# contract. Keep AudioPath as the backwards-compatible YAML fallback.
|
||||
audio_path = request.voice if hasattr(request, "voice") and request.voice else self.audio_path
|
||||
if not audio_path:
|
||||
return None
|
||||
if os.path.isabs(self.audio_path):
|
||||
return self.audio_path
|
||||
if os.path.isabs(audio_path):
|
||||
return audio_path
|
||||
if self.model_file:
|
||||
model_file_base = os.path.dirname(self.model_file)
|
||||
ref_path = os.path.join(model_file_base, self.audio_path)
|
||||
ref_path = os.path.join(model_file_base, audio_path)
|
||||
if os.path.exists(ref_path):
|
||||
return ref_path
|
||||
if self.model_path:
|
||||
ref_path = os.path.join(self.model_path, self.audio_path)
|
||||
ref_path = os.path.join(self.model_path, audio_path)
|
||||
if os.path.exists(ref_path):
|
||||
return ref_path
|
||||
return self.audio_path
|
||||
return audio_path
|
||||
|
||||
def TTS(self, request, context):
|
||||
try:
|
||||
@@ -122,13 +125,13 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
success=False,
|
||||
message="AudioPath is required for voice clone (set in LoadModel)"
|
||||
)
|
||||
ref_text = self.options.get("ref_text")
|
||||
if not ref_text and hasattr(request, 'ref_text') and request.ref_text:
|
||||
ref_text = request.ref_text
|
||||
ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
|
||||
if not ref_text:
|
||||
ref_text = self.options.get("ref_text")
|
||||
if not ref_text:
|
||||
return backend_pb2.Result(
|
||||
success=False,
|
||||
message="ref_text is required for voice clone (set via LoadModel Options, e.g. ref_text:Your reference transcript)"
|
||||
message="ref_text is required for voice clone (set in request.params or LoadModel options)"
|
||||
)
|
||||
|
||||
chunk_size = self.options.get("chunk_size")
|
||||
|
||||
@@ -267,6 +267,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
|
||||
def _get_ref_audio_path(self, voice_name=None):
|
||||
"""Get reference audio path from voices dict or stored AudioPath."""
|
||||
if voice_name and os.path.isfile(voice_name):
|
||||
return voice_name
|
||||
if voice_name and voice_name in self.voices:
|
||||
audio_path = self.voices[voice_name]["audio"]
|
||||
|
||||
@@ -332,7 +334,19 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
references = []
|
||||
voice_name = request.voice if request.voice else None
|
||||
|
||||
if voice_name and voice_name in self.voices:
|
||||
if voice_name and os.path.isfile(voice_name):
|
||||
ref_audio_path = self._get_ref_audio_path(voice_name)
|
||||
with open(ref_audio_path, "rb") as f:
|
||||
audio_bytes = f.read()
|
||||
ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
|
||||
references.append(
|
||||
ServeReferenceAudio(audio=audio_bytes, text=ref_text)
|
||||
)
|
||||
print(
|
||||
f"[INFO] Using per-request reference audio: {ref_audio_path}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
elif voice_name and voice_name in self.voices:
|
||||
ref_audio_path = self._get_ref_audio_path(voice_name)
|
||||
if ref_audio_path and os.path.exists(ref_audio_path):
|
||||
with open(ref_audio_path, "rb") as f:
|
||||
@@ -350,7 +364,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
if ref_audio_path and os.path.exists(ref_audio_path):
|
||||
with open(ref_audio_path, "rb") as f:
|
||||
audio_bytes = f.read()
|
||||
ref_text = self.options.get("ref_text", "")
|
||||
ref_text = request.params.get("ref_text", "") if hasattr(request, "params") else ""
|
||||
if not ref_text:
|
||||
ref_text = self.options.get("ref_text", "")
|
||||
references.append(
|
||||
ServeReferenceAudio(audio=audio_bytes, text=ref_text)
|
||||
)
|
||||
|
||||
@@ -119,9 +119,18 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
# add options to kwargs
|
||||
kwargs.update(self.options)
|
||||
|
||||
ref_codes = self.model.encode_reference(self.AudioPath)
|
||||
ref_audio = request.voice if request.voice else self.AudioPath
|
||||
if not ref_audio:
|
||||
return backend_pb2.Result(success=False, message="reference audio is required")
|
||||
ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
|
||||
if not ref_text:
|
||||
ref_text = self.ref_text
|
||||
if not ref_text:
|
||||
return backend_pb2.Result(success=False, message="ref_text is required")
|
||||
|
||||
wav = self.model.infer(request.text, ref_codes, self.ref_text)
|
||||
ref_codes = self.model.encode_reference(ref_audio)
|
||||
|
||||
wav = self.model.infer(request.text, ref_codes, ref_text)
|
||||
|
||||
sf.write(request.dst, wav, 24000)
|
||||
except Exception as err:
|
||||
|
||||
@@ -362,7 +362,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
# model_type explicitly set
|
||||
if self.model_type == "CustomVoice":
|
||||
return "CustomVoice"
|
||||
if self.model_type == "VoiceClone":
|
||||
if self.model_type in ("VoiceClone", "Base"):
|
||||
return "VoiceClone"
|
||||
if self.model_type == "VoiceDesign":
|
||||
return "VoiceDesign"
|
||||
@@ -380,6 +380,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
|
||||
def _get_ref_audio_path(self, request, voice_name=None):
|
||||
"""Get reference audio path from stored AudioPath or from voices dict."""
|
||||
if hasattr(request, "voice") and request.voice and os.path.isfile(request.voice):
|
||||
return request.voice
|
||||
# If voice_name is provided and exists in voices dict, use that
|
||||
if voice_name and voice_name in self.voices:
|
||||
audio_path = self.voices[voice_name]["audio"]
|
||||
@@ -735,6 +737,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
# model receives the types it expects. These override YAML-derived kwargs.
|
||||
if hasattr(request, "params") and request.params:
|
||||
for key, value in request.params.items():
|
||||
if key == "ref_text":
|
||||
continue
|
||||
generation_kwargs[key] = coerce_param_value(value)
|
||||
|
||||
# Generate audio based on mode
|
||||
@@ -743,7 +747,8 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
|
||||
# Check if multi-voice mode is active (voices dict is populated)
|
||||
voice_name = None
|
||||
if self.voices:
|
||||
request_voice_path = request.voice if request.voice and os.path.isfile(request.voice) else None
|
||||
if self.voices and request_voice_path is None:
|
||||
# Get voice from request (priority) or options
|
||||
voice_name = request.voice if request.voice else None
|
||||
if not voice_name:
|
||||
@@ -775,11 +780,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
if voice_name and voice_name in self.voices:
|
||||
ref_text_source = self.voices[voice_name]["ref_text"]
|
||||
else:
|
||||
ref_text_source = self.options.get("ref_text", None)
|
||||
ref_text_source = request.params.get("ref_text") if hasattr(request, "params") else None
|
||||
if not ref_text_source:
|
||||
# Try to get from request if available
|
||||
if hasattr(request, "ref_text") and request.ref_text:
|
||||
ref_text_source = request.ref_text
|
||||
ref_text_source = self.options.get("ref_text", None)
|
||||
|
||||
if not ref_text_source:
|
||||
# x_vector_only_mode doesn't require ref_text
|
||||
|
||||
@@ -700,10 +700,17 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
inputs["additional_information"]["non_streaming_mode"] = [True]
|
||||
elif task_type == "Base":
|
||||
# Voice cloning requires ref_audio and ref_text
|
||||
if "ref_audio" in self.options:
|
||||
inputs["additional_information"]["ref_audio"] = [self.options["ref_audio"]]
|
||||
if "ref_text" in self.options:
|
||||
inputs["additional_information"]["ref_text"] = [self.options["ref_text"]]
|
||||
ref_audio = voice if voice and os.path.isfile(voice) else self.options.get("ref_audio")
|
||||
ref_text = request.params.get("ref_text") if hasattr(request, "params") else None
|
||||
if not ref_text:
|
||||
ref_text = self.options.get("ref_text")
|
||||
if not ref_audio or not ref_text:
|
||||
return backend_pb2.Result(
|
||||
success=False,
|
||||
message="Base TTS voice cloning requires request.voice/ref_audio and params.ref_text",
|
||||
)
|
||||
inputs["additional_information"]["ref_audio"] = [ref_audio]
|
||||
inputs["additional_information"]["ref_text"] = [ref_text]
|
||||
if "x_vector_only_mode" in self.options:
|
||||
inputs["additional_information"]["x_vector_only_mode"] = [self.options["x_vector_only_mode"]]
|
||||
|
||||
|
||||
@@ -144,7 +144,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
if os.path.exists(potential_path):
|
||||
prompt_wav_path = potential_path
|
||||
|
||||
if hasattr(request, 'AudioPath') and request.AudioPath:
|
||||
if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
|
||||
if os.path.isabs(request.AudioPath):
|
||||
prompt_wav_path = request.AudioPath
|
||||
elif hasattr(request, 'ModelFile') and request.ModelFile:
|
||||
@@ -155,8 +155,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
else:
|
||||
prompt_wav_path = request.AudioPath
|
||||
|
||||
# Get prompt_text from options if available
|
||||
if "prompt_text" in self.options:
|
||||
# Per-request profile transcript takes precedence over YAML.
|
||||
if hasattr(request, "params") and request.params.get("ref_text"):
|
||||
prompt_text = request.params["ref_text"]
|
||||
elif "prompt_text" in self.options:
|
||||
prompt_text = self.options["prompt_text"]
|
||||
|
||||
# Prepare text
|
||||
@@ -241,7 +243,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
if os.path.exists(potential_path):
|
||||
prompt_wav_path = potential_path
|
||||
|
||||
if hasattr(request, 'AudioPath') and request.AudioPath:
|
||||
if prompt_wav_path is None and hasattr(request, 'AudioPath') and request.AudioPath:
|
||||
if os.path.isabs(request.AudioPath):
|
||||
prompt_wav_path = request.AudioPath
|
||||
elif hasattr(request, 'ModelFile') and request.ModelFile:
|
||||
@@ -252,8 +254,10 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
|
||||
else:
|
||||
prompt_wav_path = request.AudioPath
|
||||
|
||||
# Get prompt_text from options if available
|
||||
if "prompt_text" in self.options:
|
||||
# Per-request profile transcript takes precedence over YAML.
|
||||
if hasattr(request, "params") and request.params.get("ref_text"):
|
||||
prompt_text = request.params["ref_text"]
|
||||
elif "prompt_text" in self.options:
|
||||
prompt_text = self.options["prompt_text"]
|
||||
|
||||
# Prepare text
|
||||
|
||||
@@ -22,6 +22,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/routing/pii"
|
||||
"github.com/mudler/LocalAI/core/services/routing/piidetector"
|
||||
"github.com/mudler/LocalAI/core/services/routing/router"
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
"github.com/mudler/LocalAI/core/services/voicerecognition"
|
||||
"github.com/mudler/LocalAI/core/templates"
|
||||
pkggrpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||
@@ -58,6 +59,7 @@ type Application struct {
|
||||
agentPoolService atomic.Pointer[agentpool.AgentPoolService]
|
||||
faceRegistry facerecognition.Registry
|
||||
voiceRegistry voicerecognition.Registry
|
||||
voiceProfileStore *voiceprofile.Store
|
||||
authDB *gorm.DB
|
||||
metricsService *monitoring.LocalAIMetricsService
|
||||
statsRecorder *billing.Recorder
|
||||
@@ -118,6 +120,7 @@ func newApplication(appConfig *config.ApplicationConfig) *Application {
|
||||
modelLoader: ml,
|
||||
applicationConfig: appConfig,
|
||||
templatesEvaluator: templates.NewEvaluator(appConfig.SystemState.Model.ModelsPath),
|
||||
voiceProfileStore: voiceprofile.NewStore(appConfig.DataPath),
|
||||
}
|
||||
|
||||
// Face-recognition registry backed by LocalAI's built-in vector store.
|
||||
@@ -216,6 +219,13 @@ func (a *Application) VoiceRegistry() voicerecognition.Registry {
|
||||
return a.voiceRegistry
|
||||
}
|
||||
|
||||
// VoiceProfileStore returns the persistent library of reusable voice-cloning
|
||||
// references. It is distinct from VoiceRegistry, which stores speaker
|
||||
// recognition embeddings rather than synthesis reference audio.
|
||||
func (a *Application) VoiceProfileStore() *voiceprofile.Store {
|
||||
return a.voiceProfileStore
|
||||
}
|
||||
|
||||
// AuthDB returns the auth database connection, or nil if auth is not enabled.
|
||||
func (a *Application) AuthDB() *gorm.DB {
|
||||
return a.authDB
|
||||
@@ -461,6 +471,11 @@ func (a *Application) Shutdown() error {
|
||||
if a.modelLoader != nil {
|
||||
err = a.modelLoader.StopAllGRPC()
|
||||
}
|
||||
if a.voiceProfileStore != nil {
|
||||
if closeErr := a.voiceProfileStore.Close(); err == nil {
|
||||
err = closeErr
|
||||
}
|
||||
}
|
||||
})
|
||||
return err
|
||||
}
|
||||
@@ -525,6 +540,7 @@ func (a *Application) start() error {
|
||||
// "unavailable" error if startup ran with --disable-stats.
|
||||
assistantClient.StatsRecorder = a.statsRecorder
|
||||
assistantClient.FallbackUser = a.fallbackUser
|
||||
assistantClient.VoiceProfiles = a.voiceProfileStore
|
||||
// PII filter — same nil-or-real wiring.
|
||||
assistantClient.PIIRedactor = a.piiRedactor
|
||||
assistantClient.PIIEvents = a.piiEvents
|
||||
|
||||
@@ -209,10 +209,29 @@ type BackendCapability struct {
|
||||
AcceptsVideos bool
|
||||
// AcceptsAudios indicates multimodal audio input in Predict.
|
||||
AcceptsAudios bool
|
||||
// VoiceCloning describes the backend's per-request reference-audio
|
||||
// contract. Model variants that share a backend may narrow this further;
|
||||
// use VoiceCloningForModel for UI/API decisions.
|
||||
VoiceCloning *VoiceCloningCapability
|
||||
// Description is a human-readable summary of the backend.
|
||||
Description string
|
||||
}
|
||||
|
||||
// VoiceCloningCapability is the model-facing contract for reusable reference
|
||||
// voices. The first release intentionally accepts only browser-normalizable
|
||||
// PCM WAV so every advertised backend sees the same input shape.
|
||||
type VoiceCloningCapability struct {
|
||||
ReferenceTranscriptRequired bool `json:"reference_transcript_required"`
|
||||
AcceptedAudioFormats []string `json:"accepted_audio_formats"`
|
||||
}
|
||||
|
||||
func referenceVoiceCloning() *VoiceCloningCapability {
|
||||
return &VoiceCloningCapability{
|
||||
ReferenceTranscriptRequired: true,
|
||||
AcceptedAudioFormats: []string{"audio/wav"},
|
||||
}
|
||||
}
|
||||
|
||||
// BackendCapabilities maps each backend name (as used in model configs and gallery
|
||||
// entries) to its verified capabilities. This is the single source of truth for
|
||||
// what each backend supports.
|
||||
@@ -261,6 +280,7 @@ var BackendCapabilities = map[string]BackendCapability{
|
||||
AcceptsImages: true,
|
||||
AcceptsVideos: true,
|
||||
AcceptsAudios: true,
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "vLLM omni-modal — supports text, image, video generation and TTS",
|
||||
},
|
||||
"transformers": {
|
||||
@@ -383,6 +403,7 @@ var BackendCapabilities = map[string]BackendCapability{
|
||||
GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodTTS, MethodTTSStream},
|
||||
PossibleUsecases: []string{UsecaseTranscript, UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTranscript, UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "VibeVoice C++ — bidirectional speech, C++ backend with streaming TTS",
|
||||
},
|
||||
"sherpa-onnx": {
|
||||
@@ -409,6 +430,7 @@ var BackendCapabilities = map[string]BackendCapability{
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Coqui TTS — multi-speaker neural synthesis",
|
||||
},
|
||||
"kitten-tts": {
|
||||
@@ -427,50 +449,72 @@ var BackendCapabilities = map[string]BackendCapability{
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Pocket TTS — lightweight text-to-speech",
|
||||
},
|
||||
"qwen-tts": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Qwen TTS",
|
||||
},
|
||||
"qwen3-tts-cpp": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS, MethodTTSStream},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Qwen3 TTS C++ - text-to-speech with streaming, named speakers, voice design and cloning (qwentts.cpp / GGML)",
|
||||
},
|
||||
"faster-qwen3-tts": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Faster Qwen3 TTS — accelerated Qwen TTS",
|
||||
},
|
||||
"fish-speech": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Fish Speech TTS",
|
||||
},
|
||||
"neutts": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "NeuTTS — neural text-to-speech",
|
||||
},
|
||||
"chatterbox": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "Chatterbox TTS",
|
||||
},
|
||||
"voxcpm": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS, MethodTTSStream},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "VoxCPM TTS with streaming support",
|
||||
},
|
||||
"omnivoice-cpp": {
|
||||
GRPCMethods: []GRPCMethod{MethodTTS, MethodTTSStream},
|
||||
PossibleUsecases: []string{UsecaseTTS},
|
||||
DefaultUsecases: []string{UsecaseTTS},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "OmniVoice C++ — multilingual TTS with streaming voice cloning and voice design",
|
||||
},
|
||||
"crispasr": {
|
||||
GRPCMethods: []GRPCMethod{MethodAudioTranscription, MethodTTS, MethodTTSStream, MethodVAD},
|
||||
PossibleUsecases: []string{UsecaseTranscript, UsecaseTTS, UsecaseVAD},
|
||||
DefaultUsecases: []string{UsecaseTranscript},
|
||||
VoiceCloning: referenceVoiceCloning(),
|
||||
Description: "CrispASR GGUF runtime — speech recognition, VAD, and model-dependent TTS",
|
||||
},
|
||||
|
||||
// --- Sound generation backends ---
|
||||
"ace-step": {
|
||||
@@ -613,6 +657,82 @@ func GetBackendCapability(backend string) *BackendCapability {
|
||||
return nil
|
||||
}
|
||||
|
||||
// VoiceCloningForModel returns the reference-audio contract only when the
|
||||
// installed model variant can honor it. Several backends serve both Base
|
||||
// (voice cloning) and CustomVoice/VoiceDesign models, so backend name alone is
|
||||
// deliberately insufficient. Operators with custom filenames can opt in or
|
||||
// out explicitly with tts.voice_cloning; the model option spelling remains a
|
||||
// compatibility fallback for configurations created before the typed field.
|
||||
func VoiceCloningForModel(cfg *ModelConfig) *VoiceCloningCapability {
|
||||
if cfg == nil {
|
||||
return nil
|
||||
}
|
||||
backend := NormalizeBackendName(cfg.Backend)
|
||||
capability := GetBackendCapability(backend)
|
||||
if capability == nil || capability.VoiceCloning == nil {
|
||||
return nil
|
||||
}
|
||||
if cfg.VoiceCloning != nil {
|
||||
if !*cfg.VoiceCloning {
|
||||
return nil
|
||||
}
|
||||
return cloneVoiceCloningCapability(capability.VoiceCloning)
|
||||
}
|
||||
|
||||
if enabled, explicit := voiceCloningOverride(cfg.Options); explicit {
|
||||
if !enabled {
|
||||
return nil
|
||||
}
|
||||
return cloneVoiceCloningCapability(capability.VoiceCloning)
|
||||
}
|
||||
|
||||
identity := strings.ToLower(strings.Join([]string{cfg.Name, cfg.Model, strings.Join(cfg.Options, " ")}, " "))
|
||||
supported := false
|
||||
switch backend {
|
||||
case "qwen3-tts-cpp", "qwen-tts", "vllm-omni":
|
||||
supported = strings.Contains(identity, "base") || strings.Contains(identity, "voiceclone") || strings.Contains(identity, "voice_clone")
|
||||
case "vibevoice-cpp":
|
||||
// Realtime 0.5B consumes a precomputed .gguf voice prompt; the 1.5B
|
||||
// path consumes raw WAV references per request.
|
||||
supported = strings.Contains(identity, "1.5b")
|
||||
case "coqui":
|
||||
supported = strings.Contains(identity, "xtts") || strings.Contains(identity, "your_tts")
|
||||
case "crispasr":
|
||||
supported = strings.Contains(identity, "f5-tts") || strings.Contains(identity, "f5_tts")
|
||||
default:
|
||||
supported = true
|
||||
}
|
||||
if !supported {
|
||||
return nil
|
||||
}
|
||||
return cloneVoiceCloningCapability(capability.VoiceCloning)
|
||||
}
|
||||
|
||||
func voiceCloningOverride(options []string) (enabled, explicit bool) {
|
||||
for _, option := range options {
|
||||
parts := strings.FieldsFunc(option, func(r rune) bool { return r == ':' || r == '=' })
|
||||
if len(parts) != 2 || !strings.EqualFold(strings.TrimSpace(parts[0]), "voice_cloning") {
|
||||
continue
|
||||
}
|
||||
switch strings.ToLower(strings.TrimSpace(parts[1])) {
|
||||
case "true", "1", "yes", "on":
|
||||
return true, true
|
||||
case "false", "0", "no", "off":
|
||||
return false, true
|
||||
}
|
||||
}
|
||||
return false, false
|
||||
}
|
||||
|
||||
func cloneVoiceCloningCapability(capability *VoiceCloningCapability) *VoiceCloningCapability {
|
||||
if capability == nil {
|
||||
return nil
|
||||
}
|
||||
clone := *capability
|
||||
clone.AcceptedAudioFormats = slices.Clone(capability.AcceptedAudioFormats)
|
||||
return &clone
|
||||
}
|
||||
|
||||
// PossibleUsecasesForBackend returns all usecases a backend can support.
|
||||
// Returns nil if the backend is unknown.
|
||||
func PossibleUsecasesForBackend(backend string) []string {
|
||||
|
||||
@@ -72,6 +72,29 @@ var _ = Describe("GetBackendCapability", func() {
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("VoiceCloningForModel", func() {
|
||||
voiceCloningSetting := func(enabled bool) *bool { return &enabled }
|
||||
|
||||
DescribeTable("advertises only compatible model variants",
|
||||
func(cfg ModelConfig, expected bool) {
|
||||
Expect(VoiceCloningForModel(&cfg) != nil).To(Equal(expected))
|
||||
},
|
||||
Entry("Qwen C++ Base", ModelConfig{Name: "qwen3-tts-cpp-0.6b-base", Backend: "qwen3-tts-cpp"}, true),
|
||||
Entry("Qwen C++ CustomVoice", ModelConfig{Name: "qwen3-tts-cpp-customvoice", Backend: "qwen3-tts-cpp"}, false),
|
||||
Entry("VibeVoice realtime 0.5B", ModelConfig{Name: "vibevoice-cpp-0.5b", Backend: "vibevoice-cpp"}, false),
|
||||
Entry("VibeVoice 1.5B", ModelConfig{Name: "vibevoice-1.5b", Backend: "vibevoice-cpp"}, true),
|
||||
Entry("F5 through CrispASR", ModelConfig{Name: "f5-tts-crispasr", Backend: "crispasr"}, true),
|
||||
Entry("ASR through CrispASR", ModelConfig{Name: "parakeet-crispasr", Backend: "crispasr"}, false),
|
||||
Entry("VoxCPM", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm"}, true),
|
||||
Entry("unsupported Piper", ModelConfig{Name: "piper", Backend: "piper"}, false),
|
||||
Entry("typed custom opt-in", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(true)}}, true),
|
||||
Entry("typed opt-out", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(false)}}, false),
|
||||
Entry("typed setting wins over compatibility option", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", TTSConfig: TTSConfig{VoiceCloning: voiceCloningSetting(true)}, Options: []string{"voice_cloning:false"}}, true),
|
||||
Entry("legacy option custom opt-in", ModelConfig{Name: "private-build", Backend: "qwen3-tts-cpp", Options: []string{"voice_cloning:true"}}, true),
|
||||
Entry("legacy option opt-out", ModelConfig{Name: "voxcpm-1.5", Backend: "voxcpm", Options: []string{"voice_cloning=false"}}, false),
|
||||
)
|
||||
})
|
||||
|
||||
var _ = Describe("IsValidUsecaseForBackend", func() {
|
||||
It("accepts a backend's declared usecases", func() {
|
||||
Expect(IsValidUsecaseForBackend("piper", "tts")).To(BeTrue())
|
||||
|
||||
@@ -645,6 +645,14 @@ func DefaultRegistry() map[string]FieldMetaOverride {
|
||||
},
|
||||
|
||||
// --- TTS ---
|
||||
"tts.voice_cloning": {
|
||||
Section: "tts",
|
||||
Label: "Voice Cloning",
|
||||
Description: "Override automatic Voice Library profile support detection for this model. Leave unset to infer support from the backend and model variant.",
|
||||
Component: "toggle",
|
||||
Advanced: true,
|
||||
Order: 89,
|
||||
},
|
||||
"tts.voice": {
|
||||
Section: "tts",
|
||||
Label: "Voice",
|
||||
@@ -652,6 +660,13 @@ func DefaultRegistry() map[string]FieldMetaOverride {
|
||||
Component: "input",
|
||||
Order: 90,
|
||||
},
|
||||
"tts.audio_path": {
|
||||
Section: "tts",
|
||||
Label: "Reference Audio Path",
|
||||
Description: "Default reference audio for voice cloning. A per-request voice or saved Voice Library profile takes precedence.",
|
||||
Component: "input",
|
||||
Order: 91,
|
||||
},
|
||||
|
||||
// --- Diffusers ---
|
||||
"diffusers.pipeline_type": {
|
||||
|
||||
@@ -249,7 +249,6 @@ var grandfatheredUnregistered = []string{
|
||||
"trimspace",
|
||||
"trimsuffix",
|
||||
"trust_remote_code",
|
||||
"tts.audio_path",
|
||||
"type",
|
||||
"yarn_attn_factor",
|
||||
"yarn_beta_fast",
|
||||
|
||||
@@ -28,6 +28,11 @@ type TTSConfig struct {
|
||||
Voice string `yaml:"voice,omitempty" json:"voice,omitempty"`
|
||||
|
||||
AudioPath string `yaml:"audio_path,omitempty" json:"audio_path,omitempty"`
|
||||
|
||||
// VoiceCloning overrides saved-profile capability detection for this model.
|
||||
// A pointer preserves the distinction between an explicit false and the
|
||||
// default automatic behavior.
|
||||
VoiceCloning *bool `yaml:"voice_cloning,omitempty" json:"voice_cloning,omitempty"`
|
||||
}
|
||||
|
||||
// @Description ModelConfig represents a model configuration
|
||||
|
||||
@@ -629,6 +629,25 @@ concurrency_groups:
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("TTS capability configuration", func() {
|
||||
It("preserves an explicit voice-cloning opt-out from YAML", func() {
|
||||
var cfg ModelConfig
|
||||
Expect(yaml.Unmarshal([]byte("name: private-voice\ntts:\n voice_cloning: false\n"), &cfg)).To(Succeed())
|
||||
Expect(cfg.VoiceCloning).NotTo(BeNil())
|
||||
Expect(*cfg.VoiceCloning).To(BeFalse())
|
||||
|
||||
raw, err := yaml.Marshal(cfg)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(string(raw)).To(ContainSubstring("voice_cloning: false"))
|
||||
})
|
||||
|
||||
It("leaves voice-cloning detection automatic when the field is omitted", func() {
|
||||
var cfg ModelConfig
|
||||
Expect(yaml.Unmarshal([]byte("name: automatic-voice\ntts:\n audio_path: voices/default.wav\n"), &cfg)).To(Succeed())
|
||||
Expect(cfg.VoiceCloning).To(BeNil())
|
||||
})
|
||||
})
|
||||
|
||||
var _ = Describe("PII config accessors", func() {
|
||||
It("PIIDetectors returns a fresh copy of the consumer's detector list", func() {
|
||||
cfg := &ModelConfig{PII: PIIConfig{Detectors: []string{"a", "b"}}}
|
||||
|
||||
@@ -3,10 +3,12 @@ package gallery
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/pkg/downloader"
|
||||
"github.com/mudler/LocalAI/pkg/xsync"
|
||||
"github.com/mudler/xlog"
|
||||
@@ -26,6 +28,101 @@ func (e modelConfigCacheEntry) hasExpired() bool {
|
||||
// modelConfigCache caches parsed model config maps keyed by URL.
|
||||
var modelConfigCache = xsync.NewSyncedMap[string, modelConfigCacheEntry]()
|
||||
|
||||
// VoiceCloningCapability returns the same variant-aware reference-audio
|
||||
// contract used for installed models, but derived from a gallery entry before
|
||||
// it is installed. This keeps gallery recommendations tied to server-owned
|
||||
// capability metadata instead of a second allow-list in the frontend.
|
||||
func (m *GalleryModel) VoiceCloningCapability(basePath string) *config.VoiceCloningCapability {
|
||||
if m == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
baseConfig := m.ConfigFile
|
||||
if len(baseConfig) == 0 && m.URL != "" {
|
||||
baseConfig = fetchModelConfigMap(m.URL, basePath)
|
||||
}
|
||||
|
||||
modelReference := nestedString(m.Overrides, "parameters", "model")
|
||||
if modelReference == "" {
|
||||
modelReference = nestedString(baseConfig, "parameters", "model")
|
||||
}
|
||||
|
||||
options, overridden := stringSliceValue(m.Overrides, "options")
|
||||
if !overridden {
|
||||
options, _ = stringSliceValue(baseConfig, "options")
|
||||
}
|
||||
voiceCloning, overridden := nestedBool(m.Overrides, "tts", "voice_cloning")
|
||||
if !overridden {
|
||||
voiceCloning, _ = nestedBool(baseConfig, "tts", "voice_cloning")
|
||||
}
|
||||
|
||||
modelConfig := &config.ModelConfig{
|
||||
Name: m.Name,
|
||||
Backend: m.Backend,
|
||||
Options: options,
|
||||
TTSConfig: config.TTSConfig{
|
||||
VoiceCloning: voiceCloning,
|
||||
},
|
||||
}
|
||||
modelConfig.Model = modelReference
|
||||
return config.VoiceCloningForModel(modelConfig)
|
||||
}
|
||||
|
||||
func nestedBool(values map[string]any, outer, inner string) (*bool, bool) {
|
||||
if values == nil {
|
||||
return nil, false
|
||||
}
|
||||
nested, ok := values[outer].(map[string]any)
|
||||
if !ok {
|
||||
return nil, false
|
||||
}
|
||||
value, exists := nested[inner]
|
||||
if !exists {
|
||||
return nil, false
|
||||
}
|
||||
enabled, ok := value.(bool)
|
||||
if !ok {
|
||||
return nil, true
|
||||
}
|
||||
return &enabled, true
|
||||
}
|
||||
|
||||
func nestedString(values map[string]any, outer, inner string) string {
|
||||
if values == nil {
|
||||
return ""
|
||||
}
|
||||
nested, ok := values[outer].(map[string]any)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
value, _ := nested[inner].(string)
|
||||
return value
|
||||
}
|
||||
|
||||
func stringSliceValue(values map[string]any, key string) ([]string, bool) {
|
||||
if values == nil {
|
||||
return nil, false
|
||||
}
|
||||
raw, exists := values[key]
|
||||
if !exists {
|
||||
return nil, false
|
||||
}
|
||||
switch items := raw.(type) {
|
||||
case []string:
|
||||
return slices.Clone(items), true
|
||||
case []any:
|
||||
result := make([]string, 0, len(items))
|
||||
for _, item := range items {
|
||||
if value, ok := item.(string); ok {
|
||||
result = append(result, value)
|
||||
}
|
||||
}
|
||||
return result, true
|
||||
default:
|
||||
return nil, true
|
||||
}
|
||||
}
|
||||
|
||||
// resolveBackend determines the backend for a GalleryModel by checking (in priority order):
|
||||
// 1. Overrides["backend"] — highest priority, same as install-time merge
|
||||
// 2. Inline ConfigFile["backend"] — for models with inline config maps
|
||||
|
||||
@@ -619,4 +619,59 @@ var _ = Describe("Gallery", func() {
|
||||
Expect(filtered[0].Name).To(Equal("sd-model"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("VoiceCloningCapability", func() {
|
||||
It("derives support from an inline Base model configuration", func() {
|
||||
m := &GalleryModel{
|
||||
Metadata: Metadata{Name: "qwen-base", Backend: "qwen3-tts-cpp"},
|
||||
ConfigFile: map[string]any{
|
||||
"parameters": map[string]any{"model": "Qwen3-TTS-12Hz-0.6B-Base"},
|
||||
},
|
||||
}
|
||||
|
||||
capability := m.VoiceCloningCapability(tempDir)
|
||||
Expect(capability).NotTo(BeNil())
|
||||
Expect(capability.AcceptedAudioFormats).To(Equal([]string{"audio/wav"}))
|
||||
})
|
||||
|
||||
It("uses gallery overrides when deciding between model variants", func() {
|
||||
m := &GalleryModel{
|
||||
Metadata: Metadata{Name: "qwen-custom", Backend: "qwen3-tts-cpp"},
|
||||
ConfigFile: map[string]any{
|
||||
"parameters": map[string]any{"model": "Qwen3-TTS-12Hz-0.6B-Base"},
|
||||
},
|
||||
Overrides: map[string]any{
|
||||
"parameters": map[string]any{"model": "Qwen3-TTS-12Hz-0.6B-CustomVoice"},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(m.VoiceCloningCapability(tempDir)).To(BeNil())
|
||||
})
|
||||
|
||||
It("honors the typed capability override for custom gallery entries", func() {
|
||||
m := &GalleryModel{
|
||||
Metadata: Metadata{Name: "private-voice-model", Backend: "qwen-tts"},
|
||||
Overrides: map[string]any{
|
||||
"parameters": map[string]any{"model": "private-checkpoint"},
|
||||
"tts": map[string]any{"voice_cloning": true},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(m.VoiceCloningCapability(tempDir)).NotTo(BeNil())
|
||||
})
|
||||
|
||||
It("honors an explicit gallery opt-out", func() {
|
||||
m := &GalleryModel{
|
||||
Metadata: Metadata{Name: "voxcpm-1.5", Backend: "voxcpm"},
|
||||
ConfigFile: map[string]any{
|
||||
"tts": map[string]any{"voice_cloning": true},
|
||||
},
|
||||
Overrides: map[string]any{
|
||||
"tts": map[string]any{"voice_cloning": false},
|
||||
},
|
||||
}
|
||||
|
||||
Expect(m.VoiceCloningCapability(tempDir)).To(BeNil())
|
||||
})
|
||||
})
|
||||
})
|
||||
|
||||
@@ -57,6 +57,8 @@ var RouteFeatureRegistry = []RouteFeature{
|
||||
{"POST", "/audio/speech", FeatureAudioSpeech},
|
||||
{"POST", "/tts", FeatureAudioSpeech},
|
||||
{"POST", "/v1/text-to-speech/:voice-id", FeatureAudioSpeech},
|
||||
{"GET", "/api/voice-profiles", FeatureAudioSpeech},
|
||||
{"GET", "/api/voice-profiles/:id/audio", FeatureAudioSpeech},
|
||||
|
||||
// VAD
|
||||
{"POST", "/vad", FeatureVAD},
|
||||
|
||||
@@ -36,6 +36,12 @@ var instructionDefs = []instructionDef{
|
||||
Tags: []string{"audio"},
|
||||
Intro: "Diarization (/v1/audio/diarization) returns speaker-labelled time segments. Backends with native ASR-diarization (vibevoice-cpp) can also emit per-segment text via include_text=true; backends with a dedicated pipeline (sherpa-onnx + pyannote) emit segmentation only. Response formats: json (default), verbose_json (adds speakers summary + text), rttm (NIST format). Sound classification (/v1/audio/classification) returns scored AudioSet sound-event tags (audio tagging via the ced backend); top_k and threshold control the returned set.",
|
||||
},
|
||||
{
|
||||
Name: "voice-library",
|
||||
Description: "Create, preview, list, and delete reusable voice-cloning reference profiles",
|
||||
Tags: []string{"voice-profiles"},
|
||||
Intro: "Profiles persist below the LocalAI data directory as private PCM-WAV references plus exact transcripts. GET and audio preview require the audio_speech feature; create and delete are admin-only. Pass the returned `voice` URI to /v1/audio/speech or /tts. LocalAI resolves the URI at request time and never exposes a filesystem path.",
|
||||
},
|
||||
{
|
||||
Name: "images",
|
||||
Description: "Image generation and inpainting",
|
||||
|
||||
@@ -39,7 +39,7 @@ var _ = Describe("API Instructions Endpoints", func() {
|
||||
|
||||
instructions, ok := resp["instructions"].([]any)
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(instructions).To(HaveLen(16))
|
||||
Expect(instructions).To(HaveLen(17))
|
||||
|
||||
// Verify each instruction has required fields and correct URL format
|
||||
for _, s := range instructions {
|
||||
@@ -78,6 +78,7 @@ var _ = Describe("API Instructions Endpoints", func() {
|
||||
"pii-filtering",
|
||||
"middleware-admin",
|
||||
"intelligent-routing",
|
||||
"voice-library",
|
||||
))
|
||||
})
|
||||
})
|
||||
|
||||
@@ -1,6 +1,9 @@
|
||||
package localai
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
@@ -8,6 +11,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/config"
|
||||
"github.com/mudler/LocalAI/core/http/middleware"
|
||||
"github.com/mudler/LocalAI/core/schema"
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
"github.com/mudler/LocalAI/pkg/audio"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
"github.com/mudler/LocalAI/pkg/utils"
|
||||
@@ -24,7 +28,7 @@ import (
|
||||
// @Success 200 {string} binary "generated audio/wav file"
|
||||
// @Router /v1/audio/speech [post]
|
||||
// @Router /tts [post]
|
||||
func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig) echo.HandlerFunc {
|
||||
func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, profiles *voiceprofile.Store) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
input, ok := c.Get(middleware.CONTEXT_LOCALS_KEY_LOCALAI_REQUEST).(*schema.TTSRequest)
|
||||
if !ok || input.Model == "" {
|
||||
@@ -48,6 +52,35 @@ func TTSEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig
|
||||
|
||||
if input.Voice != "" {
|
||||
cfg.Voice = input.Voice
|
||||
if voiceprofile.IsReference(input.Voice) {
|
||||
profileID, valid := voiceprofile.ParseReference(input.Voice)
|
||||
if !valid {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "invalid voice profile reference")
|
||||
}
|
||||
if config.VoiceCloningForModel(cfg) == nil {
|
||||
return echo.NewHTTPError(http.StatusBadRequest, "selected model does not support reference-audio voice cloning")
|
||||
}
|
||||
if profiles == nil {
|
||||
return echo.NewHTTPError(http.StatusInternalServerError, "voice profile store is unavailable")
|
||||
}
|
||||
profile, referencePath, release, err := profiles.LeaseAudio(c.Request().Context(), profileID)
|
||||
if err != nil {
|
||||
if errors.Is(err, voiceprofile.ErrNotFound) {
|
||||
return echo.NewHTTPError(http.StatusNotFound, "voice profile not found")
|
||||
}
|
||||
return fmt.Errorf("resolve voice profile: %w", err)
|
||||
}
|
||||
defer release()
|
||||
cfg.Voice = referencePath
|
||||
if cfg.Language == "" && profile.Language != "" {
|
||||
cfg.Language = profile.Language
|
||||
}
|
||||
if input.Params == nil {
|
||||
input.Params = make(map[string]string)
|
||||
}
|
||||
input.Params["ref_text"] = profile.Transcript
|
||||
xlog.Debug("Resolved saved voice profile", "id", profile.ID, "model", input.Model)
|
||||
}
|
||||
}
|
||||
|
||||
// Handle streaming TTS
|
||||
|
||||
229
core/http/endpoints/localai/voice_profiles.go
Normal file
229
core/http/endpoints/localai/voice_profiles.go
Normal file
@@ -0,0 +1,229 @@
|
||||
package localai
|
||||
|
||||
import (
|
||||
"encoding/base64"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
"github.com/mudler/LocalAI/core/schema"
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
const maxVoiceProfileJSONBytes = voiceprofile.MaxAudioBytes*4/3 + 2*1024*1024
|
||||
|
||||
// VoiceProfileListResponse is returned by the profile library endpoint.
|
||||
type VoiceProfileListResponse struct {
|
||||
Data []voiceprofile.Profile `json:"data"`
|
||||
}
|
||||
|
||||
// CreateVoiceProfileRequest is the JSON alternative to multipart creation.
|
||||
// It is primarily used by the LocalAI admin MCP client; the browser sends a
|
||||
// multipart audio field to avoid base64 overhead.
|
||||
type CreateVoiceProfileRequest struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
Transcript string `json:"transcript"`
|
||||
AudioBase64 string `json:"audio_base64"`
|
||||
ConsentConfirmed bool `json:"consent_confirmed"`
|
||||
}
|
||||
|
||||
func voiceProfileError(code int, message string) schema.ErrorResponse {
|
||||
return schema.ErrorResponse{Error: &schema.APIError{
|
||||
Code: code,
|
||||
Message: message,
|
||||
Type: "voice_profile_error",
|
||||
}}
|
||||
}
|
||||
|
||||
func writeVoiceProfileError(c echo.Context, err error) error {
|
||||
switch {
|
||||
case errors.Is(err, voiceprofile.ErrNotFound):
|
||||
return c.JSON(http.StatusNotFound, voiceProfileError(http.StatusNotFound, "voice profile not found"))
|
||||
case errors.Is(err, voiceprofile.ErrAudioTooLarge):
|
||||
return c.JSON(http.StatusRequestEntityTooLarge, voiceProfileError(http.StatusRequestEntityTooLarge, err.Error()))
|
||||
case errors.Is(err, voiceprofile.ErrInvalidInput),
|
||||
errors.Is(err, voiceprofile.ErrConsentRequired),
|
||||
errors.Is(err, voiceprofile.ErrUnsupportedWAV):
|
||||
return c.JSON(http.StatusBadRequest, voiceProfileError(http.StatusBadRequest, err.Error()))
|
||||
default:
|
||||
xlog.Error("Voice profile operation failed", "error", err)
|
||||
return c.JSON(http.StatusInternalServerError, voiceProfileError(http.StatusInternalServerError, "voice profile operation failed"))
|
||||
}
|
||||
}
|
||||
|
||||
func isRequestBodyTooLarge(err error) bool {
|
||||
var maxBytesError *http.MaxBytesError
|
||||
if errors.As(err, &maxBytesError) {
|
||||
return true
|
||||
}
|
||||
var httpError *echo.HTTPError
|
||||
return errors.As(err, &httpError) && errors.As(httpError.Internal, &maxBytesError)
|
||||
}
|
||||
|
||||
// ListVoiceProfilesEndpoint lists reusable cloning references. The route is
|
||||
// available to users with the Audio Speech/TTS feature so they can select a
|
||||
// profile; profile mutation remains admin-only.
|
||||
//
|
||||
// @Summary List voice profiles
|
||||
// @Description List saved voice-cloning references without exposing filesystem paths.
|
||||
// @Tags audio,voice-profiles
|
||||
// @Produce json
|
||||
// @Success 200 {object} VoiceProfileListResponse
|
||||
// @Failure 500 {object} schema.ErrorResponse
|
||||
// @Router /api/voice-profiles [get]
|
||||
func ListVoiceProfilesEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
if store == nil {
|
||||
return writeVoiceProfileError(c, errors.New("voice profile store is unavailable"))
|
||||
}
|
||||
profiles, err := store.List(c.Request().Context())
|
||||
if err != nil {
|
||||
return writeVoiceProfileError(c, err)
|
||||
}
|
||||
return c.JSON(http.StatusOK, VoiceProfileListResponse{Data: profiles})
|
||||
}
|
||||
}
|
||||
|
||||
// CreateVoiceProfileEndpoint validates and persists a reusable cloning
|
||||
// reference. It accepts multipart/form-data (audio field) or JSON with a
|
||||
// base64-encoded 16-bit PCM WAV. Admin-only.
|
||||
//
|
||||
// @Summary Create a voice profile
|
||||
// @Description Save a consent-confirmed PCM WAV reference clip and exact transcript for voice cloning. Admin-only.
|
||||
// @Tags audio,voice-profiles
|
||||
// @Accept multipart/form-data
|
||||
// @Accept json
|
||||
// @Produce json
|
||||
// @Param name formData string true "Display name"
|
||||
// @Param description formData string false "Optional description"
|
||||
// @Param language formData string false "Optional language tag"
|
||||
// @Param transcript formData string true "Exact transcript of the reference clip"
|
||||
// @Param consent_confirmed formData bool true "Confirms authorization to clone the voice"
|
||||
// @Param audio formData file true "16-bit PCM WAV, preferably mono 24 kHz, 1-120 seconds, up to 50 MiB"
|
||||
// @Success 201 {object} voiceprofile.Profile
|
||||
// @Failure 400 {object} schema.ErrorResponse
|
||||
// @Failure 413 {object} schema.ErrorResponse
|
||||
// @Router /api/voice-profiles [post]
|
||||
func CreateVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
if store == nil {
|
||||
return writeVoiceProfileError(c, errors.New("voice profile store is unavailable"))
|
||||
}
|
||||
|
||||
contentType := c.Request().Header.Get(echo.HeaderContentType)
|
||||
if strings.HasPrefix(contentType, echo.MIMEMultipartForm) {
|
||||
c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, voiceprofile.MaxAudioBytes+2*1024*1024)
|
||||
fileHeader, err := c.FormFile("audio")
|
||||
if err != nil {
|
||||
if isRequestBodyTooLarge(err) {
|
||||
return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
|
||||
}
|
||||
return writeVoiceProfileError(c, fmt.Errorf("%w: audio is required", voiceprofile.ErrInvalidInput))
|
||||
}
|
||||
if fileHeader.Size > voiceprofile.MaxAudioBytes {
|
||||
return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
|
||||
}
|
||||
audio, err := fileHeader.Open()
|
||||
if err != nil {
|
||||
return writeVoiceProfileError(c, fmt.Errorf("open uploaded audio: %w", err))
|
||||
}
|
||||
defer func() { _ = audio.Close() }()
|
||||
|
||||
consent, _ := strconv.ParseBool(c.FormValue("consent_confirmed"))
|
||||
profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{
|
||||
Name: c.FormValue("name"),
|
||||
Description: c.FormValue("description"),
|
||||
Language: c.FormValue("language"),
|
||||
Transcript: c.FormValue("transcript"),
|
||||
ConsentConfirmed: consent,
|
||||
}, audio)
|
||||
if err != nil {
|
||||
return writeVoiceProfileError(c, err)
|
||||
}
|
||||
return c.JSON(http.StatusCreated, profile)
|
||||
}
|
||||
|
||||
c.Request().Body = http.MaxBytesReader(c.Response().Writer, c.Request().Body, maxVoiceProfileJSONBytes)
|
||||
var request CreateVoiceProfileRequest
|
||||
if err := c.Bind(&request); err != nil {
|
||||
if isRequestBodyTooLarge(err) {
|
||||
return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
|
||||
}
|
||||
return writeVoiceProfileError(c, fmt.Errorf("%w: invalid JSON body", voiceprofile.ErrInvalidInput))
|
||||
}
|
||||
if base64.StdEncoding.DecodedLen(len(request.AudioBase64)) > int(voiceprofile.MaxAudioBytes) {
|
||||
return writeVoiceProfileError(c, voiceprofile.ErrAudioTooLarge)
|
||||
}
|
||||
audio := base64.NewDecoder(base64.StdEncoding, strings.NewReader(request.AudioBase64))
|
||||
profile, err := store.Create(c.Request().Context(), voiceprofile.CreateInput{
|
||||
Name: request.Name,
|
||||
Description: request.Description,
|
||||
Language: request.Language,
|
||||
Transcript: request.Transcript,
|
||||
ConsentConfirmed: request.ConsentConfirmed,
|
||||
}, audio)
|
||||
if err != nil {
|
||||
return writeVoiceProfileError(c, err)
|
||||
}
|
||||
return c.JSON(http.StatusCreated, profile)
|
||||
}
|
||||
}
|
||||
|
||||
// ServeVoiceProfileAudioEndpoint streams an authenticated preview with range
|
||||
// request support. The response is private and never cacheable by shared
|
||||
// proxies because the clip is biometric source material.
|
||||
//
|
||||
// @Summary Preview voice profile audio
|
||||
// @Description Stream the saved reference WAV for an authenticated TTS user.
|
||||
// @Tags audio,voice-profiles
|
||||
// @Produce audio/x-wav
|
||||
// @Param id path string true "Voice profile UUID"
|
||||
// @Success 200 {string} binary
|
||||
// @Failure 404 {object} schema.ErrorResponse
|
||||
// @Router /api/voice-profiles/{id}/audio [get]
|
||||
func ServeVoiceProfileAudioEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
if store == nil {
|
||||
return writeVoiceProfileError(c, errors.New("voice profile store is unavailable"))
|
||||
}
|
||||
file, profile, err := store.OpenAudio(c.Request().Context(), c.Param("id"))
|
||||
if err != nil {
|
||||
return writeVoiceProfileError(c, err)
|
||||
}
|
||||
defer func() { _ = file.Close() }()
|
||||
|
||||
c.Response().Header().Set(echo.HeaderContentType, "audio/wav")
|
||||
c.Response().Header().Set(echo.HeaderCacheControl, "private, no-store")
|
||||
c.Response().Header().Set("X-Content-Type-Options", "nosniff")
|
||||
c.Response().Header().Set(echo.HeaderContentDisposition, `inline; filename="reference.wav"`)
|
||||
http.ServeContent(c.Response().Writer, c.Request(), "reference.wav", profile.UpdatedAt, file)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// DeleteVoiceProfileEndpoint permanently removes a profile. Admin-only.
|
||||
//
|
||||
// @Summary Delete a voice profile
|
||||
// @Description Permanently remove a saved voice-cloning profile. Admin-only.
|
||||
// @Tags audio,voice-profiles
|
||||
// @Param id path string true "Voice profile UUID"
|
||||
// @Success 204
|
||||
// @Failure 404 {object} schema.ErrorResponse
|
||||
// @Router /api/voice-profiles/{id} [delete]
|
||||
func DeleteVoiceProfileEndpoint(store *voiceprofile.Store) echo.HandlerFunc {
|
||||
return func(c echo.Context) error {
|
||||
if store == nil {
|
||||
return writeVoiceProfileError(c, errors.New("voice profile store is unavailable"))
|
||||
}
|
||||
if err := store.Delete(c.Request().Context(), c.Param("id")); err != nil {
|
||||
return writeVoiceProfileError(c, err)
|
||||
}
|
||||
return c.NoContent(http.StatusNoContent)
|
||||
}
|
||||
}
|
||||
135
core/http/endpoints/localai/voice_profiles_test.go
Normal file
135
core/http/endpoints/localai/voice_profiles_test.go
Normal file
@@ -0,0 +1,135 @@
|
||||
package localai_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/json"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"time"
|
||||
|
||||
"github.com/labstack/echo/v4"
|
||||
. "github.com/mudler/LocalAI/core/http/endpoints/localai"
|
||||
"github.com/mudler/LocalAI/core/schema"
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func voiceProfileWAV(duration time.Duration) []byte {
|
||||
const sampleRate = 16000
|
||||
samples := int(duration.Seconds() * sampleRate)
|
||||
dataSize := samples * 2
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 44+dataSize))
|
||||
buf.WriteString("RIFF")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
|
||||
buf.WriteString("WAVEfmt ")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(16))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(1))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(1))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*2))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(2))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(16))
|
||||
buf.WriteString("data")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
|
||||
buf.Write(make([]byte, dataSize))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
func voiceProfileMultipart(consent string) (*bytes.Buffer, string) {
|
||||
body := &bytes.Buffer{}
|
||||
writer := multipart.NewWriter(body)
|
||||
Expect(writer.WriteField("name", "Documentary narrator")).To(Succeed())
|
||||
Expect(writer.WriteField("description", "Measured and clear")).To(Succeed())
|
||||
Expect(writer.WriteField("language", "en-US")).To(Succeed())
|
||||
Expect(writer.WriteField("transcript", "The exact words spoken in this reference.")).To(Succeed())
|
||||
Expect(writer.WriteField("consent_confirmed", consent)).To(Succeed())
|
||||
part, err := writer.CreateFormFile("audio", "reference.wav")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
_, err = part.Write(voiceProfileWAV(2 * time.Second))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(writer.Close()).To(Succeed())
|
||||
return body, writer.FormDataContentType()
|
||||
}
|
||||
|
||||
var _ = Describe("Voice profile endpoints", func() {
|
||||
var (
|
||||
e *echo.Echo
|
||||
store *voiceprofile.Store
|
||||
)
|
||||
|
||||
BeforeEach(func() {
|
||||
store = voiceprofile.NewStore(GinkgoT().TempDir())
|
||||
DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
|
||||
e = echo.New()
|
||||
e.GET("/api/voice-profiles", ListVoiceProfilesEndpoint(store))
|
||||
e.POST("/api/voice-profiles", CreateVoiceProfileEndpoint(store))
|
||||
e.GET("/api/voice-profiles/:id/audio", ServeVoiceProfileAudioEndpoint(store))
|
||||
e.DELETE("/api/voice-profiles/:id", DeleteVoiceProfileEndpoint(store))
|
||||
})
|
||||
|
||||
It("creates, lists, previews with ranges, and deletes a multipart profile", func() {
|
||||
body, contentType := voiceProfileMultipart("true")
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", body)
|
||||
request.Header.Set(echo.HeaderContentType, contentType)
|
||||
recorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String())
|
||||
|
||||
var created voiceprofile.Profile
|
||||
Expect(json.Unmarshal(recorder.Body.Bytes(), &created)).To(Succeed())
|
||||
Expect(created.Name).To(Equal("Documentary narrator"))
|
||||
Expect(created.Voice).To(Equal(voiceprofile.Reference(created.ID)))
|
||||
|
||||
request = httptest.NewRequest(http.MethodGet, "/api/voice-profiles", nil)
|
||||
recorder = httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusOK))
|
||||
var listed VoiceProfileListResponse
|
||||
Expect(json.Unmarshal(recorder.Body.Bytes(), &listed)).To(Succeed())
|
||||
Expect(listed.Data).To(ConsistOf(created))
|
||||
|
||||
request = httptest.NewRequest(http.MethodGet, "/api/voice-profiles/"+created.ID+"/audio", nil)
|
||||
request.Header.Set("Range", "bytes=0-3")
|
||||
recorder = httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusPartialContent))
|
||||
Expect(recorder.Body.String()).To(Equal("RIFF"))
|
||||
Expect(recorder.Header().Get(echo.HeaderCacheControl)).To(Equal("private, no-store"))
|
||||
|
||||
request = httptest.NewRequest(http.MethodDelete, "/api/voice-profiles/"+created.ID, nil)
|
||||
recorder = httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusNoContent))
|
||||
})
|
||||
|
||||
It("accepts the JSON base64 shape used by the admin MCP client", func() {
|
||||
payload, err := json.Marshal(CreateVoiceProfileRequest{
|
||||
Name: "MCP voice",
|
||||
Transcript: "An exact reference transcript.",
|
||||
AudioBase64: base64.StdEncoding.EncodeToString(voiceProfileWAV(time.Second)),
|
||||
ConsentConfirmed: true,
|
||||
})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", bytes.NewReader(payload))
|
||||
request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
|
||||
recorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusCreated), recorder.Body.String())
|
||||
})
|
||||
|
||||
It("rejects creation without explicit consent", func() {
|
||||
body, contentType := voiceProfileMultipart("false")
|
||||
request := httptest.NewRequest(http.MethodPost, "/api/voice-profiles", body)
|
||||
request.Header.Set(echo.HeaderContentType, contentType)
|
||||
recorder := httptest.NewRecorder()
|
||||
e.ServeHTTP(recorder, request)
|
||||
Expect(recorder.Code).To(Equal(http.StatusBadRequest))
|
||||
var response schema.ErrorResponse
|
||||
Expect(json.Unmarshal(recorder.Body.Bytes(), &response)).To(Succeed())
|
||||
Expect(response.Error.Message).To(ContainSubstring("consent"))
|
||||
})
|
||||
})
|
||||
@@ -101,6 +101,18 @@ func (stubClient) SetBranding(_ context.Context, _ localaitools.SetBrandingReque
|
||||
return &localaitools.Branding{InstanceName: "LocalAI"}, nil
|
||||
}
|
||||
|
||||
func (stubClient) ListVoiceProfiles(_ context.Context) ([]localaitools.VoiceProfile, error) {
|
||||
return []localaitools.VoiceProfile{}, nil
|
||||
}
|
||||
|
||||
func (stubClient) CreateVoiceProfile(_ context.Context, _ localaitools.CreateVoiceProfileRequest) (*localaitools.VoiceProfile, error) {
|
||||
return &localaitools.VoiceProfile{Name: "stub-voice"}, nil
|
||||
}
|
||||
|
||||
func (stubClient) DeleteVoiceProfile(_ context.Context, _ string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (stubClient) GetUsageStats(_ context.Context, _ localaitools.UsageStatsQuery) (*localaitools.UsageStats, error) {
|
||||
return &localaitools.UsageStats{Viewer: localaitools.UsageViewer{ID: "stub", Name: "stub"}, Period: "month"}, nil
|
||||
}
|
||||
|
||||
@@ -26,6 +26,8 @@ const PAGES = [
|
||||
['/app/voice', 'Voice recognition'],
|
||||
['/app/fine-tune', 'Fine-tuning'],
|
||||
['/app/quantize', 'Quantize'],
|
||||
['/app/voice-library', 'Voice Library'],
|
||||
['/app/voice-library/new', 'Create voice profile'],
|
||||
]
|
||||
|
||||
test.describe('Page render smoke', () => {
|
||||
|
||||
175
core/http/react-ui/e2e/voice-library.spec.js
Normal file
175
core/http/react-ui/e2e/voice-library.spec.js
Normal file
@@ -0,0 +1,175 @@
|
||||
import { test, expect } from './coverage-fixtures.js'
|
||||
|
||||
const VOICE_ID = '00000000-0000-0000-0000-000000000001'
|
||||
|
||||
function pcmWav(seconds = 2) {
|
||||
const sampleRate = 16000
|
||||
const dataSize = sampleRate * seconds * 2
|
||||
const buffer = Buffer.alloc(44 + dataSize)
|
||||
buffer.write('RIFF', 0)
|
||||
buffer.writeUInt32LE(36 + dataSize, 4)
|
||||
buffer.write('WAVEfmt ', 8)
|
||||
buffer.writeUInt32LE(16, 16)
|
||||
buffer.writeUInt16LE(1, 20)
|
||||
buffer.writeUInt16LE(1, 22)
|
||||
buffer.writeUInt32LE(sampleRate, 24)
|
||||
buffer.writeUInt32LE(sampleRate * 2, 28)
|
||||
buffer.writeUInt16LE(2, 32)
|
||||
buffer.writeUInt16LE(16, 34)
|
||||
buffer.write('data', 36)
|
||||
buffer.writeUInt32LE(dataSize, 40)
|
||||
return buffer
|
||||
}
|
||||
|
||||
const profile = {
|
||||
id: VOICE_ID,
|
||||
name: 'Documentary narrator',
|
||||
description: 'Measured and clear',
|
||||
language: 'en-US',
|
||||
transcript: 'The exact words spoken in this reference.',
|
||||
voice: `localai://voice-profiles/${VOICE_ID}`,
|
||||
consent_confirmed_at: '2026-07-01T12:00:00Z',
|
||||
created_at: '2026-07-01T12:00:00Z',
|
||||
updated_at: '2026-07-01T12:00:00Z',
|
||||
audio: { duration_ms: 2000, sample_rate: 16000, channels: 1, bit_depth: 16, size_bytes: 64044, mime_type: 'audio/wav' },
|
||||
}
|
||||
|
||||
async function mockVoiceAPIs(page) {
|
||||
const state = { createdMultipart: null }
|
||||
await page.route('**/api/models/capabilities', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ data: [
|
||||
{ id: 'qwen-base', capabilities: ['FLAG_TTS'], backend: 'qwen3-tts-cpp', voice_cloning: { reference_transcript_required: true, accepted_audio_formats: ['audio/wav'] } },
|
||||
{ id: 'piper-default', capabilities: ['FLAG_TTS'], backend: 'piper' },
|
||||
] }),
|
||||
}))
|
||||
await page.route('**/api/voice-profiles', async route => {
|
||||
if (route.request().method() === 'POST') {
|
||||
state.createdMultipart = route.request().postDataBuffer()
|
||||
await route.fulfill({ status: 201, contentType: 'application/json', body: JSON.stringify(profile) })
|
||||
return
|
||||
}
|
||||
await route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ data: [profile] }) })
|
||||
})
|
||||
await page.route(`**/api/voice-profiles/${VOICE_ID}/audio`, route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'audio/wav',
|
||||
body: pcmWav(),
|
||||
}))
|
||||
return state
|
||||
}
|
||||
|
||||
test.describe('Voice Library', () => {
|
||||
let apiState
|
||||
|
||||
test.beforeEach(async ({ page }) => {
|
||||
apiState = await mockVoiceAPIs(page)
|
||||
})
|
||||
|
||||
test('renders the library-first master/detail view', async ({ page }) => {
|
||||
await page.goto('/app/voice-library')
|
||||
await expect(page.getByRole('heading', { name: /Voice Library/i })).toBeVisible()
|
||||
await expect(page.locator('.voice-row', { hasText: 'Documentary narrator' })).toBeVisible()
|
||||
await expect(page.locator('.voice-library-detail')).toContainText('The exact words spoken in this reference.')
|
||||
await expect(page.locator('.voice-library-detail')).toContainText('Consent confirmed')
|
||||
await expect(page.getByRole('button', { name: /Use in Text to Speech/i })).toBeEnabled()
|
||||
})
|
||||
|
||||
test('shows inline API usage and installed model compatibility', async ({ page }) => {
|
||||
await page.goto('/app/voice-library')
|
||||
await page.getByText('API usage and compatible models').click()
|
||||
const apiHelp = page.locator('.voice-detail__api')
|
||||
await expect(apiHelp).toContainText('qwen-base')
|
||||
await expect(apiHelp).toContainText('qwen3-tts-cpp')
|
||||
await expect(apiHelp.locator('code')).toContainText('/v1/audio/speech')
|
||||
await expect(apiHelp.locator('code')).toContainText(`localai://voice-profiles/${VOICE_ID}`)
|
||||
})
|
||||
|
||||
test('offers server-declared gallery models when none are installed', async ({ page }) => {
|
||||
let galleryCapability = null
|
||||
let installedModel = null
|
||||
await page.route('**/api/models/capabilities', route => route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ data: [] }),
|
||||
}))
|
||||
await page.route('**/api/models?**', route => {
|
||||
galleryCapability = new URL(route.request().url()).searchParams.get('capability')
|
||||
return route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'application/json',
|
||||
body: JSON.stringify({ models: [{
|
||||
id: 'localai@omnivoice-cpp',
|
||||
name: 'omnivoice-cpp',
|
||||
backend: 'omnivoice-cpp',
|
||||
installed: false,
|
||||
voice_cloning: { reference_transcript_required: true, accepted_audio_formats: ['audio/wav'] },
|
||||
}] }),
|
||||
})
|
||||
})
|
||||
await page.route('**/api/models/install/**', route => {
|
||||
installedModel = decodeURIComponent(route.request().url().split('/').pop())
|
||||
return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify({ jobID: 'voice-install' }) })
|
||||
})
|
||||
|
||||
await page.goto('/app/voice-library')
|
||||
await expect(page.getByRole('heading', { name: 'Install a voice-cloning model' })).toBeVisible()
|
||||
await expect(page.locator('.voice-detail__model-list')).toContainText('omnivoice-cpp')
|
||||
expect(galleryCapability).toBe('voice_cloning')
|
||||
await page.locator('.voice-detail__model-list').getByRole('button', { name: 'Install' }).click()
|
||||
await expect.poll(() => installedModel).toBe('localai@omnivoice-cpp')
|
||||
await expect(page.getByText(/Installing omnivoice-cpp/)).toBeVisible()
|
||||
})
|
||||
|
||||
test('keeps the Operate rail compact and accessible on small screens', async ({ page }) => {
|
||||
await page.setViewportSize({ width: 390, height: 844 })
|
||||
await page.goto('/app/voice-library')
|
||||
const rail = page.locator('.console-rail')
|
||||
await expect(rail.getByRole('link', { name: 'Backends' })).toBeHidden()
|
||||
await rail.getByRole('button', { name: 'Expand Operate navigation' }).click()
|
||||
await expect(rail.getByRole('link', { name: 'Backends' })).toBeVisible()
|
||||
await expect(rail.getByRole('button', { name: 'Collapse Operate navigation' })).toBeVisible()
|
||||
})
|
||||
|
||||
test('passes the stable voice URI from the library into TTS', async ({ page }) => {
|
||||
let ttsBody = null
|
||||
await page.route('**/tts', async route => {
|
||||
ttsBody = route.request().postDataJSON()
|
||||
await route.fulfill({
|
||||
status: 200,
|
||||
contentType: 'audio/wav',
|
||||
headers: { 'Content-Disposition': 'attachment; filename="speech.wav"' },
|
||||
body: pcmWav(1),
|
||||
})
|
||||
})
|
||||
|
||||
await page.goto(`/app/tts?voice=${VOICE_ID}`)
|
||||
await expect(page.locator('#tts-voice')).toHaveValue(VOICE_ID)
|
||||
await page.getByPlaceholder('Enter text to synthesize...').fill('Hello from the saved voice.')
|
||||
await page.getByRole('button', { name: /Generate$/ }).click()
|
||||
await expect.poll(() => ttsBody?.voice).toBe(`localai://voice-profiles/${VOICE_ID}`)
|
||||
expect(ttsBody.model).toBe('qwen-base')
|
||||
})
|
||||
|
||||
test('normalizes an upload and creates a consented profile', async ({ page }) => {
|
||||
await page.goto('/app/voice-library/new')
|
||||
await page.locator('#voice-profile-audio-file').setInputFiles({
|
||||
name: 'reference.wav',
|
||||
mimeType: 'audio/wav',
|
||||
buffer: pcmWav(2),
|
||||
})
|
||||
await expect(page.getByText('Normalized reference')).toBeVisible()
|
||||
await page.locator('#voice-profile-name').fill('Documentary narrator')
|
||||
await page.locator('#voice-profile-transcript').fill('The exact words spoken in this reference.')
|
||||
await page.getByText('I confirm that this voice may be cloned').click()
|
||||
await page.getByRole('button', { name: /Save voice/i }).click()
|
||||
await expect(page).toHaveURL(new RegExp(`/app/voice-library\\?selected=${VOICE_ID}$`))
|
||||
|
||||
const wavOffset = apiState.createdMultipart.indexOf(Buffer.from('RIFF'))
|
||||
expect(wavOffset).toBeGreaterThanOrEqual(0)
|
||||
expect(apiState.createdMultipart.readUInt16LE(wavOffset + 22)).toBe(1)
|
||||
expect(apiState.createdMultipart.readUInt32LE(wavOffset + 24)).toBe(24000)
|
||||
expect(apiState.createdMultipart.readUInt16LE(wavOffset + 34)).toBe(16)
|
||||
})
|
||||
})
|
||||
@@ -75,7 +75,7 @@
|
||||
"labels": {
|
||||
"model": "Model",
|
||||
"voice": "Voice",
|
||||
"voicePlaceholder": "Optional voice ID",
|
||||
"voicePlaceholder": "Optional speaker or voice ID",
|
||||
"input": "Text",
|
||||
"inputPlaceholder": "Enter text to synthesize..."
|
||||
},
|
||||
@@ -87,7 +87,175 @@
|
||||
"toasts": {
|
||||
"noText": "Please enter text",
|
||||
"noModel": "Please select a model",
|
||||
"generated": "Speech generated",
|
||||
"generateFailed": "Generation failed"
|
||||
},
|
||||
"voiceLibrary": {
|
||||
"cloningReady": "Voice cloning",
|
||||
"loading": "Loading saved voices…",
|
||||
"modelDefault": "Use model default",
|
||||
"empty": "No saved voices yet.",
|
||||
"create": "Create one",
|
||||
"manage": "Manage Voice Library",
|
||||
"namedVoiceHint": "This model does not accept reference-audio profiles. You can still enter a named speaker supported by its backend."
|
||||
}
|
||||
},
|
||||
"voiceLibrary": {
|
||||
"title": "Voice Library",
|
||||
"subtitle": "Create and manage reusable reference voices for every installed model that supports voice cloning.",
|
||||
"loading": "Loading saved voices…",
|
||||
"listLabel": "Saved voice profiles",
|
||||
"status": {
|
||||
"ready": "Ready"
|
||||
},
|
||||
"summary": {
|
||||
"label": "Voice library status",
|
||||
"profiles": "saved voices",
|
||||
"modelsReady_one": "{{count}} compatible model ready",
|
||||
"modelsReady_other": "{{count}} compatible models ready",
|
||||
"noModels": "No compatible model installed"
|
||||
},
|
||||
"search": {
|
||||
"label": "Search voice profiles",
|
||||
"placeholder": "Search voices, languages, or transcripts"
|
||||
},
|
||||
"filters": {
|
||||
"language": "Filter by language",
|
||||
"allLanguages": "All languages"
|
||||
},
|
||||
"empty": {
|
||||
"title": "Your voice library is empty",
|
||||
"body": "Record or upload one consented reference clip, add its exact transcript, and reuse it across compatible TTS models."
|
||||
},
|
||||
"noResults": {
|
||||
"title": "No voices match these filters",
|
||||
"body": "Try a different search or show every language."
|
||||
},
|
||||
"detail": {
|
||||
"eyebrow": "Selected voice",
|
||||
"emptyTitle": "Select a voice to inspect it",
|
||||
"emptyBody": "Reference audio, transcript, compatibility, and consent details will appear here.",
|
||||
"referenceAudio": "Reference audio",
|
||||
"transcript": "Reference transcript"
|
||||
},
|
||||
"metadata": {
|
||||
"language": "Language",
|
||||
"languageUnknown": "Not specified",
|
||||
"duration": "Duration",
|
||||
"sampleRate": "Sample rate",
|
||||
"created": "Created"
|
||||
},
|
||||
"consent": {
|
||||
"confirmed": "Consent confirmed",
|
||||
"confirmedAt": "Confirmed {{date}}"
|
||||
},
|
||||
"modelSetup": {
|
||||
"title": "Install a voice-cloning model",
|
||||
"body": "This voice is ready, but it needs a model that accepts reference audio. Choose one verified by this server.",
|
||||
"loading": "Finding compatible models in your galleries…",
|
||||
"backendUnknown": "Backend selected during install",
|
||||
"install": "Install",
|
||||
"installing": "Installing…",
|
||||
"loadFailed": "Compatible recommendations could not be loaded.",
|
||||
"noneAvailable": "No compatible uninstalled model is available in the configured galleries.",
|
||||
"capabilityNote": "Recommendations come from model capability metadata, not a browser-side model list."
|
||||
},
|
||||
"api": {
|
||||
"title": "API usage and compatible models",
|
||||
"summary": "Use this voice outside the browser or check model support",
|
||||
"body": "Send the profile’s stable voice URI with each speech request. LocalAI resolves the private reference audio and transcript at generation time; model YAML is not modified.",
|
||||
"compatibleModels": "Compatible installed models",
|
||||
"noInstalledModels": "None yet. Install one of the server-verified models above before sending this request.",
|
||||
"curlExample": "cURL example",
|
||||
"copy": "Copy request",
|
||||
"endpointNote": "The same model, input, and voice fields also work with POST /tts."
|
||||
},
|
||||
"actions": {
|
||||
"create": "Create voice",
|
||||
"createFirst": "Create your first voice",
|
||||
"retry": "Try again",
|
||||
"clearFilters": "Clear filters",
|
||||
"useInTTS": "Use in Text to Speech",
|
||||
"delete": "Delete voice",
|
||||
"installModel": "Install a compatible model",
|
||||
"browseModels": "Browse all models"
|
||||
},
|
||||
"deleteDialog": {
|
||||
"title": "Delete this voice?",
|
||||
"message": "“{{name}}” and its reference audio will be permanently removed. Existing model YAML is not changed.",
|
||||
"deleting": "Deleting…"
|
||||
},
|
||||
"toasts": {
|
||||
"deleted": "Deleted {{name}}",
|
||||
"installStarted": "Installing {{name}}. Progress is available in Operations.",
|
||||
"installFailed": "Could not start installation: {{message}}",
|
||||
"apiCopied": "API request copied",
|
||||
"copyFailed": "Could not copy the API request"
|
||||
}
|
||||
},
|
||||
"voiceCreate": {
|
||||
"eyebrow": "Voice Library",
|
||||
"title": "Create a reusable voice",
|
||||
"subtitle": "Add one clean reference clip and its exact transcript. LocalAI handles the backend-specific cloning parameters at generation time.",
|
||||
"sections": {
|
||||
"reference": {
|
||||
"title": "Reference audio",
|
||||
"body": "Record in the browser or upload an existing clip. Speech should be clean, natural, and from one speaker."
|
||||
},
|
||||
"details": {
|
||||
"title": "Identity and transcript",
|
||||
"body": "Give admins a recognizable name and transcribe the clip exactly, including fillers and punctuation."
|
||||
},
|
||||
"consent": {
|
||||
"title": "Authorization",
|
||||
"body": "Voice cloning can be sensitive. Confirm permission before this reference is saved."
|
||||
}
|
||||
},
|
||||
"audio": {
|
||||
"label": "Voice reference",
|
||||
"normalizing": "Preparing a secure PCM WAV…",
|
||||
"preview": "Normalized reference",
|
||||
"durationError": "Reference audio must be between 1 second and 2 minutes.",
|
||||
"decodeError": "This browser could not decode the selected audio file.",
|
||||
"qualityReady": "Recommended length",
|
||||
"qualityHint": "6–30 seconds usually produces a stronger clone"
|
||||
},
|
||||
"fields": {
|
||||
"name": "Voice name",
|
||||
"namePlaceholder": "e.g. Documentary narrator",
|
||||
"language": "Language (optional)",
|
||||
"languagePlaceholder": "e.g. en-US",
|
||||
"description": "Description (optional)",
|
||||
"descriptionPlaceholder": "Tone, source, or intended use",
|
||||
"transcript": "Exact transcript",
|
||||
"transcriptPlaceholder": "Type every word spoken in the reference clip…",
|
||||
"transcriptHint": "Match the recording exactly. Do not rewrite grammar or remove pauses and fillers."
|
||||
},
|
||||
"consent": {
|
||||
"title": "I confirm that this voice may be cloned",
|
||||
"body": "I have the speaker’s permission or another lawful basis to store and use this recording for synthesized speech."
|
||||
},
|
||||
"readiness": {
|
||||
"title": "Profile readiness",
|
||||
"body": "A voice becomes available in Text to Speech when every required item is complete.",
|
||||
"audio": "Valid PCM-WAV reference",
|
||||
"quality": "6–30 second quality window",
|
||||
"name": "Recognizable voice name",
|
||||
"transcript": "Exact reference transcript",
|
||||
"consent": "Authorization confirmed"
|
||||
},
|
||||
"privacy": {
|
||||
"title": "Private by default. ",
|
||||
"body": "Reference audio is stored in LocalAI’s data directory with restricted file permissions and is never exposed as a model-folder path."
|
||||
},
|
||||
"actions": {
|
||||
"back": "Back to library",
|
||||
"cancel": "Cancel",
|
||||
"save": "Save voice",
|
||||
"saving": "Saving…"
|
||||
},
|
||||
"toasts": {
|
||||
"created": "{{name}} is ready to use"
|
||||
}
|
||||
},
|
||||
"sound": {
|
||||
|
||||
@@ -49,6 +49,7 @@
|
||||
"users": "Users",
|
||||
"middleware": "Middleware",
|
||||
"backends": "Backends",
|
||||
"voiceLibrary": "Voice Library",
|
||||
"traces": "Traces",
|
||||
"nodes": "Nodes",
|
||||
"scheduling": "Scheduling",
|
||||
@@ -65,6 +66,8 @@
|
||||
},
|
||||
"console": {
|
||||
"automation": "Automation",
|
||||
"training": "Training"
|
||||
"training": "Training",
|
||||
"expandNavigation": "Expand {{section}} navigation",
|
||||
"collapseNavigation": "Collapse {{section}} navigation"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8560,13 +8560,28 @@ button.collapsible-header:focus-visible {
|
||||
.console-rail-header {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-sm);
|
||||
padding: var(--spacing-sm) var(--spacing-sm) var(--spacing-xs);
|
||||
font-size: var(--text-sm);
|
||||
font-weight: var(--font-weight-semibold);
|
||||
color: var(--color-text-primary);
|
||||
}
|
||||
.console-rail-header i { color: var(--color-primary); font-size: 0.9rem; }
|
||||
.console-rail-header__title { display: inline-flex; align-items: center; gap: var(--spacing-sm); }
|
||||
.console-rail-header__title i { color: var(--color-primary); font-size: 0.9rem; }
|
||||
.console-rail-toggle {
|
||||
display: none;
|
||||
width: 34px;
|
||||
height: 34px;
|
||||
place-items: center;
|
||||
border: 1px solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--color-surface-elevated);
|
||||
color: var(--color-text-secondary);
|
||||
cursor: pointer;
|
||||
}
|
||||
.console-rail-toggle:hover { border-color: var(--color-border-strong); color: var(--color-text-primary); }
|
||||
.console-rail-groups { display: flex; flex-direction: column; gap: var(--spacing-xs); }
|
||||
.console-group { display: flex; flex-direction: column; gap: 1px; }
|
||||
.console-group + .console-group {
|
||||
margin-top: var(--spacing-xs);
|
||||
@@ -8600,6 +8615,9 @@ button.collapsible-header:focus-visible {
|
||||
@media (max-width: 768px) {
|
||||
.console-layout { flex-direction: column; padding: var(--spacing-sm); }
|
||||
.console-rail { position: static; flex-basis: auto; width: 100%; }
|
||||
.console-rail-toggle { display: grid; }
|
||||
.console-rail-groups { display: none; }
|
||||
.console-rail--open .console-rail-groups { display: flex; }
|
||||
}
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
.console-rail.console-rail--enter { animation: none; }
|
||||
@@ -8620,6 +8638,480 @@ button.collapsible-header:focus-visible {
|
||||
.status-pill--info .status-pill__dot { background: var(--color-info); }
|
||||
.status-pill--muted .status-pill__dot { background: var(--color-text-muted); }
|
||||
|
||||
/* ──────────────────── Voice Library ──────────────────── */
|
||||
|
||||
.sr-only {
|
||||
position: absolute;
|
||||
width: 1px;
|
||||
height: 1px;
|
||||
padding: 0;
|
||||
margin: -1px;
|
||||
overflow: hidden;
|
||||
clip: rect(0, 0, 0, 0);
|
||||
white-space: nowrap;
|
||||
border: 0;
|
||||
}
|
||||
|
||||
.voice-library-page,
|
||||
.voice-create-page {
|
||||
width: 100%;
|
||||
max-width: var(--page-max-default);
|
||||
margin: 0 auto;
|
||||
padding: var(--spacing-md) var(--spacing-md) var(--spacing-xl);
|
||||
animation: fadeIn var(--duration-normal) var(--ease-default);
|
||||
}
|
||||
|
||||
.voice-library-page .page-header,
|
||||
.voice-create-page .page-header {
|
||||
margin-bottom: var(--spacing-lg);
|
||||
}
|
||||
|
||||
.voice-library-page .page-title i,
|
||||
.voice-create-page .page-title i {
|
||||
color: var(--color-primary);
|
||||
margin-right: var(--spacing-xs);
|
||||
}
|
||||
|
||||
.voice-library-summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-md);
|
||||
min-height: 36px;
|
||||
margin: calc(-1 * var(--spacing-sm)) 0 var(--spacing-md);
|
||||
color: var(--color-text-muted);
|
||||
font-size: var(--text-sm);
|
||||
}
|
||||
.voice-library-summary strong {
|
||||
color: var(--color-text-primary);
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--text-base);
|
||||
}
|
||||
.voice-library-summary__divider {
|
||||
width: 1px;
|
||||
height: 18px;
|
||||
background: var(--color-border-default);
|
||||
}
|
||||
|
||||
.voice-library-error {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-md);
|
||||
margin-bottom: var(--spacing-md);
|
||||
}
|
||||
|
||||
.voice-library-shell {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(300px, 0.78fr) minmax(430px, 1.35fr);
|
||||
min-height: 620px;
|
||||
overflow: hidden;
|
||||
background: var(--color-surface-raised);
|
||||
border: 1px solid var(--color-border-default);
|
||||
border-radius: var(--radius-xl);
|
||||
box-shadow: var(--shadow-sm), var(--shadow-inset-top);
|
||||
}
|
||||
|
||||
.voice-library-master {
|
||||
min-width: 0;
|
||||
border-right: 1px solid var(--color-border-divider);
|
||||
background: var(--color-bg-secondary);
|
||||
}
|
||||
|
||||
.voice-library-toolbar {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: var(--spacing-sm);
|
||||
padding: var(--spacing-md);
|
||||
border-bottom: 1px solid var(--color-border-divider);
|
||||
}
|
||||
.voice-library-search {
|
||||
position: relative;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
min-width: 0;
|
||||
}
|
||||
.voice-library-search i {
|
||||
position: absolute;
|
||||
left: 12px;
|
||||
color: var(--color-text-tertiary);
|
||||
font-size: var(--text-xs);
|
||||
pointer-events: none;
|
||||
}
|
||||
.voice-library-search input {
|
||||
width: 100%;
|
||||
height: 38px;
|
||||
padding: 0 12px 0 34px;
|
||||
color: var(--color-text-primary);
|
||||
background: var(--color-surface-sunken);
|
||||
border: 1px solid var(--color-border-default);
|
||||
border-radius: var(--radius-md);
|
||||
font: inherit;
|
||||
font-size: var(--text-sm);
|
||||
}
|
||||
.voice-library-search input::placeholder { color: var(--color-text-tertiary); }
|
||||
.voice-library-language { width: 138px; height: 38px; font-size: var(--text-sm); }
|
||||
|
||||
.voice-library-list {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
max-height: min(70vh, 760px);
|
||||
overflow-y: auto;
|
||||
overscroll-behavior: contain;
|
||||
}
|
||||
.voice-library-loading {
|
||||
min-height: 260px;
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: var(--spacing-sm);
|
||||
color: var(--color-text-muted);
|
||||
font-size: var(--text-sm);
|
||||
}
|
||||
.voice-library-empty { min-height: 430px; border: 0; background: transparent; }
|
||||
|
||||
.voice-row {
|
||||
appearance: none;
|
||||
width: 100%;
|
||||
display: grid;
|
||||
grid-template-columns: 38px minmax(0, 1fr) 14px;
|
||||
align-items: center;
|
||||
gap: var(--spacing-sm);
|
||||
padding: var(--spacing-md);
|
||||
color: var(--color-text-primary);
|
||||
text-align: left;
|
||||
background: transparent;
|
||||
border: 0;
|
||||
border-bottom: 1px solid var(--color-border-divider);
|
||||
cursor: pointer;
|
||||
transition: background var(--duration-fast) var(--ease-default), box-shadow var(--duration-fast) var(--ease-default);
|
||||
}
|
||||
.voice-row:hover { background: var(--color-surface-hover); }
|
||||
.voice-row--selected {
|
||||
background: var(--color-primary-light);
|
||||
box-shadow: inset 2px 0 0 var(--color-primary);
|
||||
}
|
||||
.voice-row--selected:hover { background: var(--color-primary-light); }
|
||||
.voice-row__avatar {
|
||||
width: 38px;
|
||||
height: 38px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
border-radius: var(--radius-full);
|
||||
background: var(--color-surface-elevated);
|
||||
border: 1px solid var(--color-border-default);
|
||||
color: var(--color-primary);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.65rem;
|
||||
font-weight: var(--font-weight-semibold);
|
||||
letter-spacing: 0.05em;
|
||||
}
|
||||
.voice-row--selected .voice-row__avatar {
|
||||
background: var(--color-primary);
|
||||
border-color: var(--color-primary);
|
||||
color: var(--color-primary-text);
|
||||
}
|
||||
.voice-row__content { min-width: 0; display: flex; flex-direction: column; gap: 5px; }
|
||||
.voice-row__head { min-width: 0; display: flex; align-items: baseline; justify-content: space-between; gap: var(--spacing-sm); }
|
||||
.voice-row__head strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: var(--text-sm); }
|
||||
.voice-row__head > span { flex-shrink: 0; color: var(--color-text-tertiary); font-family: var(--font-mono); font-size: 0.625rem; }
|
||||
.voice-row__waveform {
|
||||
height: 22px;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 2px;
|
||||
overflow: hidden;
|
||||
}
|
||||
.voice-row__waveform > span {
|
||||
flex: 1 1 auto;
|
||||
max-width: 3px;
|
||||
min-width: 1px;
|
||||
border-radius: 1px;
|
||||
background: var(--color-border-strong);
|
||||
}
|
||||
.voice-row--selected .voice-row__waveform > span { background: var(--color-primary); opacity: 0.72; }
|
||||
.voice-row__meta { display: flex; align-items: center; gap: 5px; color: var(--color-text-tertiary); font-size: 0.65rem; }
|
||||
.voice-row__chevron { color: var(--color-text-tertiary); font-size: 0.65rem; }
|
||||
.voice-row--selected .voice-row__chevron { color: var(--color-primary); }
|
||||
|
||||
.voice-library-detail {
|
||||
min-width: 0;
|
||||
padding: clamp(var(--spacing-lg), 3vw, var(--spacing-xl));
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
background: var(--color-surface-raised);
|
||||
}
|
||||
.voice-library-detail > .empty-state { flex: 1; min-height: 420px; }
|
||||
.voice-detail__header { display: flex; align-items: flex-start; justify-content: space-between; gap: var(--spacing-lg); margin-bottom: var(--spacing-lg); }
|
||||
.voice-detail__eyebrow {
|
||||
display: block;
|
||||
margin-bottom: var(--spacing-xs);
|
||||
color: var(--color-eyebrow);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.625rem;
|
||||
text-transform: uppercase;
|
||||
letter-spacing: 0.16em;
|
||||
}
|
||||
.voice-detail__header h2 { margin: 0; font-size: clamp(var(--text-xl), 2.5vw, var(--text-2xl)); line-height: var(--leading-tight); }
|
||||
.voice-detail__header p { margin: var(--spacing-xs) 0 0; max-width: 54ch; color: var(--color-text-secondary); line-height: var(--leading-normal); }
|
||||
.voice-detail__player {
|
||||
padding: var(--spacing-md);
|
||||
border: 1px solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-lg);
|
||||
background: var(--color-surface-sunken);
|
||||
}
|
||||
.voice-detail__section { margin-top: var(--spacing-lg); }
|
||||
.voice-detail__section h3 { margin: 0 0 var(--spacing-sm); color: var(--color-text-secondary); font-size: var(--text-xs); text-transform: uppercase; letter-spacing: 0.08em; }
|
||||
.voice-detail__section blockquote {
|
||||
margin: 0;
|
||||
padding: var(--spacing-md) var(--spacing-lg);
|
||||
border-left: 2px solid var(--color-primary);
|
||||
background: var(--color-bg-tertiary);
|
||||
color: var(--color-text-secondary);
|
||||
font-size: var(--text-base);
|
||||
line-height: var(--leading-relaxed);
|
||||
}
|
||||
.voice-detail__metadata {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(2, minmax(0, 1fr));
|
||||
gap: 1px;
|
||||
margin: var(--spacing-lg) 0 0;
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--color-border-divider);
|
||||
border-radius: var(--radius-md);
|
||||
background: var(--color-border-divider);
|
||||
}
|
||||
.voice-detail__metadata > div { min-width: 0; padding: var(--spacing-sm) var(--spacing-md); background: var(--color-bg-secondary); }
|
||||
.voice-detail__metadata dt { color: var(--color-text-tertiary); font-size: 0.65rem; text-transform: uppercase; letter-spacing: 0.06em; }
|
||||
.voice-detail__metadata dd { margin: 3px 0 0; overflow: hidden; text-overflow: ellipsis; color: var(--color-text-primary); font-size: var(--text-sm); }
|
||||
.voice-detail__consent {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: var(--spacing-sm);
|
||||
margin-top: var(--spacing-lg);
|
||||
padding: var(--spacing-sm) var(--spacing-md);
|
||||
color: var(--color-success);
|
||||
background: var(--color-success-light);
|
||||
border: 1px solid var(--color-success-border);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.voice-detail__consent > i { width: 20px; text-align: center; }
|
||||
.voice-detail__consent > div { display: flex; flex-direction: column; }
|
||||
.voice-detail__consent strong { font-size: var(--text-sm); }
|
||||
.voice-detail__consent span { color: var(--color-text-secondary); font-size: var(--text-xs); }
|
||||
.voice-detail__model-setup {
|
||||
margin-top: var(--spacing-lg);
|
||||
padding: var(--spacing-md);
|
||||
border: 1px solid var(--color-warning-border);
|
||||
border-radius: var(--radius-lg);
|
||||
background: var(--color-warning-light);
|
||||
}
|
||||
.voice-detail__model-setup-heading { display: flex; align-items: flex-start; gap: var(--spacing-sm); }
|
||||
.voice-detail__model-setup-heading > i { width: 24px; padding-top: 2px; color: var(--color-warning); text-align: center; }
|
||||
.voice-detail__model-setup-heading h3 { margin: 0; font-size: var(--text-sm); }
|
||||
.voice-detail__model-setup-heading p { margin: 3px 0 0; color: var(--color-text-secondary); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-detail__model-loading { display: flex; align-items: center; gap: var(--spacing-sm); padding: var(--spacing-md) 0 var(--spacing-xs) 32px; color: var(--color-text-secondary); font-size: var(--text-xs); }
|
||||
.voice-detail__model-list { display: flex; flex-direction: column; gap: var(--spacing-xs); margin: var(--spacing-md) 0 0 32px; padding: 0; list-style: none; }
|
||||
.voice-detail__model-list li {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-md);
|
||||
padding: var(--spacing-sm);
|
||||
background: var(--color-surface-raised);
|
||||
border: 1px solid var(--color-border-subtle);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.voice-detail__model-list li > span { min-width: 0; display: flex; flex-direction: column; }
|
||||
.voice-detail__model-list strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: var(--text-xs); }
|
||||
.voice-detail__model-list small { color: var(--color-text-tertiary); font-family: var(--font-mono); font-size: 0.625rem; }
|
||||
.voice-detail__model-list .btn { flex-shrink: 0; }
|
||||
.voice-detail__model-fallback { margin: var(--spacing-md) 0 0 32px; color: var(--color-text-secondary); font-size: var(--text-xs); }
|
||||
.voice-detail__model-fallback a { color: var(--color-primary); font-weight: var(--font-weight-semibold); }
|
||||
.voice-detail__capability-note { margin: var(--spacing-sm) 0 0 32px; color: var(--color-text-tertiary); font-size: 0.625rem; line-height: var(--leading-normal); }
|
||||
.voice-detail__capability-note i { color: var(--color-success); }
|
||||
|
||||
.voice-detail__api {
|
||||
margin-top: var(--spacing-lg);
|
||||
overflow: hidden;
|
||||
border: 1px solid var(--color-border-default);
|
||||
border-radius: var(--radius-lg);
|
||||
background: var(--color-surface-sunken);
|
||||
}
|
||||
.voice-detail__api > summary {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
justify-content: space-between;
|
||||
gap: var(--spacing-md);
|
||||
padding: var(--spacing-md);
|
||||
cursor: pointer;
|
||||
list-style: none;
|
||||
}
|
||||
.voice-detail__api > summary::-webkit-details-marker { display: none; }
|
||||
.voice-detail__api > summary > span { display: flex; flex-direction: column; }
|
||||
.voice-detail__api > summary strong { font-size: var(--text-sm); }
|
||||
.voice-detail__api > summary small { margin-top: 2px; color: var(--color-text-tertiary); font-size: var(--text-xs); }
|
||||
.voice-detail__api > summary > i { color: var(--color-text-tertiary); font-size: var(--text-xs); transition: transform var(--duration-fast) var(--ease-default); }
|
||||
.voice-detail__api[open] > summary > i { transform: rotate(180deg); }
|
||||
.voice-detail__api-body { padding: var(--spacing-md); border-top: 1px solid var(--color-border-divider); background: var(--color-surface-raised); }
|
||||
.voice-detail__api-body > p { margin: 0; color: var(--color-text-secondary); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-detail__compatible-models { display: flex; flex-direction: column; gap: var(--spacing-xs); margin-top: var(--spacing-md); }
|
||||
.voice-detail__compatible-models > strong { color: var(--color-text-secondary); font-size: var(--text-xs); }
|
||||
.voice-detail__compatible-models > span { color: var(--color-text-tertiary); font-size: var(--text-xs); }
|
||||
.voice-detail__compatible-models ul { display: flex; flex-wrap: wrap; gap: var(--spacing-xs); margin: 0; padding: 0; list-style: none; }
|
||||
.voice-detail__compatible-models li { display: inline-flex; align-items: center; gap: var(--spacing-xs); padding: 4px 8px; background: var(--color-success-light); border: 1px solid var(--color-success-border); border-radius: var(--radius-full); font-size: var(--text-xs); }
|
||||
.voice-detail__compatible-models li small { color: var(--color-text-tertiary); font-family: var(--font-mono); font-size: 0.6rem; }
|
||||
.voice-detail__code-heading { display: flex; align-items: center; justify-content: space-between; gap: var(--spacing-sm); margin-top: var(--spacing-md); color: var(--color-text-secondary); font-size: var(--text-xs); font-weight: var(--font-weight-semibold); }
|
||||
.voice-detail__api pre { max-width: 100%; margin: var(--spacing-xs) 0 0; padding: var(--spacing-md); overflow-x: auto; background: var(--color-bg-primary); border: 1px solid var(--color-border-divider); border-radius: var(--radius-md); color: var(--color-text-primary); font-size: 0.6875rem; line-height: var(--leading-relaxed); }
|
||||
.voice-detail__api code { font-family: var(--font-mono); }
|
||||
.voice-detail__api-body > .voice-detail__api-note { margin-top: var(--spacing-sm); color: var(--color-text-tertiary); }
|
||||
.voice-detail__actions { display: flex; align-items: center; gap: var(--spacing-sm); margin-top: auto; padding-top: var(--spacing-xl); }
|
||||
.voice-detail__actions .btn-danger { margin-left: auto; }
|
||||
|
||||
/* Compact profile picker inside the existing TTS control column. */
|
||||
.tts-voice-label-row { display: flex; align-items: center; justify-content: space-between; gap: var(--spacing-sm); }
|
||||
.tts-voice-label-row .badge { margin-bottom: var(--spacing-xs); }
|
||||
.tts-voice-picker { display: flex; flex-direction: column; gap: var(--spacing-xs); }
|
||||
.tts-voice-picker__selection {
|
||||
display: grid;
|
||||
grid-template-columns: 30px minmax(0, 1fr) auto;
|
||||
align-items: center;
|
||||
gap: var(--spacing-sm);
|
||||
padding: var(--spacing-sm);
|
||||
color: var(--color-text-primary);
|
||||
background: var(--color-primary-light);
|
||||
border: 1px solid var(--color-primary-border);
|
||||
border-radius: var(--radius-md);
|
||||
}
|
||||
.tts-voice-picker__avatar {
|
||||
width: 30px;
|
||||
height: 30px;
|
||||
display: grid;
|
||||
place-items: center;
|
||||
color: var(--color-primary-text);
|
||||
background: var(--color-primary);
|
||||
border-radius: var(--radius-full);
|
||||
font-family: var(--font-mono);
|
||||
font-size: 0.58rem;
|
||||
font-weight: var(--font-weight-semibold);
|
||||
}
|
||||
.tts-voice-picker__selection > span:nth-child(2) { min-width: 0; display: flex; flex-direction: column; }
|
||||
.tts-voice-picker__selection strong { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; font-size: var(--text-sm); }
|
||||
.tts-voice-picker__selection small { color: var(--color-text-secondary); font-size: var(--text-xs); }
|
||||
.tts-voice-picker__selection > i { color: var(--color-primary); }
|
||||
.tts-voice-picker__hint,
|
||||
.tts-voice-picker__error { margin: 0; color: var(--color-text-muted); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.tts-voice-picker__error { color: var(--color-error); }
|
||||
.tts-voice-picker__hint a,
|
||||
.tts-voice-picker__manage { color: var(--color-primary); font-weight: var(--font-weight-medium); text-decoration: none; }
|
||||
.tts-voice-picker__manage { display: inline-flex; align-items: center; gap: 5px; width: fit-content; font-size: var(--text-xs); }
|
||||
.tts-voice-picker__manage i { font-size: 0.65em; transition: transform var(--duration-fast) var(--ease-default); }
|
||||
.tts-voice-picker__manage:hover i { transform: translateX(2px); }
|
||||
|
||||
/* Full-page creation flow: the content form and a quiet sticky readiness rail. */
|
||||
.voice-create-page { max-width: 1180px; padding-top: var(--spacing-xl); }
|
||||
.voice-create-grid { display: grid; grid-template-columns: minmax(0, 1fr) 300px; gap: var(--spacing-lg); align-items: start; }
|
||||
.voice-create-form {
|
||||
overflow: hidden;
|
||||
background: var(--color-surface-raised);
|
||||
border: 1px solid var(--color-border-default);
|
||||
border-radius: var(--radius-xl);
|
||||
box-shadow: var(--shadow-sm), var(--shadow-inset-top);
|
||||
}
|
||||
.voice-create-section { padding: var(--spacing-xl); border-bottom: 1px solid var(--color-border-divider); }
|
||||
.voice-create-section__heading { display: grid; grid-template-columns: 32px minmax(0, 1fr); gap: var(--spacing-md); margin-bottom: var(--spacing-lg); }
|
||||
.voice-create-section__heading > span {
|
||||
padding-top: 3px;
|
||||
color: var(--color-primary);
|
||||
font-family: var(--font-mono);
|
||||
font-size: var(--text-xs);
|
||||
letter-spacing: 0.08em;
|
||||
}
|
||||
.voice-create-section__heading h2 { margin: 0; color: var(--color-text-primary); font-size: var(--text-lg); }
|
||||
.voice-create-section__heading p { margin: 4px 0 0; max-width: 62ch; color: var(--color-text-muted); font-size: var(--text-sm); line-height: var(--leading-normal); }
|
||||
.voice-create-processing { display: flex; align-items: center; gap: var(--spacing-sm); margin-top: var(--spacing-sm); color: var(--color-text-secondary); font-size: var(--text-sm); }
|
||||
.form-error { margin: var(--spacing-sm) 0 0; color: var(--color-error); font-size: var(--text-sm); }
|
||||
.form-hint { margin: var(--spacing-xs) 0 0; color: var(--color-text-tertiary); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-create-preview { margin-top: var(--spacing-md); padding: var(--spacing-md); background: var(--color-surface-sunken); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-lg); }
|
||||
.voice-create-preview__meta { display: flex; flex-wrap: wrap; gap: var(--spacing-xs) var(--spacing-md); margin-top: var(--spacing-sm); color: var(--color-text-muted); font-size: var(--text-xs); }
|
||||
.voice-create-preview__meta span { display: inline-flex; align-items: center; gap: 5px; }
|
||||
.voice-create-preview__meta .tone-success { color: var(--color-success); }
|
||||
.voice-create-preview__meta .tone-warning { color: var(--color-warning); }
|
||||
.voice-create-label-row { display: flex; align-items: center; justify-content: space-between; gap: var(--spacing-md); }
|
||||
.voice-create-label-row > span { color: var(--color-text-tertiary); font-family: var(--font-mono); font-size: 0.625rem; }
|
||||
|
||||
.voice-consent-check { position: relative; display: grid; grid-template-columns: 24px minmax(0, 1fr); gap: var(--spacing-md); align-items: start; padding: var(--spacing-md); background: var(--color-bg-tertiary); border: 1px solid var(--color-border-default); border-radius: var(--radius-lg); cursor: pointer; }
|
||||
.voice-consent-check:hover { border-color: var(--color-border-strong); }
|
||||
.voice-consent-check > input { position: absolute; opacity: 0; pointer-events: none; }
|
||||
.voice-consent-check__box { width: 22px; height: 22px; display: grid; place-items: center; color: transparent; background: var(--color-surface-sunken); border: 1px solid var(--color-border-strong); border-radius: var(--radius-sm); transition: background var(--duration-fast), color var(--duration-fast), border-color var(--duration-fast); }
|
||||
.voice-consent-check > input:checked + .voice-consent-check__box { color: var(--color-primary-text); background: var(--color-primary); border-color: var(--color-primary); }
|
||||
.voice-consent-check > input:focus-visible + .voice-consent-check__box { box-shadow: 0 0 0 3px var(--color-focus-ring); }
|
||||
.voice-consent-check strong { display: block; color: var(--color-text-primary); font-size: var(--text-sm); }
|
||||
.voice-consent-check small { display: block; margin-top: 3px; color: var(--color-text-secondary); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-create-actions { display: flex; justify-content: flex-end; gap: var(--spacing-sm); padding: var(--spacing-lg) var(--spacing-xl); background: var(--color-bg-secondary); }
|
||||
|
||||
.voice-readiness {
|
||||
position: sticky;
|
||||
top: var(--spacing-lg);
|
||||
padding: var(--spacing-lg);
|
||||
background: var(--color-bg-secondary);
|
||||
border: 1px solid var(--color-border-default);
|
||||
border-radius: var(--radius-xl);
|
||||
box-shadow: var(--shadow-subtle), var(--shadow-inset-top);
|
||||
}
|
||||
.voice-readiness__header { display: flex; align-items: flex-start; gap: var(--spacing-sm); padding-bottom: var(--spacing-md); border-bottom: 1px solid var(--color-border-divider); }
|
||||
.voice-readiness__icon { width: 34px; height: 34px; flex: 0 0 34px; display: grid; place-items: center; color: var(--color-primary); background: var(--color-primary-light); border-radius: var(--radius-lg); }
|
||||
.voice-readiness__header h2 { margin: 0; font-size: var(--text-base); }
|
||||
.voice-readiness__header p { margin: 3px 0 0; color: var(--color-text-muted); font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-readiness ul { display: flex; flex-direction: column; gap: var(--spacing-xs); list-style: none; padding: var(--spacing-md) 0; margin: 0; }
|
||||
.voice-readiness__item { display: grid; grid-template-columns: 22px minmax(0, 1fr); align-items: center; gap: var(--spacing-sm); min-height: 30px; color: var(--color-text-muted); font-size: var(--text-sm); }
|
||||
.voice-readiness__item i { width: 20px; height: 20px; display: grid; place-items: center; border-radius: var(--radius-full); background: var(--color-surface-sunken); font-size: 0.6rem; }
|
||||
.voice-readiness__item--ready { color: var(--color-text-primary); }
|
||||
.voice-readiness__item--ready i { color: var(--color-success); background: var(--color-success-light); }
|
||||
.voice-readiness__item--warning i { color: var(--color-warning); background: var(--color-warning-light); }
|
||||
.voice-readiness__privacy { display: flex; align-items: flex-start; gap: var(--spacing-sm); padding-top: var(--spacing-md); border-top: 1px solid var(--color-border-divider); color: var(--color-text-muted); }
|
||||
.voice-readiness__privacy i { margin-top: 2px; color: var(--color-primary); }
|
||||
.voice-readiness__privacy p { margin: 0; font-size: var(--text-xs); line-height: var(--leading-normal); }
|
||||
.voice-readiness__privacy strong { color: var(--color-text-secondary); }
|
||||
|
||||
@media (max-width: 1120px) {
|
||||
.voice-library-shell { grid-template-columns: 1fr; }
|
||||
.voice-library-master { border-right: 0; border-bottom: 1px solid var(--color-border-divider); }
|
||||
.voice-library-list { max-height: 420px; }
|
||||
.voice-create-grid { grid-template-columns: 1fr; }
|
||||
.voice-readiness { position: static; order: -1; }
|
||||
.voice-readiness ul { display: grid; grid-template-columns: repeat(2, minmax(0, 1fr)); }
|
||||
}
|
||||
|
||||
@media (max-width: 640px) {
|
||||
.voice-library-page,
|
||||
.voice-create-page { padding: var(--spacing-sm); }
|
||||
.voice-library-page .page-header__meta,
|
||||
.voice-create-page .page-header__meta { width: 100%; }
|
||||
.voice-library-page .page-header__meta .btn,
|
||||
.voice-create-page .page-header__meta .btn { width: 100%; }
|
||||
.voice-library-summary { align-items: flex-start; flex-direction: column; gap: var(--spacing-xs); }
|
||||
.voice-library-summary__divider { display: none; }
|
||||
.voice-library-toolbar { grid-template-columns: 1fr; }
|
||||
.voice-library-language { width: 100%; }
|
||||
.voice-detail__header { flex-direction: column; }
|
||||
.voice-detail__metadata { grid-template-columns: 1fr; }
|
||||
.voice-detail__model-list,
|
||||
.voice-detail__model-fallback,
|
||||
.voice-detail__capability-note { margin-left: 0; }
|
||||
.voice-detail__model-list li { align-items: stretch; flex-direction: column; }
|
||||
.voice-detail__model-list .btn { width: 100%; }
|
||||
.voice-detail__actions { align-items: stretch; flex-direction: column; }
|
||||
.voice-detail__actions .btn { width: 100%; }
|
||||
.voice-detail__actions .btn-danger { margin-left: 0; }
|
||||
.voice-create-section { padding: var(--spacing-lg); }
|
||||
.voice-create-section__heading { grid-template-columns: 1fr; gap: var(--spacing-xs); }
|
||||
.voice-create-actions { padding: var(--spacing-md) var(--spacing-lg); }
|
||||
.voice-create-actions .btn { flex: 1; }
|
||||
.voice-readiness { order: initial; }
|
||||
.voice-readiness ul { grid-template-columns: 1fr; }
|
||||
}
|
||||
|
||||
/* Nodes: cluster pulse + attention callout (replaces the stat-card strip) */
|
||||
.cluster-pulse {
|
||||
font-size: var(--text-sm);
|
||||
|
||||
@@ -41,7 +41,7 @@ function UnsupportedNotice({ mode }) {
|
||||
)
|
||||
}
|
||||
|
||||
export default function MediaInput({ mode, label, value, onChange, idPrefix = 'media' }) {
|
||||
export default function MediaInput({ mode, label, value, onChange, onError, maxBytes, preferBlob = false, idPrefix = 'media' }) {
|
||||
const [tab, setTab] = useState('file') // 'file' | 'live'
|
||||
const fileRef = useRef(null)
|
||||
const cap = useMediaCapture(mode)
|
||||
@@ -54,13 +54,30 @@ export default function MediaInput({ mode, label, value, onChange, idPrefix = 'm
|
||||
const handleFile = async (e) => {
|
||||
const f = e.target.files?.[0]
|
||||
if (!f) { onChange(null); return }
|
||||
const base64 = await fileToBase64(f)
|
||||
const dataUrl = await new Promise((resolve) => {
|
||||
const reader = new FileReader()
|
||||
reader.onload = () => resolve(reader.result)
|
||||
reader.readAsDataURL(f)
|
||||
})
|
||||
onChange({ base64, dataUrl, mime: f.type, source: 'file', name: f.name })
|
||||
if (maxBytes && f.size > maxBytes) {
|
||||
const error = new Error(`Selected file exceeds the ${Math.round(maxBytes / (1024 * 1024))} MiB limit`)
|
||||
if (fileRef.current) fileRef.current.value = ''
|
||||
onChange(null)
|
||||
onError?.(error)
|
||||
return
|
||||
}
|
||||
try {
|
||||
if (preferBlob) {
|
||||
onChange({ blob: f, mime: f.type, source: 'file', name: f.name })
|
||||
return
|
||||
}
|
||||
const base64 = await fileToBase64(f)
|
||||
const dataUrl = await new Promise((resolve, reject) => {
|
||||
const reader = new FileReader()
|
||||
reader.onerror = () => reject(reader.error)
|
||||
reader.onload = () => resolve(reader.result)
|
||||
reader.readAsDataURL(f)
|
||||
})
|
||||
onChange({ base64, blob: f, dataUrl, mime: f.type, source: 'file', name: f.name })
|
||||
} catch (error) {
|
||||
onChange(null)
|
||||
onError?.(error)
|
||||
}
|
||||
}
|
||||
|
||||
const handleSnap = () => {
|
||||
@@ -75,7 +92,9 @@ export default function MediaInput({ mode, label, value, onChange, idPrefix = 'm
|
||||
const pending = cap.startRecording()
|
||||
if (!pending) return
|
||||
const result = await pending
|
||||
onChange({ ...result, source: 'live' })
|
||||
onChange(preferBlob
|
||||
? { blob: result.blob, mime: result.mime, source: 'live', name: 'recording.wav' }
|
||||
: { ...result, source: 'live' })
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -50,6 +50,7 @@ export default function ConsoleLayout({ config }) {
|
||||
const { t } = useTranslation('nav')
|
||||
const { isAdmin, authEnabled, hasFeature } = useAuth()
|
||||
const [features, setFeatures] = useState(featuresCache)
|
||||
const [railOpen, setRailOpen] = useState(false)
|
||||
const location = useLocation()
|
||||
// Forward the App-level outlet context (e.g. addToast) — a nested bare
|
||||
// <Outlet/> would otherwise shadow it with undefined and crash pages.
|
||||
@@ -72,23 +73,37 @@ export default function ConsoleLayout({ config }) {
|
||||
|
||||
return (
|
||||
<div className="console-layout">
|
||||
<nav className={`console-rail${entering ? ' console-rail--enter' : ''}`} aria-label={t(config.titleKey)}>
|
||||
<nav className={`console-rail${entering ? ' console-rail--enter' : ''}${railOpen ? ' console-rail--open' : ''}`} aria-label={t(config.titleKey)}>
|
||||
<div className="console-rail-header">
|
||||
<i className={config.icon} aria-hidden="true" />
|
||||
<span>{t(config.titleKey)}</span>
|
||||
<span className="console-rail-header__title">
|
||||
<i className={config.icon} aria-hidden="true" />
|
||||
<span>{t(config.titleKey)}</span>
|
||||
</span>
|
||||
<button
|
||||
type="button"
|
||||
className="console-rail-toggle"
|
||||
aria-expanded={railOpen}
|
||||
aria-controls={`console-rail-groups-${config.id}`}
|
||||
aria-label={t(railOpen ? 'console.collapseNavigation' : 'console.expandNavigation', { section: t(config.titleKey) })}
|
||||
onClick={() => setRailOpen(open => !open)}
|
||||
>
|
||||
<i className={`fas fa-chevron-${railOpen ? 'up' : 'down'}`} aria-hidden="true" />
|
||||
</button>
|
||||
</div>
|
||||
<div id={`console-rail-groups-${config.id}`} className="console-rail-groups">
|
||||
{config.groups.map((group, gi) => {
|
||||
const items = group.items.filter(item => isConsoleItemVisible(item, auth))
|
||||
if (items.length === 0) return null
|
||||
return (
|
||||
<div key={group.titleKey || gi} className="console-group">
|
||||
{group.titleKey && <div className="console-group-title">{t(group.titleKey)}</div>}
|
||||
{items.map(item => (
|
||||
<RailItem key={item.path || item.href} item={item} label={t(item.labelKey)} />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
{config.groups.map((group, gi) => {
|
||||
const items = group.items.filter(item => isConsoleItemVisible(item, auth))
|
||||
if (items.length === 0) return null
|
||||
return (
|
||||
<div key={group.titleKey || gi} className="console-group">
|
||||
{group.titleKey && <div className="console-group-title">{t(group.titleKey)}</div>}
|
||||
{items.map(item => (
|
||||
<RailItem key={item.path || item.href} item={item} label={t(item.labelKey)} />
|
||||
))}
|
||||
</div>
|
||||
)
|
||||
})}
|
||||
</nav>
|
||||
<div className="console-body" key={location.pathname}>
|
||||
{/* Own Suspense so a lazy page shows the loader in the body while the
|
||||
|
||||
@@ -53,6 +53,7 @@ export const operateConsole = {
|
||||
titleKey: 'operate.inference',
|
||||
items: [
|
||||
{ path: '/app/backends', icon: 'fas fa-server', labelKey: 'items.backends', adminOnly: true },
|
||||
{ path: '/app/voice-library', icon: 'fas fa-wave-square', labelKey: 'items.voiceLibrary', adminOnly: true },
|
||||
],
|
||||
},
|
||||
{
|
||||
|
||||
18
core/http/react-ui/src/hooks/useMediaCapture.js
vendored
18
core/http/react-ui/src/hooks/useMediaCapture.js
vendored
@@ -3,10 +3,12 @@ import { useCallback, useEffect, useRef, useState } from 'react'
|
||||
// Encode an AudioBuffer as a 16-bit PCM mono WAV blob. Libsndfile (which the
|
||||
// SpeechBrain / ONNX voice backends use) reads this shape without extra
|
||||
// decoders. We downmix to mono because speaker-encoder models expect a single
|
||||
// channel and sample-rate resampling is handled server-side.
|
||||
function audioBufferToWavBlob(audioBuffer) {
|
||||
const sampleRate = audioBuffer.sampleRate
|
||||
const numFrames = audioBuffer.length
|
||||
// channel. Callers may request a target sample rate for backends with a shared
|
||||
// reference-audio contract; ordinary media capture preserves the source rate.
|
||||
export function audioBufferToWavBlob(audioBuffer, targetSampleRate = audioBuffer.sampleRate) {
|
||||
const sourceSampleRate = audioBuffer.sampleRate
|
||||
const sampleRate = targetSampleRate
|
||||
const numFrames = Math.max(1, Math.round(audioBuffer.length * sampleRate / sourceSampleRate))
|
||||
const bitsPerSample = 16
|
||||
const blockAlign = bitsPerSample / 8 // mono, 1 channel
|
||||
const byteRate = sampleRate * blockAlign
|
||||
@@ -37,8 +39,14 @@ function audioBufferToWavBlob(audioBuffer) {
|
||||
for (let c = 0; c < numChannels; c++) channels.push(audioBuffer.getChannelData(c))
|
||||
let offset = 44
|
||||
for (let i = 0; i < numFrames; i++) {
|
||||
const sourcePosition = i * sourceSampleRate / sampleRate
|
||||
const left = Math.min(Math.floor(sourcePosition), audioBuffer.length - 1)
|
||||
const right = Math.min(left + 1, audioBuffer.length - 1)
|
||||
const fraction = sourcePosition - left
|
||||
let sum = 0
|
||||
for (let c = 0; c < numChannels; c++) sum += channels[c][i]
|
||||
for (let c = 0; c < numChannels; c++) {
|
||||
sum += channels[c][left] + (channels[c][right] - channels[c][left]) * fraction
|
||||
}
|
||||
const mono = Math.max(-1, Math.min(1, sum / numChannels))
|
||||
view.setInt16(offset, mono < 0 ? mono * 0x8000 : mono * 0x7FFF, true)
|
||||
offset += 2
|
||||
|
||||
72
core/http/react-ui/src/hooks/useVoiceProfiles.js
vendored
Normal file
72
core/http/react-ui/src/hooks/useVoiceProfiles.js
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
import { useCallback, useEffect, useState } from 'react'
|
||||
import { modelsApi, voiceProfilesApi } from '../utils/api'
|
||||
|
||||
export function useVoiceProfiles({ enabled = true } = {}) {
|
||||
const [profiles, setProfiles] = useState([])
|
||||
const [loading, setLoading] = useState(enabled)
|
||||
const [error, setError] = useState(null)
|
||||
|
||||
const fetchProfiles = useCallback(async ({ silent = false } = {}) => {
|
||||
if (!enabled) {
|
||||
setProfiles([])
|
||||
setLoading(false)
|
||||
setError(null)
|
||||
return
|
||||
}
|
||||
if (!silent) setLoading(true)
|
||||
try {
|
||||
const response = await voiceProfilesApi.list()
|
||||
setProfiles(response?.data || [])
|
||||
setError(null)
|
||||
} catch (err) {
|
||||
setError(err?.message || 'Could not load voice profiles')
|
||||
} finally {
|
||||
if (!silent) setLoading(false)
|
||||
}
|
||||
}, [enabled])
|
||||
|
||||
useEffect(() => {
|
||||
fetchProfiles()
|
||||
}, [fetchProfiles])
|
||||
|
||||
const refetch = useCallback(() => fetchProfiles({ silent: true }), [fetchProfiles])
|
||||
return { profiles, loading, error, refetch }
|
||||
}
|
||||
|
||||
export function useVoiceCloningGallery({ enabled = true, limit = 4 } = {}) {
|
||||
const [models, setModels] = useState([])
|
||||
const [loading, setLoading] = useState(enabled)
|
||||
const [error, setError] = useState(null)
|
||||
|
||||
const fetchModels = useCallback(async ({ silent = false } = {}) => {
|
||||
if (!enabled) {
|
||||
setModels([])
|
||||
setLoading(false)
|
||||
setError(null)
|
||||
return
|
||||
}
|
||||
if (!silent) setLoading(true)
|
||||
try {
|
||||
const response = await modelsApi.list({
|
||||
capability: 'voice_cloning',
|
||||
items: limit,
|
||||
page: 1,
|
||||
sort: 'name',
|
||||
order: 'asc',
|
||||
})
|
||||
setModels(response?.models || [])
|
||||
setError(null)
|
||||
} catch (err) {
|
||||
setError(err?.message || 'Could not load compatible models')
|
||||
} finally {
|
||||
if (!silent) setLoading(false)
|
||||
}
|
||||
}, [enabled, limit])
|
||||
|
||||
useEffect(() => {
|
||||
fetchModels()
|
||||
}, [fetchModels])
|
||||
|
||||
const refetch = useCallback(() => fetchModels({ silent: true }), [fetchModels])
|
||||
return { models, loading, error, refetch }
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
import { useState } from 'react'
|
||||
import { useParams, useOutletContext } from 'react-router-dom'
|
||||
import { useEffect, useMemo, useRef, useState } from 'react'
|
||||
import { Link, useParams, useOutletContext, useSearchParams } from 'react-router-dom'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import ModelSelector from '../components/ModelSelector'
|
||||
import PageHeader from '../components/PageHeader'
|
||||
@@ -11,17 +11,53 @@ import MediaHistory from '../components/MediaHistory'
|
||||
import WaveformPlayer from '../components/audio/WaveformPlayer'
|
||||
import { ttsApi } from '../utils/api'
|
||||
import { useMediaHistory } from '../hooks/useMediaHistory'
|
||||
import { useModels } from '../hooks/useModels'
|
||||
import { useVoiceProfiles } from '../hooks/useVoiceProfiles'
|
||||
import { useAuth } from '../context/AuthContext'
|
||||
|
||||
function formatProfileDuration(milliseconds) {
|
||||
const seconds = Math.round((milliseconds || 0) / 1000)
|
||||
return seconds >= 60 ? `${Math.floor(seconds / 60)}:${String(seconds % 60).padStart(2, '0')}` : `${seconds}s`
|
||||
}
|
||||
|
||||
export default function TTS() {
|
||||
const { model: urlModel } = useParams()
|
||||
const { addToast } = useOutletContext()
|
||||
const { t } = useTranslation('media')
|
||||
const { isAdmin } = useAuth()
|
||||
const [searchParams] = useSearchParams()
|
||||
const requestedVoiceID = searchParams.get('voice') || ''
|
||||
const [model, setModel] = useState(urlModel || '')
|
||||
const [manualVoice, setManualVoice] = useState('')
|
||||
const [voiceProfileID, setVoiceProfileID] = useState(requestedVoiceID)
|
||||
const [text, setText] = useState('')
|
||||
const [loading, setLoading] = useState(false)
|
||||
const [error, setError] = useState(null)
|
||||
const [audioUrl, setAudioUrl] = useState(null)
|
||||
const appliedVoiceLinkRef = useRef('')
|
||||
const { addEntry, selectEntry, selectedEntry, historyProps } = useMediaHistory('tts')
|
||||
const { models } = useModels(CAP_TTS)
|
||||
const selectedModel = useMemo(() => models.find(item => item.id === model), [models, model])
|
||||
const compatibleModels = useMemo(() => models.filter(item => item.voice_cloning), [models])
|
||||
const supportsVoiceProfiles = !!selectedModel?.voice_cloning
|
||||
const { profiles, loading: profilesLoading, error: profilesError } = useVoiceProfiles({ enabled: supportsVoiceProfiles })
|
||||
const selectedProfile = profiles.find(profile => profile.id === voiceProfileID) || null
|
||||
|
||||
// A library deep-link should land on a compatible model even when the last
|
||||
// TTS model used was a named-speaker or non-cloning variant. Apply it once
|
||||
// per requested profile so the user can still change models afterwards.
|
||||
useEffect(() => {
|
||||
if (!requestedVoiceID || models.length === 0 || appliedVoiceLinkRef.current === requestedVoiceID) return
|
||||
const targetModel = selectedModel?.voice_cloning ? selectedModel : compatibleModels[0]
|
||||
if (!targetModel) return
|
||||
if (targetModel.id !== model) setModel(targetModel.id)
|
||||
setVoiceProfileID(requestedVoiceID)
|
||||
appliedVoiceLinkRef.current = requestedVoiceID
|
||||
}, [requestedVoiceID, models, model, selectedModel, compatibleModels])
|
||||
|
||||
useEffect(() => {
|
||||
if (selectedModel && !selectedModel.voice_cloning) setVoiceProfileID('')
|
||||
}, [selectedModel])
|
||||
|
||||
const handleGenerate = async (e) => {
|
||||
e.preventDefault()
|
||||
@@ -33,12 +69,22 @@ export default function TTS() {
|
||||
setError(null)
|
||||
|
||||
try {
|
||||
const { blob, serverUrl } = await ttsApi.generate({ model, input: text.trim() })
|
||||
const selectedVoice = supportsVoiceProfiles ? selectedProfile?.voice : manualVoice.trim()
|
||||
const request = { model, input: text.trim() }
|
||||
if (selectedVoice) request.voice = selectedVoice
|
||||
const { blob, serverUrl } = await ttsApi.generate(request)
|
||||
const url = URL.createObjectURL(blob)
|
||||
setAudioUrl(url)
|
||||
addToast(t('tts.actions.generate'), 'success')
|
||||
addToast(t('tts.toasts.generated'), 'success')
|
||||
if (serverUrl) {
|
||||
addEntry({ prompt: text.trim(), model, params: {}, results: [{ url: serverUrl }] })
|
||||
addEntry({
|
||||
prompt: text.trim(),
|
||||
model,
|
||||
params: selectedProfile
|
||||
? { voice: selectedProfile.name }
|
||||
: (!supportsVoiceProfiles && manualVoice.trim() ? { voice: manualVoice.trim() } : {}),
|
||||
results: [{ url: serverUrl }],
|
||||
})
|
||||
}
|
||||
selectEntry(null)
|
||||
} catch (err) {
|
||||
@@ -58,6 +104,56 @@ export default function TTS() {
|
||||
<label className="form-label">{t('tts.labels.model')}</label>
|
||||
<ModelSelector value={model} onChange={setModel} capability={CAP_TTS} />
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<div className="tts-voice-label-row">
|
||||
<label className="form-label" htmlFor="tts-voice">{t('tts.labels.voice')}</label>
|
||||
{supportsVoiceProfiles && <span className="badge badge-success">{t('tts.voiceLibrary.cloningReady')}</span>}
|
||||
</div>
|
||||
{supportsVoiceProfiles ? (
|
||||
<div className="tts-voice-picker">
|
||||
<select
|
||||
id="tts-voice"
|
||||
className="input"
|
||||
value={voiceProfileID}
|
||||
disabled={profilesLoading}
|
||||
onChange={(event) => setVoiceProfileID(event.target.value)}
|
||||
>
|
||||
<option value="">{profilesLoading ? t('tts.voiceLibrary.loading') : t('tts.voiceLibrary.modelDefault')}</option>
|
||||
{profiles.map(profile => (
|
||||
<option key={profile.id} value={profile.id}>
|
||||
{profile.name}{profile.language ? ` · ${profile.language}` : ''}
|
||||
</option>
|
||||
))}
|
||||
</select>
|
||||
{selectedProfile && (
|
||||
<div className="tts-voice-picker__selection">
|
||||
<span className="tts-voice-picker__avatar">{selectedProfile.name.slice(0, 2).toUpperCase()}</span>
|
||||
<span><strong>{selectedProfile.name}</strong><small>{selectedProfile.language || t('voiceLibrary.metadata.languageUnknown')} · {formatProfileDuration(selectedProfile.audio?.duration_ms)}</small></span>
|
||||
<i className="fas fa-wave-square" aria-hidden="true" />
|
||||
</div>
|
||||
)}
|
||||
{!profilesLoading && profiles.length === 0 && (
|
||||
<p className="tts-voice-picker__hint">
|
||||
{t('tts.voiceLibrary.empty')}{' '}
|
||||
{isAdmin && <Link to="/app/voice-library/new">{t('tts.voiceLibrary.create')}</Link>}
|
||||
</p>
|
||||
)}
|
||||
{profilesError && <p className="tts-voice-picker__error" role="alert">{profilesError}</p>}
|
||||
{isAdmin && profiles.length > 0 && <Link className="tts-voice-picker__manage" to="/app/voice-library">{t('tts.voiceLibrary.manage')} <i className="fas fa-arrow-right" aria-hidden="true" /></Link>}
|
||||
</div>
|
||||
) : (
|
||||
<>
|
||||
<input
|
||||
id="tts-voice"
|
||||
className="input"
|
||||
value={manualVoice}
|
||||
onChange={(event) => setManualVoice(event.target.value)}
|
||||
placeholder={t('tts.labels.voicePlaceholder')}
|
||||
/>
|
||||
<p className="form-hint">{t('tts.voiceLibrary.namedVoiceHint')}</p>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label className="form-label">{t('tts.labels.input')}</label>
|
||||
<textarea
|
||||
|
||||
422
core/http/react-ui/src/pages/VoiceLibrary.jsx
Normal file
422
core/http/react-ui/src/pages/VoiceLibrary.jsx
Normal file
@@ -0,0 +1,422 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { Link, useNavigate, useOutletContext, useSearchParams } from 'react-router-dom'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import ConfirmDialog from '../components/ConfirmDialog'
|
||||
import EmptyState from '../components/EmptyState'
|
||||
import ErrorWithTraceLink from '../components/ErrorWithTraceLink'
|
||||
import LoadingSpinner from '../components/LoadingSpinner'
|
||||
import PageHeader from '../components/PageHeader'
|
||||
import StatusPill from '../components/StatusPill'
|
||||
import WaveformPlayer from '../components/audio/WaveformPlayer'
|
||||
import { useModels } from '../hooks/useModels'
|
||||
import { useOperations } from '../hooks/useOperations'
|
||||
import { useVoiceCloningGallery, useVoiceProfiles } from '../hooks/useVoiceProfiles'
|
||||
import { CAP_TTS } from '../utils/capabilities'
|
||||
import { modelsApi, voiceProfilesApi } from '../utils/api'
|
||||
import { copyToClipboard } from '../utils/clipboard'
|
||||
|
||||
const ROW_BARS = [35, 58, 78, 44, 68, 92, 55, 72, 38, 82, 64, 46, 74, 52, 88, 42, 66, 48]
|
||||
|
||||
function formatDuration(milliseconds) {
|
||||
const seconds = Math.max(0, Math.round((milliseconds || 0) / 1000))
|
||||
const minutes = Math.floor(seconds / 60)
|
||||
const rest = seconds % 60
|
||||
return minutes ? `${minutes}:${String(rest).padStart(2, '0')}` : `${rest}s`
|
||||
}
|
||||
|
||||
function CompactWaveform() {
|
||||
return (
|
||||
<span className="voice-row__waveform" aria-hidden="true">
|
||||
{ROW_BARS.map((height, index) => <span key={index} style={{ height: `${height}%` }} />)}
|
||||
</span>
|
||||
)
|
||||
}
|
||||
|
||||
function VoiceModelSetup({ t, galleryLoading, installableModels, galleryError, installing, operations, onInstall }) {
|
||||
return (
|
||||
<section className="voice-detail__model-setup" aria-labelledby="voice-model-setup-title">
|
||||
<div className="voice-detail__model-setup-heading">
|
||||
<i className="fas fa-cube" aria-hidden="true" />
|
||||
<div>
|
||||
<h3 id="voice-model-setup-title">{t('voiceLibrary.modelSetup.title')}</h3>
|
||||
<p>{t('voiceLibrary.modelSetup.body')}</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{galleryLoading && (
|
||||
<div className="voice-detail__model-loading"><LoadingSpinner size="sm" /><span>{t('voiceLibrary.modelSetup.loading')}</span></div>
|
||||
)}
|
||||
{!galleryLoading && installableModels.length > 0 && (
|
||||
<ul className="voice-detail__model-list">
|
||||
{installableModels.map(model => {
|
||||
const busy = installing.has(model.id) || operations.some(operation => operation.name === model.id && !operation.completed && !operation.error)
|
||||
return (
|
||||
<li key={model.id}>
|
||||
<span>
|
||||
<strong>{model.name}</strong>
|
||||
<small>{model.backend || t('voiceLibrary.modelSetup.backendUnknown')}</small>
|
||||
</span>
|
||||
<button type="button" className="btn btn-secondary btn-sm" disabled={busy} onClick={() => onInstall(model)}>
|
||||
<i className={`fas ${busy ? 'fa-spinner fa-spin' : 'fa-download'}`} aria-hidden="true" />{' '}
|
||||
{t(busy ? 'voiceLibrary.modelSetup.installing' : 'voiceLibrary.modelSetup.install')}
|
||||
</button>
|
||||
</li>
|
||||
)
|
||||
})}
|
||||
</ul>
|
||||
)}
|
||||
{!galleryLoading && installableModels.length === 0 && (
|
||||
<p className="voice-detail__model-fallback">
|
||||
{galleryError ? t('voiceLibrary.modelSetup.loadFailed') : t('voiceLibrary.modelSetup.noneAvailable')}{' '}
|
||||
<Link to="/app/models">{t('voiceLibrary.actions.browseModels')}</Link>
|
||||
</p>
|
||||
)}
|
||||
<p className="voice-detail__capability-note">
|
||||
<i className="fas fa-circle-check" aria-hidden="true" /> {t('voiceLibrary.modelSetup.capabilityNote')}
|
||||
</p>
|
||||
</section>
|
||||
)
|
||||
}
|
||||
|
||||
export default function VoiceLibrary() {
|
||||
const { t, i18n } = useTranslation('media')
|
||||
const { addToast } = useOutletContext()
|
||||
const navigate = useNavigate()
|
||||
const [searchParams, setSearchParams] = useSearchParams()
|
||||
const selectedID = searchParams.get('selected') || ''
|
||||
const [search, setSearch] = useState('')
|
||||
const [language, setLanguage] = useState('all')
|
||||
const [confirmDelete, setConfirmDelete] = useState(false)
|
||||
const [deleting, setDeleting] = useState(false)
|
||||
const [installing, setInstalling] = useState(() => new Map())
|
||||
const { profiles, loading, error, refetch } = useVoiceProfiles()
|
||||
const { models, loading: modelsLoading, refetch: refetchModels } = useModels(CAP_TTS)
|
||||
const { operations } = useOperations()
|
||||
|
||||
const cloningModels = useMemo(() => models.filter(model => model.voice_cloning), [models])
|
||||
const {
|
||||
models: galleryModels,
|
||||
loading: galleryLoading,
|
||||
error: galleryError,
|
||||
refetch: refetchGalleryModels,
|
||||
} = useVoiceCloningGallery({ enabled: !modelsLoading && cloningModels.length === 0 })
|
||||
const installableModels = useMemo(() => galleryModels.filter(model => !model.installed).slice(0, 3), [galleryModels])
|
||||
const languages = useMemo(() => [...new Set(profiles.map(profile => profile.language).filter(Boolean))].sort(), [profiles])
|
||||
const filteredProfiles = useMemo(() => {
|
||||
const needle = search.trim().toLocaleLowerCase(i18n.language)
|
||||
return profiles.filter(profile => {
|
||||
if (language !== 'all' && profile.language !== language) return false
|
||||
if (!needle) return true
|
||||
return [profile.name, profile.description, profile.transcript, profile.language]
|
||||
.some(value => value?.toLocaleLowerCase(i18n.language).includes(needle))
|
||||
})
|
||||
}, [profiles, search, language, i18n.language])
|
||||
|
||||
const selected = profiles.find(profile => profile.id === selectedID) || null
|
||||
const exampleModel = cloningModels[0]?.id || installableModels[0]?.name || '<voice-cloning-model>'
|
||||
const apiExample = selected ? `curl -X POST "$LOCALAI_URL/v1/audio/speech" \\
|
||||
-H "Authorization: Bearer $LOCALAI_API_KEY" \\
|
||||
-H "Content-Type: application/json" \\
|
||||
--data-binary @- \\
|
||||
--output speech.wav <<'JSON'
|
||||
${JSON.stringify({
|
||||
model: exampleModel,
|
||||
input: 'Text to synthesize with this saved voice.',
|
||||
voice: selected.voice,
|
||||
}, null, 2)}
|
||||
JSON` : ''
|
||||
|
||||
useEffect(() => {
|
||||
if (loading || profiles.length === 0) return
|
||||
if (!selected) setSearchParams({ selected: profiles[0].id }, { replace: true })
|
||||
}, [loading, profiles, selected, setSearchParams])
|
||||
|
||||
// Installation progress lives in the shared operations feed. Refresh the
|
||||
// installed capability view as that operation advances so this page becomes
|
||||
// usable as soon as the new model is ready.
|
||||
useEffect(() => {
|
||||
if (installing.size === 0) return
|
||||
refetchModels()
|
||||
setInstalling(previous => {
|
||||
const next = new Map(previous)
|
||||
let changed = false
|
||||
for (const [modelID, startedAt] of previous) {
|
||||
const active = operations.some(operation => operation.name === modelID && !operation.completed && !operation.error)
|
||||
const finished = operations.some(operation => operation.name === modelID && (operation.completed || operation.error))
|
||||
if (finished || (!active && Date.now() - startedAt > 5000)) {
|
||||
next.delete(modelID)
|
||||
changed = true
|
||||
}
|
||||
}
|
||||
return changed ? next : previous
|
||||
})
|
||||
refetchGalleryModels()
|
||||
}, [operations, installing.size, refetchGalleryModels, refetchModels])
|
||||
|
||||
const selectProfile = (id) => setSearchParams({ selected: id })
|
||||
|
||||
const deleteSelected = async () => {
|
||||
if (!selected) return
|
||||
setDeleting(true)
|
||||
try {
|
||||
await voiceProfilesApi.delete(selected.id)
|
||||
setConfirmDelete(false)
|
||||
addToast(t('voiceLibrary.toasts.deleted', { name: selected.name }), 'success')
|
||||
await refetch()
|
||||
} catch (err) {
|
||||
addToast(err.message, 'error')
|
||||
} finally {
|
||||
setDeleting(false)
|
||||
}
|
||||
}
|
||||
|
||||
const installModel = async (model) => {
|
||||
setInstalling(previous => new Map(previous).set(model.id, Date.now()))
|
||||
try {
|
||||
await modelsApi.install(model.id)
|
||||
addToast(t('voiceLibrary.toasts.installStarted', { name: model.name }), 'success')
|
||||
} catch (err) {
|
||||
setInstalling(previous => {
|
||||
const next = new Map(previous)
|
||||
next.delete(model.id)
|
||||
return next
|
||||
})
|
||||
addToast(t('voiceLibrary.toasts.installFailed', { message: err.message }), 'error')
|
||||
}
|
||||
}
|
||||
|
||||
const copyAPIExample = async () => {
|
||||
const copied = await copyToClipboard(apiExample)
|
||||
addToast(t(copied ? 'voiceLibrary.toasts.apiCopied' : 'voiceLibrary.toasts.copyFailed'), copied ? 'success' : 'error')
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="voice-library-page">
|
||||
<PageHeader
|
||||
title={<><i className="fas fa-wave-square" aria-hidden="true" /> {t('voiceLibrary.title')}</>}
|
||||
supporting={t('voiceLibrary.subtitle')}
|
||||
actions={(
|
||||
<Link className="btn btn-primary" to="/app/voice-library/new">
|
||||
<i className="fas fa-plus" aria-hidden="true" /> {t('voiceLibrary.actions.create')}
|
||||
</Link>
|
||||
)}
|
||||
/>
|
||||
|
||||
<div className="voice-library-summary" aria-label={t('voiceLibrary.summary.label')}>
|
||||
<span><strong>{profiles.length}</strong> {t('voiceLibrary.summary.profiles', { count: profiles.length })}</span>
|
||||
<span className="voice-library-summary__divider" aria-hidden="true" />
|
||||
{cloningModels.length > 0 ? (
|
||||
<StatusPill status="healthy" label={t('voiceLibrary.summary.modelsReady', { count: cloningModels.length })} />
|
||||
) : (
|
||||
<StatusPill status="warning" label={t('voiceLibrary.summary.noModels')} />
|
||||
)}
|
||||
</div>
|
||||
|
||||
{error && (
|
||||
<div className="voice-library-error">
|
||||
<ErrorWithTraceLink message={error} />
|
||||
<button type="button" className="btn btn-secondary btn-sm" onClick={() => refetch()}>{t('voiceLibrary.actions.retry')}</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<section className="voice-library-shell" aria-label={t('voiceLibrary.title')}>
|
||||
<div className="voice-library-master">
|
||||
<div className="voice-library-toolbar">
|
||||
<label className="voice-library-search">
|
||||
<span className="sr-only">{t('voiceLibrary.search.label')}</span>
|
||||
<i className="fas fa-magnifying-glass" aria-hidden="true" />
|
||||
<input
|
||||
type="search"
|
||||
value={search}
|
||||
onChange={(event) => setSearch(event.target.value)}
|
||||
placeholder={t('voiceLibrary.search.placeholder')}
|
||||
/>
|
||||
</label>
|
||||
<label>
|
||||
<span className="sr-only">{t('voiceLibrary.filters.language')}</span>
|
||||
<select className="input voice-library-language" value={language} onChange={(event) => setLanguage(event.target.value)}>
|
||||
<option value="all">{t('voiceLibrary.filters.allLanguages')}</option>
|
||||
{languages.map(value => <option key={value} value={value}>{value}</option>)}
|
||||
</select>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div className="voice-library-list" role="listbox" aria-label={t('voiceLibrary.listLabel')}>
|
||||
{loading && (
|
||||
<div className="voice-library-loading"><LoadingSpinner size="lg" /><span>{t('voiceLibrary.loading')}</span></div>
|
||||
)}
|
||||
{!loading && !error && profiles.length === 0 && (
|
||||
<EmptyState
|
||||
icon="fa-microphone-lines"
|
||||
title={t('voiceLibrary.empty.title')}
|
||||
body={t('voiceLibrary.empty.body')}
|
||||
actions={<Link className="btn btn-primary" to="/app/voice-library/new">{t('voiceLibrary.actions.createFirst')}</Link>}
|
||||
className="voice-library-empty"
|
||||
/>
|
||||
)}
|
||||
{!loading && !error && profiles.length > 0 && filteredProfiles.length === 0 && (
|
||||
<EmptyState
|
||||
icon="fa-filter-circle-xmark"
|
||||
title={t('voiceLibrary.noResults.title')}
|
||||
body={t('voiceLibrary.noResults.body')}
|
||||
actions={<button type="button" className="btn btn-secondary" onClick={() => { setSearch(''); setLanguage('all') }}>{t('voiceLibrary.actions.clearFilters')}</button>}
|
||||
className="voice-library-empty"
|
||||
/>
|
||||
)}
|
||||
{!loading && !error && filteredProfiles.map(profile => (
|
||||
<button
|
||||
type="button"
|
||||
role="option"
|
||||
aria-selected={profile.id === selected?.id}
|
||||
className={`voice-row${profile.id === selected?.id ? ' voice-row--selected' : ''}`}
|
||||
key={profile.id}
|
||||
onClick={() => selectProfile(profile.id)}
|
||||
>
|
||||
<span className="voice-row__avatar">{profile.name.slice(0, 2).toLocaleUpperCase(i18n.language)}</span>
|
||||
<span className="voice-row__content">
|
||||
<span className="voice-row__head">
|
||||
<strong>{profile.name}</strong>
|
||||
<span>{formatDuration(profile.audio?.duration_ms)}</span>
|
||||
</span>
|
||||
<CompactWaveform />
|
||||
<span className="voice-row__meta">
|
||||
{profile.language || t('voiceLibrary.metadata.languageUnknown')}
|
||||
<span aria-hidden="true">·</span>
|
||||
{new Intl.DateTimeFormat(i18n.language, { dateStyle: 'medium' }).format(new Date(profile.created_at))}
|
||||
</span>
|
||||
</span>
|
||||
<i className="fas fa-chevron-right voice-row__chevron" aria-hidden="true" />
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="voice-library-detail">
|
||||
{!selected ? (
|
||||
<>
|
||||
<EmptyState
|
||||
icon="fa-wave-square"
|
||||
title={t('voiceLibrary.detail.emptyTitle')}
|
||||
body={t('voiceLibrary.detail.emptyBody')}
|
||||
/>
|
||||
{!modelsLoading && cloningModels.length === 0 && (
|
||||
<VoiceModelSetup
|
||||
t={t}
|
||||
galleryLoading={galleryLoading}
|
||||
installableModels={installableModels}
|
||||
galleryError={galleryError}
|
||||
installing={installing}
|
||||
operations={operations}
|
||||
onInstall={installModel}
|
||||
/>
|
||||
)}
|
||||
</>
|
||||
) : (
|
||||
<>
|
||||
<div className="voice-detail__header">
|
||||
<div>
|
||||
<span className="voice-detail__eyebrow">{t('voiceLibrary.detail.eyebrow')}</span>
|
||||
<h2>{selected.name}</h2>
|
||||
{selected.description && <p>{selected.description}</p>}
|
||||
</div>
|
||||
<StatusPill status="healthy" label={t('voiceLibrary.status.ready')} />
|
||||
</div>
|
||||
|
||||
<div className="voice-detail__player">
|
||||
<WaveformPlayer
|
||||
src={voiceProfilesApi.audioUrl(selected.id)}
|
||||
height={88}
|
||||
label={t('voiceLibrary.detail.referenceAudio')}
|
||||
audioTestId="voice-profile-audio"
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="voice-detail__section">
|
||||
<h3>{t('voiceLibrary.detail.transcript')}</h3>
|
||||
<blockquote>{selected.transcript}</blockquote>
|
||||
</div>
|
||||
|
||||
<dl className="voice-detail__metadata">
|
||||
<div><dt>{t('voiceLibrary.metadata.language')}</dt><dd>{selected.language || t('voiceLibrary.metadata.languageUnknown')}</dd></div>
|
||||
<div><dt>{t('voiceLibrary.metadata.duration')}</dt><dd>{formatDuration(selected.audio?.duration_ms)}</dd></div>
|
||||
<div><dt>{t('voiceLibrary.metadata.sampleRate')}</dt><dd>{Math.round((selected.audio?.sample_rate || 0) / 1000)} kHz</dd></div>
|
||||
<div><dt>{t('voiceLibrary.metadata.created')}</dt><dd>{new Intl.DateTimeFormat(i18n.language, { dateStyle: 'long' }).format(new Date(selected.created_at))}</dd></div>
|
||||
</dl>
|
||||
|
||||
<div className="voice-detail__consent">
|
||||
<i className="fas fa-shield-halved" aria-hidden="true" />
|
||||
<div><strong>{t('voiceLibrary.consent.confirmed')}</strong><span>{t('voiceLibrary.consent.confirmedAt', { date: new Intl.DateTimeFormat(i18n.language, { dateStyle: 'medium' }).format(new Date(selected.consent_confirmed_at)) })}</span></div>
|
||||
</div>
|
||||
|
||||
{!modelsLoading && cloningModels.length === 0 && (
|
||||
<VoiceModelSetup
|
||||
t={t}
|
||||
galleryLoading={galleryLoading}
|
||||
installableModels={installableModels}
|
||||
galleryError={galleryError}
|
||||
installing={installing}
|
||||
operations={operations}
|
||||
onInstall={installModel}
|
||||
/>
|
||||
)}
|
||||
|
||||
<details className="voice-detail__api">
|
||||
<summary>
|
||||
<span><strong>{t('voiceLibrary.api.title')}</strong><small>{t('voiceLibrary.api.summary')}</small></span>
|
||||
<i className="fas fa-chevron-down" aria-hidden="true" />
|
||||
</summary>
|
||||
<div className="voice-detail__api-body">
|
||||
<p>{t('voiceLibrary.api.body')}</p>
|
||||
<div className="voice-detail__compatible-models">
|
||||
<strong>{t('voiceLibrary.api.compatibleModels')}</strong>
|
||||
{cloningModels.length > 0 ? (
|
||||
<ul>
|
||||
{cloningModels.map(model => (
|
||||
<li key={model.id}><span>{model.id}</span><small>{model.backend}</small></li>
|
||||
))}
|
||||
</ul>
|
||||
) : (
|
||||
<span>{t('voiceLibrary.api.noInstalledModels')}</span>
|
||||
)}
|
||||
</div>
|
||||
<div className="voice-detail__code-heading">
|
||||
<span>{t('voiceLibrary.api.curlExample')}</span>
|
||||
<button type="button" className="btn btn-ghost btn-sm" onClick={copyAPIExample}>
|
||||
<i className="far fa-copy" aria-hidden="true" /> {t('voiceLibrary.api.copy')}
|
||||
</button>
|
||||
</div>
|
||||
<pre><code>{apiExample}</code></pre>
|
||||
<p className="voice-detail__api-note">{t('voiceLibrary.api.endpointNote')}</p>
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div className="voice-detail__actions">
|
||||
<button
|
||||
type="button"
|
||||
className="btn btn-primary"
|
||||
disabled={cloningModels.length === 0}
|
||||
onClick={() => navigate(`/app/tts?voice=${encodeURIComponent(selected.id)}`)}
|
||||
>
|
||||
<i className="fas fa-headphones" aria-hidden="true" /> {t('voiceLibrary.actions.useInTTS')}
|
||||
</button>
|
||||
<button type="button" className="btn btn-danger" onClick={() => setConfirmDelete(true)}>
|
||||
<i className="fas fa-trash" aria-hidden="true" /> {t('voiceLibrary.actions.delete')}
|
||||
</button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<ConfirmDialog
|
||||
open={confirmDelete}
|
||||
title={t('voiceLibrary.deleteDialog.title')}
|
||||
message={selected ? t('voiceLibrary.deleteDialog.message', { name: selected.name }) : ''}
|
||||
confirmLabel={deleting ? t('voiceLibrary.deleteDialog.deleting') : t('voiceLibrary.actions.delete')}
|
||||
danger
|
||||
onConfirm={deleteSelected}
|
||||
onCancel={() => !deleting && setConfirmDelete(false)}
|
||||
/>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
247
core/http/react-ui/src/pages/VoiceProfileCreate.jsx
Normal file
247
core/http/react-ui/src/pages/VoiceProfileCreate.jsx
Normal file
@@ -0,0 +1,247 @@
|
||||
import { useEffect, useMemo, useState } from 'react'
|
||||
import { Link, useNavigate, useOutletContext } from 'react-router-dom'
|
||||
import { useTranslation } from 'react-i18next'
|
||||
import LoadingSpinner from '../components/LoadingSpinner'
|
||||
import PageHeader from '../components/PageHeader'
|
||||
import UnsavedChangesGuard from '../components/UnsavedChangesGuard'
|
||||
import MediaInput from '../components/biometrics/MediaInput'
|
||||
import WaveformPlayer from '../components/audio/WaveformPlayer'
|
||||
import { audioBufferToWavBlob } from '../hooks/useMediaCapture'
|
||||
import { voiceProfilesApi } from '../utils/api'
|
||||
|
||||
const MAX_AUDIO_BYTES = 50 * 1024 * 1024
|
||||
const REFERENCE_SAMPLE_RATE = 24000
|
||||
|
||||
function base64ToArrayBuffer(value) {
|
||||
const binary = window.atob(value)
|
||||
const bytes = new Uint8Array(binary.length)
|
||||
for (let index = 0; index < binary.length; index += 1) bytes[index] = binary.charCodeAt(index)
|
||||
return bytes.buffer
|
||||
}
|
||||
|
||||
async function normalizeAudioSample(sample) {
|
||||
const source = sample.blob?.arrayBuffer
|
||||
? await sample.blob.arrayBuffer()
|
||||
: base64ToArrayBuffer(sample.base64)
|
||||
const AudioCtx = window.AudioContext || window.webkitAudioContext
|
||||
if (!AudioCtx) throw new Error('Web Audio API is not available in this browser')
|
||||
const context = new AudioCtx()
|
||||
try {
|
||||
const decoded = await context.decodeAudioData(source.slice(0))
|
||||
const blob = audioBufferToWavBlob(decoded, REFERENCE_SAMPLE_RATE)
|
||||
if (blob.size > MAX_AUDIO_BYTES) throw new Error('Normalized audio is larger than 50 MiB')
|
||||
return {
|
||||
...sample,
|
||||
blob,
|
||||
dataUrl: URL.createObjectURL(blob),
|
||||
objectUrl: true,
|
||||
mime: 'audio/wav',
|
||||
duration: decoded.duration,
|
||||
sampleRate: REFERENCE_SAMPLE_RATE,
|
||||
name: sample.name || 'recording.wav',
|
||||
}
|
||||
} finally {
|
||||
await context.close().catch(() => {})
|
||||
}
|
||||
}
|
||||
|
||||
function ReadinessItem({ ready, warning, children }) {
|
||||
const tone = ready ? 'ready' : warning ? 'warning' : 'pending'
|
||||
return (
|
||||
<li className={`voice-readiness__item voice-readiness__item--${tone}`}>
|
||||
<i className={`fas ${ready ? 'fa-check' : warning ? 'fa-triangle-exclamation' : 'fa-minus'}`} aria-hidden="true" />
|
||||
<span>{children}</span>
|
||||
</li>
|
||||
)
|
||||
}
|
||||
|
||||
export default function VoiceProfileCreate() {
|
||||
const { t } = useTranslation('media')
|
||||
const { addToast } = useOutletContext()
|
||||
const navigate = useNavigate()
|
||||
const [audio, setAudio] = useState(null)
|
||||
const [audioProcessing, setAudioProcessing] = useState(false)
|
||||
const [audioError, setAudioError] = useState('')
|
||||
const [name, setName] = useState('')
|
||||
const [description, setDescription] = useState('')
|
||||
const [language, setLanguage] = useState('')
|
||||
const [transcript, setTranscript] = useState('')
|
||||
const [consent, setConsent] = useState(false)
|
||||
const [submitting, setSubmitting] = useState(false)
|
||||
|
||||
useEffect(() => () => {
|
||||
if (audio?.objectUrl) URL.revokeObjectURL(audio.dataUrl)
|
||||
}, [audio])
|
||||
|
||||
const durationValid = audio?.duration >= 1 && audio?.duration <= 120
|
||||
const durationRecommended = audio?.duration >= 6 && audio?.duration <= 30
|
||||
const formReady = !!audio && durationValid && name.trim() && transcript.trim() && consent && !audioProcessing
|
||||
|
||||
const readiness = useMemo(() => ({
|
||||
audio: !!audio && durationValid,
|
||||
quality: !!audio && durationRecommended,
|
||||
name: !!name.trim(),
|
||||
transcript: !!transcript.trim(),
|
||||
consent,
|
||||
}), [audio, durationValid, durationRecommended, name, transcript, consent])
|
||||
|
||||
const handleAudio = async (sample) => {
|
||||
if (!sample) {
|
||||
setAudio(null)
|
||||
setAudioError('')
|
||||
return
|
||||
}
|
||||
setAudioProcessing(true)
|
||||
setAudioError('')
|
||||
try {
|
||||
const normalized = await normalizeAudioSample(sample)
|
||||
if (normalized.duration < 1 || normalized.duration > 120) {
|
||||
if (normalized.objectUrl) URL.revokeObjectURL(normalized.dataUrl)
|
||||
throw new Error(t('voiceCreate.audio.durationError'))
|
||||
}
|
||||
setAudio(normalized)
|
||||
} catch (err) {
|
||||
setAudio(null)
|
||||
setAudioError(err.message || t('voiceCreate.audio.decodeError'))
|
||||
} finally {
|
||||
setAudioProcessing(false)
|
||||
}
|
||||
}
|
||||
|
||||
const submit = async (event) => {
|
||||
event.preventDefault()
|
||||
if (!formReady) return
|
||||
setSubmitting(true)
|
||||
try {
|
||||
const formData = new FormData()
|
||||
formData.append('name', name.trim())
|
||||
formData.append('description', description.trim())
|
||||
formData.append('language', language.trim())
|
||||
formData.append('transcript', transcript.trim())
|
||||
formData.append('consent_confirmed', 'true')
|
||||
formData.append('audio', audio.blob, 'reference.wav')
|
||||
const profile = await voiceProfilesApi.create(formData)
|
||||
addToast(t('voiceCreate.toasts.created', { name: profile.name }), 'success')
|
||||
navigate(`/app/voice-library?selected=${encodeURIComponent(profile.id)}`)
|
||||
} catch (err) {
|
||||
addToast(err.message, 'error')
|
||||
} finally {
|
||||
setSubmitting(false)
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<main className="voice-create-page">
|
||||
<UnsavedChangesGuard
|
||||
when={!submitting && !!(audio || name || description || language || transcript || consent)}
|
||||
/>
|
||||
<PageHeader
|
||||
eyebrow={t('voiceCreate.eyebrow')}
|
||||
title={<><i className="fas fa-microphone-lines" aria-hidden="true" /> {t('voiceCreate.title')}</>}
|
||||
supporting={t('voiceCreate.subtitle')}
|
||||
actions={<Link className="btn btn-secondary" to="/app/voice-library"><i className="fas fa-arrow-left" aria-hidden="true" /> {t('voiceCreate.actions.back')}</Link>}
|
||||
/>
|
||||
|
||||
<form className="voice-create-grid" onSubmit={submit}>
|
||||
<div className="voice-create-form">
|
||||
<section className="voice-create-section" aria-labelledby="voice-reference-heading">
|
||||
<div className="voice-create-section__heading">
|
||||
<span>01</span>
|
||||
<div><h2 id="voice-reference-heading">{t('voiceCreate.sections.reference.title')}</h2><p>{t('voiceCreate.sections.reference.body')}</p></div>
|
||||
</div>
|
||||
<MediaInput
|
||||
mode="audio"
|
||||
label={t('voiceCreate.audio.label')}
|
||||
value={audio}
|
||||
onChange={handleAudio}
|
||||
onError={(error) => setAudioError(error.message)}
|
||||
maxBytes={MAX_AUDIO_BYTES}
|
||||
preferBlob
|
||||
idPrefix="voice-profile"
|
||||
/>
|
||||
{audioProcessing && <div className="voice-create-processing"><LoadingSpinner size="sm" /> {t('voiceCreate.audio.normalizing')}</div>}
|
||||
{audioError && <p className="form-error" role="alert">{audioError}</p>}
|
||||
{audio && (
|
||||
<div className="voice-create-preview">
|
||||
<WaveformPlayer src={audio.dataUrl} height={72} label={t('voiceCreate.audio.preview')} />
|
||||
<div className="voice-create-preview__meta">
|
||||
<span><i className="fas fa-clock" aria-hidden="true" /> {audio.duration.toFixed(1)}s</span>
|
||||
<span><i className="fas fa-wave-square" aria-hidden="true" /> {Math.round(audio.sampleRate / 1000)} kHz · PCM WAV</span>
|
||||
<span className={durationRecommended ? 'tone-success' : 'tone-warning'}>
|
||||
{durationRecommended ? t('voiceCreate.audio.qualityReady') : t('voiceCreate.audio.qualityHint')}
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</section>
|
||||
|
||||
<section className="voice-create-section" aria-labelledby="voice-details-heading">
|
||||
<div className="voice-create-section__heading">
|
||||
<span>02</span>
|
||||
<div><h2 id="voice-details-heading">{t('voiceCreate.sections.details.title')}</h2><p>{t('voiceCreate.sections.details.body')}</p></div>
|
||||
</div>
|
||||
<div className="form-grid-2col">
|
||||
<div className="form-group">
|
||||
<label className="form-label" htmlFor="voice-profile-name">{t('voiceCreate.fields.name')}</label>
|
||||
<input id="voice-profile-name" className="input" value={name} maxLength={80} onChange={(event) => setName(event.target.value)} placeholder={t('voiceCreate.fields.namePlaceholder')} required />
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label className="form-label" htmlFor="voice-profile-language">{t('voiceCreate.fields.language')}</label>
|
||||
<input id="voice-profile-language" className="input" value={language} maxLength={35} onChange={(event) => setLanguage(event.target.value)} placeholder={t('voiceCreate.fields.languagePlaceholder')} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<label className="form-label" htmlFor="voice-profile-description">{t('voiceCreate.fields.description')}</label>
|
||||
<input id="voice-profile-description" className="input" value={description} maxLength={500} onChange={(event) => setDescription(event.target.value)} placeholder={t('voiceCreate.fields.descriptionPlaceholder')} />
|
||||
</div>
|
||||
<div className="form-group">
|
||||
<div className="voice-create-label-row">
|
||||
<label className="form-label" htmlFor="voice-profile-transcript">{t('voiceCreate.fields.transcript')}</label>
|
||||
<span>{transcript.length}/4000</span>
|
||||
</div>
|
||||
<textarea id="voice-profile-transcript" className="textarea" rows={5} maxLength={4000} value={transcript} onChange={(event) => setTranscript(event.target.value)} placeholder={t('voiceCreate.fields.transcriptPlaceholder')} required />
|
||||
<p className="form-hint">{t('voiceCreate.fields.transcriptHint')}</p>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<section className="voice-create-section" aria-labelledby="voice-consent-heading">
|
||||
<div className="voice-create-section__heading">
|
||||
<span>03</span>
|
||||
<div><h2 id="voice-consent-heading">{t('voiceCreate.sections.consent.title')}</h2><p>{t('voiceCreate.sections.consent.body')}</p></div>
|
||||
</div>
|
||||
<label className="voice-consent-check">
|
||||
<input type="checkbox" checked={consent} onChange={(event) => setConsent(event.target.checked)} />
|
||||
<span className="voice-consent-check__box"><i className="fas fa-check" aria-hidden="true" /></span>
|
||||
<span><strong>{t('voiceCreate.consent.title')}</strong><small>{t('voiceCreate.consent.body')}</small></span>
|
||||
</label>
|
||||
</section>
|
||||
|
||||
<div className="voice-create-actions">
|
||||
<Link className="btn btn-secondary" to="/app/voice-library">{t('voiceCreate.actions.cancel')}</Link>
|
||||
<button type="submit" className="btn btn-primary" disabled={!formReady || submitting}>
|
||||
{submitting ? <><LoadingSpinner size="sm" /> {t('voiceCreate.actions.saving')}</> : <><i className="fas fa-floppy-disk" aria-hidden="true" /> {t('voiceCreate.actions.save')}</>}
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<aside className="voice-readiness" aria-label={t('voiceCreate.readiness.title')}>
|
||||
<div className="voice-readiness__header">
|
||||
<span className="voice-readiness__icon"><i className="fas fa-shield-halved" aria-hidden="true" /></span>
|
||||
<div><h2>{t('voiceCreate.readiness.title')}</h2><p>{t('voiceCreate.readiness.body')}</p></div>
|
||||
</div>
|
||||
<ul>
|
||||
<ReadinessItem ready={readiness.audio}>{t('voiceCreate.readiness.audio')}</ReadinessItem>
|
||||
<ReadinessItem ready={readiness.quality} warning={!!audio && !readiness.quality}>{t('voiceCreate.readiness.quality')}</ReadinessItem>
|
||||
<ReadinessItem ready={readiness.name}>{t('voiceCreate.readiness.name')}</ReadinessItem>
|
||||
<ReadinessItem ready={readiness.transcript}>{t('voiceCreate.readiness.transcript')}</ReadinessItem>
|
||||
<ReadinessItem ready={readiness.consent}>{t('voiceCreate.readiness.consent')}</ReadinessItem>
|
||||
</ul>
|
||||
<div className="voice-readiness__privacy">
|
||||
<i className="fas fa-lock" aria-hidden="true" />
|
||||
<p><strong>{t('voiceCreate.privacy.title')}</strong>{t('voiceCreate.privacy.body')}</p>
|
||||
</div>
|
||||
</aside>
|
||||
</form>
|
||||
</main>
|
||||
)
|
||||
}
|
||||
@@ -68,6 +68,8 @@ const Quantize = page('quantize', () => import('./pages/Quantize'))
|
||||
const Studio = page('studio', () => import('./pages/Studio'))
|
||||
const FaceRecognition = page('face', () => import('./pages/FaceRecognition'))
|
||||
const VoiceRecognition = page('voice', () => import('./pages/VoiceRecognition'))
|
||||
const VoiceLibrary = page('voice-library', () => import('./pages/VoiceLibrary'))
|
||||
const VoiceProfileCreate = page(null, () => import('./pages/VoiceProfileCreate'))
|
||||
const Nodes = page('nodes', () => import('./pages/Nodes'))
|
||||
const Scheduling = page('scheduling', () => import('./pages/Scheduling'))
|
||||
const NodeBackendLogs = page(null, () => import('./pages/NodeBackendLogs'))
|
||||
@@ -149,6 +151,7 @@ const appChildren = [
|
||||
element: <ConsoleLayout config={operateConsole} />,
|
||||
children: [
|
||||
{ path: 'backends', element: <Admin><Backends /></Admin> },
|
||||
{ path: 'voice-library', element: <Admin><VoiceLibrary /></Admin> },
|
||||
{ path: 'settings', element: <Admin><Settings /></Admin> },
|
||||
{ path: 'traces', element: <Admin><Traces /></Admin> },
|
||||
{ path: 'backend-logs/:modelId', element: <Admin><BackendLogs /></Admin> },
|
||||
@@ -166,6 +169,7 @@ const appChildren = [
|
||||
|
||||
// Models management (Install Models) — top-level destination, full-width.
|
||||
{ path: 'models', element: <Admin><Models /></Admin> },
|
||||
{ path: 'voice-library/new', element: <Admin><VoiceProfileCreate /></Admin> },
|
||||
{ path: 'model-editor', element: <Admin><ModelEditor /></Admin> },
|
||||
{ path: 'model-editor/:name', element: <Admin><ModelEditor /></Admin> },
|
||||
{ path: 'import-model', element: <Admin><ImportModel /></Admin> },
|
||||
|
||||
15
core/http/react-ui/src/utils/api.js
vendored
15
core/http/react-ui/src/utils/api.js
vendored
@@ -287,6 +287,21 @@ export const ttsApi = {
|
||||
generateV1: (body) => postAudioBlob(API_CONFIG.endpoints.audioSpeech, body),
|
||||
}
|
||||
|
||||
// Reusable voice-cloning profiles. The browser uploads multipart PCM-WAV;
|
||||
// list/preview are available to TTS-authorized users while mutations are
|
||||
// enforced as admin-only by the server.
|
||||
export const voiceProfilesApi = {
|
||||
list: () => fetchJSON('/api/voice-profiles'),
|
||||
create: (formData) => fetch(apiUrl('/api/voice-profiles'), {
|
||||
method: 'POST',
|
||||
body: formData,
|
||||
}).then(handleResponse),
|
||||
delete: (id) => fetch(apiUrl(`/api/voice-profiles/${enc(id)}`), {
|
||||
method: 'DELETE',
|
||||
}).then(handleResponse),
|
||||
audioUrl: (id) => apiUrl(`/api/voice-profiles/${enc(id)}/audio`),
|
||||
}
|
||||
|
||||
// Sound generation
|
||||
export const soundApi = {
|
||||
generate: (body) => postAudioBlob(API_CONFIG.endpoints.soundGeneration, body),
|
||||
|
||||
@@ -151,7 +151,13 @@ func RegisterLocalAIRoutes(router *echo.Echo,
|
||||
// Forget does not load a voice model — it only needs the registry.
|
||||
router.POST("/v1/voice/forget", localai.VoiceForgetEndpoint(app.VoiceRegistry()))
|
||||
|
||||
ttsHandler := localai.TTSEndpoint(cl, ml, appConfig)
|
||||
voiceProfiles := app.VoiceProfileStore()
|
||||
router.GET("/api/voice-profiles", localai.ListVoiceProfilesEndpoint(voiceProfiles))
|
||||
router.GET("/api/voice-profiles/:id/audio", localai.ServeVoiceProfileAudioEndpoint(voiceProfiles))
|
||||
router.POST("/api/voice-profiles", localai.CreateVoiceProfileEndpoint(voiceProfiles), adminMiddleware)
|
||||
router.DELETE("/api/voice-profiles/:id", localai.DeleteVoiceProfileEndpoint(voiceProfiles), adminMiddleware)
|
||||
|
||||
ttsHandler := localai.TTSEndpoint(cl, ml, appConfig, voiceProfiles)
|
||||
router.POST("/tts",
|
||||
ttsHandler,
|
||||
requestExtractor.BuildFilteredFirstAvailableDefaultModel(config.BuildUsecaseFilterFn(config.FLAG_TTS)),
|
||||
@@ -283,6 +289,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
|
||||
"autocomplete": "/api/models/config-metadata/autocomplete/:provider",
|
||||
"vram_estimate": "/api/models/vram-estimate",
|
||||
"tts": "/tts",
|
||||
"voice_profiles": "/api/voice-profiles",
|
||||
"transcription": "/v1/audio/transcriptions",
|
||||
"image_generation": "/v1/images/generations",
|
||||
"swagger": "/swagger/index.html",
|
||||
@@ -318,11 +325,12 @@ func RegisterLocalAIRoutes(router *echo.Echo,
|
||||
"list_aliases": "/api/aliases",
|
||||
},
|
||||
"ai_functions": map[string]string{
|
||||
"tts": "/tts",
|
||||
"vad": "/vad",
|
||||
"video": "/video",
|
||||
"detection": "/v1/detection",
|
||||
"tokenize": "/v1/tokenize",
|
||||
"tts": "/tts",
|
||||
"voice_profiles": "/api/voice-profiles",
|
||||
"vad": "/vad",
|
||||
"video": "/video",
|
||||
"detection": "/v1/detection",
|
||||
"tokenize": "/v1/tokenize",
|
||||
},
|
||||
"monitoring": monitoringRoutes,
|
||||
"mcp": map[string]string{
|
||||
@@ -363,6 +371,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
|
||||
"agents": appConfig.AgentPool.Enabled,
|
||||
"p2p": appConfig.P2PToken != "",
|
||||
"tracing": true,
|
||||
"voice_profiles": true,
|
||||
},
|
||||
})
|
||||
})
|
||||
|
||||
@@ -217,7 +217,7 @@ func RegisterOpenAIRoutes(app *echo.Echo,
|
||||
app.POST("/v1/audio/classification", soundClassificationHandler, soundClassificationMiddleware...)
|
||||
app.POST("/audio/classification", soundClassificationHandler, soundClassificationMiddleware...)
|
||||
|
||||
audioSpeechHandler := localai.TTSEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())
|
||||
audioSpeechHandler := localai.TTSEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig(), application.VoiceProfileStore())
|
||||
audioSpeechMiddleware := []echo.MiddlewareFunc{
|
||||
nodeHeaderMiddleware,
|
||||
traceMiddleware,
|
||||
|
||||
@@ -37,6 +37,7 @@ const (
|
||||
statusSortFieldName = "status"
|
||||
ascSortOrder = "asc"
|
||||
multimodalFilterKey = "multimodal"
|
||||
voiceCloningCapability = "voice_cloning"
|
||||
)
|
||||
|
||||
// usecaseFilters maps UI filter keys to ModelConfigUsecase flags for
|
||||
@@ -464,6 +465,27 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
models = filtered
|
||||
}
|
||||
|
||||
// Capability filters are derived from the effective gallery model
|
||||
// configuration. In particular, voice cloning is variant-sensitive, so
|
||||
// filtering by the TTS usecase or backend name alone would advertise
|
||||
// incompatible CustomVoice/VoiceDesign models.
|
||||
capabilityFilter := strings.ToLower(strings.TrimSpace(c.QueryParam("capability")))
|
||||
switch capabilityFilter {
|
||||
case "":
|
||||
case voiceCloningCapability:
|
||||
filtered := make(gallery.GalleryElements[*gallery.GalleryModel], 0, len(models))
|
||||
for _, m := range models {
|
||||
if m.VoiceCloningCapability(appConfig.SystemState.Model.ModelsPath) != nil {
|
||||
filtered = append(filtered, m)
|
||||
}
|
||||
}
|
||||
models = filtered
|
||||
default:
|
||||
return c.JSON(http.StatusBadRequest, map[string]any{
|
||||
"error": fmt.Sprintf("unsupported capability filter %q", capabilityFilter),
|
||||
})
|
||||
}
|
||||
|
||||
// Get model statuses
|
||||
processingModelsData, taskTypes := opcache.GetStatus()
|
||||
|
||||
@@ -545,6 +567,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
"trustRemoteCode": trustRemoteCodeExists,
|
||||
"additionalFiles": m.AdditionalFiles,
|
||||
"backend": m.Backend,
|
||||
"voice_cloning": m.VoiceCloningCapability(appConfig.SystemState.Model.ModelsPath),
|
||||
}
|
||||
|
||||
modelsJSON = append(modelsJSON, obj)
|
||||
@@ -597,11 +620,12 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
NodeStatus string `json:"node_status"`
|
||||
}
|
||||
type modelCapability struct {
|
||||
ID string `json:"id"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
Backend string `json:"backend"`
|
||||
Disabled bool `json:"disabled"`
|
||||
Pinned bool `json:"pinned"`
|
||||
ID string `json:"id"`
|
||||
Capabilities []string `json:"capabilities"`
|
||||
Backend string `json:"backend"`
|
||||
Disabled bool `json:"disabled"`
|
||||
Pinned bool `json:"pinned"`
|
||||
VoiceCloning *config.VoiceCloningCapability `json:"voice_cloning,omitempty"`
|
||||
// LoadedOn is populated only when the node registry is active
|
||||
// (distributed mode). Lets the UI show "loaded on worker-1" without
|
||||
// the operator having to expand every node manually. An empty slice
|
||||
@@ -649,6 +673,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
|
||||
Backend: cfg.Backend,
|
||||
Disabled: cfg.IsDisabled(),
|
||||
Pinned: cfg.IsPinned(),
|
||||
VoiceCloning: config.VoiceCloningForModel(&cfg),
|
||||
LoadedOn: loadedByModel[cfg.Name],
|
||||
})
|
||||
}
|
||||
|
||||
@@ -210,12 +210,23 @@ func (f *FileStagingClient) GenerateVideo(ctx context.Context, in *pb.GenerateVi
|
||||
}
|
||||
|
||||
func (f *FileStagingClient) TTS(ctx context.Context, in *pb.TTSRequest, opts ...ggrpc.CallOption) (*pb.Result, error) {
|
||||
reqID := requestID()
|
||||
|
||||
// Translate model path from frontend to remote worker path.
|
||||
// The model and its companion files (e.g. .onnx.json) were already staged
|
||||
// during LoadModel, so we just need to point to the correct remote location.
|
||||
if in.Model != "" && isFilePath(in.Model) {
|
||||
in.Model = f.translateModelPath(in.Model)
|
||||
}
|
||||
// Voice may be a named backend speaker or a request-scoped reference WAV.
|
||||
// Only path-shaped values are staged; speaker IDs pass through unchanged.
|
||||
if in.Voice != "" && isFilePath(in.Voice) {
|
||||
backendPath, _, err := f.stageInputFile(ctx, reqID, in.Voice, "inputs")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("staging TTS voice reference: %w", err)
|
||||
}
|
||||
in.Voice = backendPath
|
||||
}
|
||||
|
||||
// Handle output destination
|
||||
frontendDst := in.Dst
|
||||
@@ -242,10 +253,19 @@ func (f *FileStagingClient) TTS(ctx context.Context, in *pb.TTSRequest, opts ...
|
||||
}
|
||||
|
||||
func (f *FileStagingClient) TTSStream(ctx context.Context, in *pb.TTSRequest, fn func(*pb.Reply), opts ...ggrpc.CallOption) error {
|
||||
reqID := requestID()
|
||||
|
||||
// Translate model path from frontend to remote worker path (same as TTS above)
|
||||
if in.Model != "" && isFilePath(in.Model) {
|
||||
in.Model = f.translateModelPath(in.Model)
|
||||
}
|
||||
if in.Voice != "" && isFilePath(in.Voice) {
|
||||
backendPath, _, err := f.stageInputFile(ctx, reqID, in.Voice, "inputs")
|
||||
if err != nil {
|
||||
return fmt.Errorf("staging streaming TTS voice reference: %w", err)
|
||||
}
|
||||
in.Voice = backendPath
|
||||
}
|
||||
|
||||
return f.Backend.TTSStream(ctx, in, fn, opts...)
|
||||
}
|
||||
|
||||
66
core/services/nodes/file_staging_tts_test.go
Normal file
66
core/services/nodes/file_staging_tts_test.go
Normal file
@@ -0,0 +1,66 @@
|
||||
package nodes
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
grpc "github.com/mudler/LocalAI/pkg/grpc"
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
ggrpc "google.golang.org/grpc"
|
||||
)
|
||||
|
||||
type capturingTTSBackend struct {
|
||||
grpc.Backend
|
||||
ttsRequest *pb.TTSRequest
|
||||
streamTTSRequest *pb.TTSRequest
|
||||
}
|
||||
|
||||
func (b *capturingTTSBackend) TTS(_ context.Context, request *pb.TTSRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
|
||||
b.ttsRequest = request
|
||||
return &pb.Result{Success: true}, nil
|
||||
}
|
||||
|
||||
func (b *capturingTTSBackend) TTSStream(_ context.Context, request *pb.TTSRequest, _ func(*pb.Reply), _ ...ggrpc.CallOption) error {
|
||||
b.streamTTSRequest = request
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ = Describe("FileStagingClient TTS references", func() {
|
||||
It("stages a reference WAV before non-streaming synthesis", func(ctx SpecContext) {
|
||||
backend := &capturingTTSBackend{}
|
||||
stager := &fakeFileStager{}
|
||||
client := NewFileStagingClient(backend, stager, "worker-1")
|
||||
request := &pb.TTSRequest{Voice: "/data/voice-profiles/profile/reference.wav"}
|
||||
|
||||
_, err := client.TTS(ctx, request)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(stager.ensureCalls).To(HaveLen(1))
|
||||
Expect(stager.ensureCalls[0].localPath).To(Equal("/data/voice-profiles/profile/reference.wav"))
|
||||
Expect(backend.ttsRequest.Voice).To(HavePrefix("/remote/ephemeral/"))
|
||||
Expect(backend.ttsRequest.Voice).To(MatchRegexp(`/inputs/[0-9a-f]{8}/reference\.wav$`))
|
||||
})
|
||||
|
||||
It("stages a reference WAV before streaming synthesis", func(ctx SpecContext) {
|
||||
backend := &capturingTTSBackend{}
|
||||
stager := &fakeFileStager{}
|
||||
client := NewFileStagingClient(backend, stager, "worker-1")
|
||||
request := &pb.TTSRequest{Voice: "/data/reference.wav"}
|
||||
|
||||
Expect(client.TTSStream(ctx, request, func(*pb.Reply) {})).To(Succeed())
|
||||
Expect(stager.ensureCalls).To(HaveLen(1))
|
||||
Expect(backend.streamTTSRequest.Voice).To(HavePrefix("/remote/ephemeral/"))
|
||||
})
|
||||
|
||||
It("does not stage a named speaker", func(ctx SpecContext) {
|
||||
backend := &capturingTTSBackend{}
|
||||
stager := &fakeFileStager{}
|
||||
client := NewFileStagingClient(backend, stager, "worker-1")
|
||||
|
||||
_, err := client.TTS(ctx, &pb.TTSRequest{Voice: "serena"})
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(stager.ensureCalls).To(BeEmpty())
|
||||
Expect(backend.ttsRequest.Voice).To(Equal("serena"))
|
||||
})
|
||||
})
|
||||
601
core/services/voiceprofile/store.go
Normal file
601
core/services/voiceprofile/store.go
Normal file
@@ -0,0 +1,601 @@
|
||||
// Package voiceprofile stores reusable voice-cloning reference clips and
|
||||
// their transcripts. Profiles are immutable after creation so an in-flight
|
||||
// synthesis request always sees the same audio and text it selected.
|
||||
package voiceprofile
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/go-audio/wav"
|
||||
"github.com/google/uuid"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
const (
|
||||
// DirectoryName is the persistent subdirectory created below DataPath.
|
||||
DirectoryName = "voice-profiles"
|
||||
|
||||
// MaxAudioBytes caps a stored reference clip at 50 MiB. The HTTP and MCP
|
||||
// surfaces share this service-level limit so neither can bypass it.
|
||||
MaxAudioBytes int64 = 50 * 1024 * 1024
|
||||
|
||||
MinAudioDuration = time.Second
|
||||
MaxAudioDuration = 2 * time.Minute
|
||||
|
||||
maxNameRunes = 80
|
||||
maxDescriptionRunes = 500
|
||||
maxTranscriptRunes = 4000
|
||||
maxLanguageRunes = 35
|
||||
|
||||
metadataFileName = "profile.json"
|
||||
audioFileName = "reference.wav"
|
||||
referencePrefix = "localai://voice-profiles/"
|
||||
)
|
||||
|
||||
var (
|
||||
ErrNotFound = errors.New("voice profile not found")
|
||||
ErrInvalidInput = errors.New("invalid voice profile")
|
||||
ErrConsentRequired = errors.New("voice cloning consent must be confirmed")
|
||||
ErrAudioTooLarge = errors.New("voice reference exceeds the maximum size")
|
||||
ErrUnsupportedWAV = errors.New("voice reference must be a PCM WAV file")
|
||||
)
|
||||
|
||||
// AudioMetadata describes the validated reference clip without exposing its
|
||||
// filesystem location.
|
||||
type AudioMetadata struct {
|
||||
DurationMilliseconds int64 `json:"duration_ms"`
|
||||
SampleRate uint32 `json:"sample_rate"`
|
||||
Channels uint16 `json:"channels"`
|
||||
BitDepth uint16 `json:"bit_depth"`
|
||||
SizeBytes int64 `json:"size_bytes"`
|
||||
MIMEType string `json:"mime_type"`
|
||||
}
|
||||
|
||||
// Profile is the public, path-free representation of a saved cloned voice.
|
||||
type Profile struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description,omitempty"`
|
||||
Language string `json:"language,omitempty"`
|
||||
Transcript string `json:"transcript"`
|
||||
Voice string `json:"voice"`
|
||||
ConsentConfirmedAt time.Time `json:"consent_confirmed_at"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
UpdatedAt time.Time `json:"updated_at"`
|
||||
Audio AudioMetadata `json:"audio"`
|
||||
}
|
||||
|
||||
// CreateInput contains the administrator-provided profile metadata.
|
||||
type CreateInput struct {
|
||||
Name string
|
||||
Description string
|
||||
Language string
|
||||
Transcript string
|
||||
ConsentConfirmed bool
|
||||
}
|
||||
|
||||
// Store persists voice profiles below one root-confined directory. A Store is
|
||||
// safe for concurrent use.
|
||||
type Store struct {
|
||||
baseDir string
|
||||
leaseDir string
|
||||
initErr error
|
||||
now func() time.Time
|
||||
|
||||
initOnce sync.Once
|
||||
mu sync.RWMutex
|
||||
}
|
||||
|
||||
// NewStore creates a profile store rooted at <dataPath>/voice-profiles. It
|
||||
// performs no filesystem I/O; errors are reported by the first operation.
|
||||
func NewStore(dataPath string) *Store {
|
||||
s := &Store{
|
||||
leaseDir: filepath.Join(".leases", uuid.NewString()),
|
||||
now: time.Now,
|
||||
}
|
||||
if strings.TrimSpace(dataPath) == "" {
|
||||
s.initErr = errors.New("voiceprofile: data path is empty")
|
||||
return s
|
||||
}
|
||||
abs, err := filepath.Abs(filepath.Join(dataPath, DirectoryName))
|
||||
if err != nil {
|
||||
s.initErr = fmt.Errorf("voiceprofile: resolve data path: %w", err)
|
||||
return s
|
||||
}
|
||||
s.baseDir = abs
|
||||
return s
|
||||
}
|
||||
|
||||
// Reference returns the stable value clients pass in TTSRequest.voice.
|
||||
func Reference(id string) string {
|
||||
return referencePrefix + id
|
||||
}
|
||||
|
||||
// IsReference reports whether value uses the saved-profile URI scheme. It is
|
||||
// useful for distinguishing a malformed profile URI from a backend speaker
|
||||
// ID before ParseReference validates the UUID.
|
||||
func IsReference(value string) bool {
|
||||
return strings.HasPrefix(value, referencePrefix)
|
||||
}
|
||||
|
||||
// ParseReference recognizes a saved-profile voice value. The bool is false
|
||||
// for ordinary backend speaker IDs and paths, which remain backwards
|
||||
// compatible.
|
||||
func ParseReference(value string) (string, bool) {
|
||||
if !IsReference(value) {
|
||||
return "", false
|
||||
}
|
||||
id := strings.TrimPrefix(value, referencePrefix)
|
||||
if validateID(id) != nil {
|
||||
return "", false
|
||||
}
|
||||
return id, true
|
||||
}
|
||||
|
||||
func (s *Store) initialize() error {
|
||||
s.initOnce.Do(func() {
|
||||
if s.initErr != nil {
|
||||
return
|
||||
}
|
||||
if err := os.MkdirAll(s.baseDir, 0o750); err != nil {
|
||||
s.initErr = fmt.Errorf("voiceprofile: create store: %w", err)
|
||||
return
|
||||
}
|
||||
root, err := os.OpenRoot(s.baseDir)
|
||||
if err != nil {
|
||||
s.initErr = fmt.Errorf("voiceprofile: open store: %w", err)
|
||||
return
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
if err := root.MkdirAll(s.leaseDir, 0o700); err != nil {
|
||||
s.initErr = fmt.Errorf("voiceprofile: create lease directory: %w", err)
|
||||
}
|
||||
})
|
||||
return s.initErr
|
||||
}
|
||||
|
||||
func (s *Store) openRoot() (*os.Root, error) {
|
||||
if err := s.initialize(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
root, err := os.OpenRoot(s.baseDir)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("voiceprofile: open store: %w", err)
|
||||
}
|
||||
return root, nil
|
||||
}
|
||||
|
||||
// Create validates and atomically persists a profile and its WAV clip.
|
||||
func (s *Store) Create(ctx context.Context, input CreateInput, audio io.Reader) (Profile, error) {
|
||||
input = normalizeInput(input)
|
||||
if err := validateInput(input); err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
if audio == nil {
|
||||
return Profile{}, fmt.Errorf("%w: audio is required", ErrInvalidInput)
|
||||
}
|
||||
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
id := uuid.NewString()
|
||||
tempDir := ".creating-" + uuid.NewString()
|
||||
if err := root.Mkdir(tempDir, 0o700); err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: create temporary profile: %w", err)
|
||||
}
|
||||
committed := false
|
||||
defer func() {
|
||||
if !committed {
|
||||
_ = root.RemoveAll(tempDir)
|
||||
}
|
||||
}()
|
||||
|
||||
audioPath := filepath.Join(tempDir, audioFileName)
|
||||
dst, err := root.OpenFile(audioPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: create reference audio: %w", err)
|
||||
}
|
||||
|
||||
written, copyErr := copyWithContext(ctx, dst, io.LimitReader(audio, MaxAudioBytes+1))
|
||||
if syncErr := dst.Sync(); copyErr == nil && syncErr != nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := dst.Close(); copyErr == nil && closeErr != nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: store reference audio: %w", copyErr)
|
||||
}
|
||||
if written > MaxAudioBytes {
|
||||
return Profile{}, ErrAudioTooLarge
|
||||
}
|
||||
|
||||
storedAudio, err := root.Open(audioPath)
|
||||
if err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: reopen reference audio: %w", err)
|
||||
}
|
||||
audioMeta, validateErr := validateWAV(storedAudio, written)
|
||||
_ = storedAudio.Close()
|
||||
if validateErr != nil {
|
||||
return Profile{}, validateErr
|
||||
}
|
||||
|
||||
now := s.now().UTC()
|
||||
profile := Profile{
|
||||
ID: id,
|
||||
Name: input.Name,
|
||||
Description: input.Description,
|
||||
Language: input.Language,
|
||||
Transcript: input.Transcript,
|
||||
Voice: Reference(id),
|
||||
ConsentConfirmedAt: now,
|
||||
CreatedAt: now,
|
||||
UpdatedAt: now,
|
||||
Audio: audioMeta,
|
||||
}
|
||||
if err := writeJSON(root, filepath.Join(tempDir, metadataFileName), profile); err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
if err := root.Rename(tempDir, id); err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: commit profile: %w", err)
|
||||
}
|
||||
committed = true
|
||||
_ = syncDirectory(root)
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
// List returns every readable profile, newest first. One damaged entry is
|
||||
// skipped and logged rather than making the entire library unavailable.
|
||||
func (s *Store) List(_ context.Context) ([]Profile, error) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
entries, err := fs.ReadDir(root.FS(), ".")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("voiceprofile: list store: %w", err)
|
||||
}
|
||||
profiles := make([]Profile, 0, len(entries))
|
||||
for _, entry := range entries {
|
||||
if !entry.IsDir() || validateID(entry.Name()) != nil {
|
||||
continue
|
||||
}
|
||||
profile, err := readProfile(root, entry.Name())
|
||||
if err != nil {
|
||||
xlog.Warn("Skipping unreadable voice profile", "id", entry.Name(), "error", err)
|
||||
continue
|
||||
}
|
||||
profiles = append(profiles, profile)
|
||||
}
|
||||
sort.Slice(profiles, func(i, j int) bool {
|
||||
if profiles[i].UpdatedAt.Equal(profiles[j].UpdatedAt) {
|
||||
return profiles[i].Name < profiles[j].Name
|
||||
}
|
||||
return profiles[i].UpdatedAt.After(profiles[j].UpdatedAt)
|
||||
})
|
||||
return profiles, nil
|
||||
}
|
||||
|
||||
// Get returns one profile by opaque UUID.
|
||||
func (s *Store) Get(_ context.Context, id string) (Profile, error) {
|
||||
if err := validateID(id); err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return Profile{}, err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
return readProfile(root, id)
|
||||
}
|
||||
|
||||
// OpenAudio opens a profile's reference clip for authenticated preview.
|
||||
func (s *Store) OpenAudio(_ context.Context, id string) (*os.File, Profile, error) {
|
||||
if err := validateID(id); err != nil {
|
||||
return nil, Profile{}, err
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return nil, Profile{}, err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
profile, err := readProfile(root, id)
|
||||
if err != nil {
|
||||
return nil, Profile{}, err
|
||||
}
|
||||
file, err := root.Open(filepath.Join(id, audioFileName))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return nil, Profile{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return nil, Profile{}, fmt.Errorf("voiceprofile: open reference audio: %w", err)
|
||||
}
|
||||
return file, profile, nil
|
||||
}
|
||||
|
||||
// LeaseAudio creates a request-scoped hard link to the immutable clip. This
|
||||
// keeps an in-flight local or remote-worker synthesis safe if an administrator
|
||||
// deletes the library entry concurrently. The caller must invoke release.
|
||||
func (s *Store) LeaseAudio(_ context.Context, id string) (Profile, string, func(), error) {
|
||||
if err := validateID(id); err != nil {
|
||||
return Profile{}, "", nil, err
|
||||
}
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return Profile{}, "", nil, err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
|
||||
profile, err := readProfile(root, id)
|
||||
if err != nil {
|
||||
return Profile{}, "", nil, err
|
||||
}
|
||||
source := filepath.Join(id, audioFileName)
|
||||
lease := filepath.Join(s.leaseDir, uuid.NewString()+".wav")
|
||||
if err := root.Link(source, lease); err != nil {
|
||||
if err := copyWithinRoot(root, source, lease); err != nil {
|
||||
return Profile{}, "", nil, fmt.Errorf("voiceprofile: lease reference audio: %w", err)
|
||||
}
|
||||
}
|
||||
|
||||
var once sync.Once
|
||||
release := func() {
|
||||
once.Do(func() {
|
||||
leaseRoot, openErr := s.openRoot()
|
||||
if openErr != nil {
|
||||
xlog.Warn("Failed to open voice profile store while releasing audio lease", "error", openErr)
|
||||
return
|
||||
}
|
||||
defer func() { _ = leaseRoot.Close() }()
|
||||
if removeErr := leaseRoot.Remove(lease); removeErr != nil && !errors.Is(removeErr, fs.ErrNotExist) {
|
||||
xlog.Warn("Failed to release voice profile audio", "error", removeErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
return profile, filepath.Join(s.baseDir, lease), release, nil
|
||||
}
|
||||
|
||||
// Delete removes one profile. Existing audio leases remain valid until their
|
||||
// synthesis requests complete.
|
||||
func (s *Store) Delete(_ context.Context, id string) error {
|
||||
if err := validateID(id); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
if _, err := root.Stat(id); errors.Is(err, fs.ErrNotExist) {
|
||||
return ErrNotFound
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("voiceprofile: stat profile: %w", err)
|
||||
}
|
||||
if err := root.RemoveAll(id); err != nil {
|
||||
return fmt.Errorf("voiceprofile: delete profile: %w", err)
|
||||
}
|
||||
_ = syncDirectory(root)
|
||||
return nil
|
||||
}
|
||||
|
||||
// Close removes this process's empty request-lease directory. Profiles are
|
||||
// persistent and are never removed by Close.
|
||||
func (s *Store) Close() error {
|
||||
if s == nil || s.baseDir == "" {
|
||||
return nil
|
||||
}
|
||||
root, err := s.openRoot()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = root.Close() }()
|
||||
if err := root.RemoveAll(s.leaseDir); err != nil && !errors.Is(err, fs.ErrNotExist) {
|
||||
return fmt.Errorf("voiceprofile: remove lease directory: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func normalizeInput(input CreateInput) CreateInput {
|
||||
input.Name = strings.TrimSpace(input.Name)
|
||||
input.Description = strings.TrimSpace(input.Description)
|
||||
input.Language = strings.TrimSpace(input.Language)
|
||||
input.Transcript = strings.TrimSpace(input.Transcript)
|
||||
return input
|
||||
}
|
||||
|
||||
func validateInput(input CreateInput) error {
|
||||
if input.Name == "" {
|
||||
return fmt.Errorf("%w: name is required", ErrInvalidInput)
|
||||
}
|
||||
if !utf8.ValidString(input.Name) || utf8.RuneCountInString(input.Name) > maxNameRunes {
|
||||
return fmt.Errorf("%w: name must be valid UTF-8 and at most %d characters", ErrInvalidInput, maxNameRunes)
|
||||
}
|
||||
if !utf8.ValidString(input.Description) || utf8.RuneCountInString(input.Description) > maxDescriptionRunes {
|
||||
return fmt.Errorf("%w: description must be valid UTF-8 and at most %d characters", ErrInvalidInput, maxDescriptionRunes)
|
||||
}
|
||||
if !utf8.ValidString(input.Language) || utf8.RuneCountInString(input.Language) > maxLanguageRunes {
|
||||
return fmt.Errorf("%w: language must be valid UTF-8 and at most %d characters", ErrInvalidInput, maxLanguageRunes)
|
||||
}
|
||||
if input.Transcript == "" {
|
||||
return fmt.Errorf("%w: transcript is required", ErrInvalidInput)
|
||||
}
|
||||
if !utf8.ValidString(input.Transcript) || utf8.RuneCountInString(input.Transcript) > maxTranscriptRunes {
|
||||
return fmt.Errorf("%w: transcript must be valid UTF-8 and at most %d characters", ErrInvalidInput, maxTranscriptRunes)
|
||||
}
|
||||
if !input.ConsentConfirmed {
|
||||
return ErrConsentRequired
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateID(id string) error {
|
||||
parsed, err := uuid.Parse(id)
|
||||
if err != nil || parsed.String() != id {
|
||||
return fmt.Errorf("%w: invalid id", ErrInvalidInput)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateWAV(file *os.File, size int64) (AudioMetadata, error) {
|
||||
decoder := wav.NewDecoder(file)
|
||||
if !decoder.IsValidFile() {
|
||||
return AudioMetadata{}, ErrUnsupportedWAV
|
||||
}
|
||||
duration, err := decoder.Duration()
|
||||
if err != nil {
|
||||
return AudioMetadata{}, fmt.Errorf("%w: cannot read duration: %v", ErrUnsupportedWAV, err)
|
||||
}
|
||||
if decoder.WavAudioFormat != 1 || decoder.BitDepth != 16 {
|
||||
return AudioMetadata{}, fmt.Errorf("%w: expected 16-bit PCM", ErrUnsupportedWAV)
|
||||
}
|
||||
if decoder.NumChans < 1 || decoder.NumChans > 2 {
|
||||
return AudioMetadata{}, fmt.Errorf("%w: expected mono or stereo audio", ErrUnsupportedWAV)
|
||||
}
|
||||
if decoder.SampleRate < 8000 || decoder.SampleRate > 192000 {
|
||||
return AudioMetadata{}, fmt.Errorf("%w: sample rate must be between 8 kHz and 192 kHz", ErrUnsupportedWAV)
|
||||
}
|
||||
if duration < MinAudioDuration || duration > MaxAudioDuration {
|
||||
return AudioMetadata{}, fmt.Errorf("%w: duration must be between %s and %s", ErrInvalidInput, MinAudioDuration, MaxAudioDuration)
|
||||
}
|
||||
return AudioMetadata{
|
||||
DurationMilliseconds: duration.Milliseconds(),
|
||||
SampleRate: decoder.SampleRate,
|
||||
Channels: decoder.NumChans,
|
||||
BitDepth: decoder.BitDepth,
|
||||
SizeBytes: size,
|
||||
MIMEType: "audio/wav",
|
||||
}, nil
|
||||
}
|
||||
|
||||
func readProfile(root *os.Root, id string) (Profile, error) {
|
||||
raw, err := root.ReadFile(filepath.Join(id, metadataFileName))
|
||||
if errors.Is(err, fs.ErrNotExist) {
|
||||
return Profile{}, ErrNotFound
|
||||
}
|
||||
if err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: read profile metadata: %w", err)
|
||||
}
|
||||
var profile Profile
|
||||
if err := json.Unmarshal(raw, &profile); err != nil {
|
||||
return Profile{}, fmt.Errorf("voiceprofile: decode profile metadata: %w", err)
|
||||
}
|
||||
if profile.ID != id {
|
||||
return Profile{}, errors.New("voiceprofile: metadata id does not match directory")
|
||||
}
|
||||
profile.Voice = Reference(id)
|
||||
return profile, nil
|
||||
}
|
||||
|
||||
func writeJSON(root *os.Root, name string, value any) error {
|
||||
file, err := root.OpenFile(name, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return fmt.Errorf("voiceprofile: create metadata: %w", err)
|
||||
}
|
||||
encoder := json.NewEncoder(file)
|
||||
encoder.SetEscapeHTML(true)
|
||||
encodeErr := encoder.Encode(value)
|
||||
if syncErr := file.Sync(); encodeErr == nil && syncErr != nil {
|
||||
encodeErr = syncErr
|
||||
}
|
||||
if closeErr := file.Close(); encodeErr == nil && closeErr != nil {
|
||||
encodeErr = closeErr
|
||||
}
|
||||
if encodeErr != nil {
|
||||
return fmt.Errorf("voiceprofile: write metadata: %w", encodeErr)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func copyWithinRoot(root *os.Root, source, destination string) error {
|
||||
src, err := root.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = src.Close() }()
|
||||
dst, err := root.OpenFile(destination, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, copyErr := io.Copy(dst, src)
|
||||
if syncErr := dst.Sync(); copyErr == nil && syncErr != nil {
|
||||
copyErr = syncErr
|
||||
}
|
||||
if closeErr := dst.Close(); copyErr == nil && closeErr != nil {
|
||||
copyErr = closeErr
|
||||
}
|
||||
if copyErr != nil {
|
||||
_ = root.Remove(destination)
|
||||
}
|
||||
return copyErr
|
||||
}
|
||||
|
||||
func copyWithContext(ctx context.Context, dst io.Writer, src io.Reader) (int64, error) {
|
||||
buf := make([]byte, 32*1024)
|
||||
var written int64
|
||||
for {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return written, err
|
||||
}
|
||||
n, readErr := src.Read(buf)
|
||||
if n > 0 {
|
||||
m, writeErr := dst.Write(buf[:n])
|
||||
written += int64(m)
|
||||
if writeErr != nil {
|
||||
return written, writeErr
|
||||
}
|
||||
if m != n {
|
||||
return written, io.ErrShortWrite
|
||||
}
|
||||
}
|
||||
if errors.Is(readErr, io.EOF) {
|
||||
return written, nil
|
||||
}
|
||||
if readErr != nil {
|
||||
return written, readErr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func syncDirectory(root *os.Root) error {
|
||||
dir, err := root.Open(".")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() { _ = dir.Close() }()
|
||||
return dir.Sync()
|
||||
}
|
||||
122
core/services/voiceprofile/store_test.go
Normal file
122
core/services/voiceprofile/store_test.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package voiceprofile_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"time"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
)
|
||||
|
||||
func pcmWAV(duration time.Duration) []byte {
|
||||
const (
|
||||
sampleRate = 16000
|
||||
channels = 1
|
||||
bitsPerSample = 16
|
||||
)
|
||||
samples := int(duration.Seconds() * sampleRate)
|
||||
dataSize := samples * channels * bitsPerSample / 8
|
||||
buf := bytes.NewBuffer(make([]byte, 0, 44+dataSize))
|
||||
buf.WriteString("RIFF")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(36+dataSize))
|
||||
buf.WriteString("WAVEfmt ")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(16))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(1))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(channels))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(sampleRate*channels*bitsPerSample/8))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(channels*bitsPerSample/8))
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint16(bitsPerSample))
|
||||
buf.WriteString("data")
|
||||
_ = binary.Write(buf, binary.LittleEndian, uint32(dataSize))
|
||||
buf.Write(make([]byte, dataSize))
|
||||
return buf.Bytes()
|
||||
}
|
||||
|
||||
var _ = Describe("Store", func() {
|
||||
var store *voiceprofile.Store
|
||||
|
||||
BeforeEach(func() {
|
||||
store = voiceprofile.NewStore(GinkgoT().TempDir())
|
||||
DeferCleanup(func() { Expect(store.Close()).To(Succeed()) })
|
||||
})
|
||||
|
||||
It("atomically creates, lists, opens, leases, and deletes a profile", func(ctx SpecContext) {
|
||||
created, err := store.Create(ctx, voiceprofile.CreateInput{
|
||||
Name: " Studio narrator ",
|
||||
Description: "Warm and measured",
|
||||
Language: "en-US",
|
||||
Transcript: "This reference sentence is spoken clearly.",
|
||||
ConsentConfirmed: true,
|
||||
}, bytes.NewReader(pcmWAV(2*time.Second)))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(created.ID).To(MatchRegexp(`^[0-9a-f-]{36}$`))
|
||||
Expect(created.Name).To(Equal("Studio narrator"))
|
||||
Expect(created.Voice).To(Equal(voiceprofile.Reference(created.ID)))
|
||||
Expect(created.Audio.DurationMilliseconds).To(BeNumerically("~", 2000, 2))
|
||||
Expect(created.Audio.SampleRate).To(Equal(uint32(16000)))
|
||||
|
||||
profiles, err := store.List(ctx)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(profiles).To(ConsistOf(created))
|
||||
|
||||
file, profile, err := store.OpenAudio(ctx, created.ID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(profile).To(Equal(created))
|
||||
opened, err := os.ReadFile(file.Name())
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(file.Close()).To(Succeed())
|
||||
Expect(opened).To(Equal(pcmWAV(2 * time.Second)))
|
||||
|
||||
_, leasePath, release, err := store.LeaseAudio(ctx, created.ID)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(leasePath).To(BeAnExistingFile())
|
||||
Expect(store.Delete(ctx, created.ID)).To(Succeed())
|
||||
Expect(leasePath).To(BeAnExistingFile(), "a concurrent synthesis lease must survive deletion")
|
||||
release()
|
||||
Expect(leasePath).NotTo(BeAnExistingFile())
|
||||
|
||||
_, err = store.Get(ctx, created.ID)
|
||||
Expect(errors.Is(err, voiceprofile.ErrNotFound)).To(BeTrue())
|
||||
})
|
||||
|
||||
DescribeTable("rejects invalid creation input",
|
||||
func(input voiceprofile.CreateInput, audio []byte, expected error) {
|
||||
_, err := store.Create(GinkgoT().Context(), input, bytes.NewReader(audio))
|
||||
Expect(errors.Is(err, expected)).To(BeTrue(), "error was %v", err)
|
||||
},
|
||||
Entry("missing name", voiceprofile.CreateInput{Transcript: "words", ConsentConfirmed: true}, pcmWAV(2*time.Second), voiceprofile.ErrInvalidInput),
|
||||
Entry("missing transcript", voiceprofile.CreateInput{Name: "Voice", ConsentConfirmed: true}, pcmWAV(2*time.Second), voiceprofile.ErrInvalidInput),
|
||||
Entry("missing consent", voiceprofile.CreateInput{Name: "Voice", Transcript: "words"}, pcmWAV(2*time.Second), voiceprofile.ErrConsentRequired),
|
||||
Entry("non-WAV audio", voiceprofile.CreateInput{Name: "Voice", Transcript: "words", ConsentConfirmed: true}, []byte("not audio"), voiceprofile.ErrUnsupportedWAV),
|
||||
Entry("too-short audio", voiceprofile.CreateInput{Name: "Voice", Transcript: "words", ConsentConfirmed: true}, pcmWAV(500*time.Millisecond), voiceprofile.ErrInvalidInput),
|
||||
)
|
||||
|
||||
It("rejects traversal and non-canonical profile IDs", func(ctx SpecContext) {
|
||||
_, err := store.Get(ctx, "../../etc/passwd")
|
||||
Expect(errors.Is(err, voiceprofile.ErrInvalidInput)).To(BeTrue())
|
||||
Expect(store.Delete(ctx, "NOT-A-UUID")).To(MatchError(ContainSubstring("invalid id")))
|
||||
})
|
||||
|
||||
It("stores private files below the data path", func(ctx SpecContext) {
|
||||
dataPath := GinkgoT().TempDir()
|
||||
privateStore := voiceprofile.NewStore(dataPath)
|
||||
DeferCleanup(func() { Expect(privateStore.Close()).To(Succeed()) })
|
||||
created, err := privateStore.Create(ctx, voiceprofile.CreateInput{
|
||||
Name: "Private",
|
||||
Transcript: "A private reference.",
|
||||
ConsentConfirmed: true,
|
||||
}, bytes.NewReader(pcmWAV(time.Second)))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
|
||||
audioInfo, err := os.Stat(filepath.Join(dataPath, voiceprofile.DirectoryName, created.ID, "reference.wav"))
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(audioInfo.Mode().Perm()).To(Equal(os.FileMode(0o600)))
|
||||
})
|
||||
})
|
||||
13
core/services/voiceprofile/voiceprofile_suite_test.go
Normal file
13
core/services/voiceprofile/voiceprofile_suite_test.go
Normal file
@@ -0,0 +1,13 @@
|
||||
package voiceprofile_test
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
func TestVoiceProfile(t *testing.T) {
|
||||
RegisterFailHandler(Fail)
|
||||
RunSpecs(t, "Voice profile service suite")
|
||||
}
|
||||
@@ -674,8 +674,25 @@ For text-to-speech models:
|
||||
|
||||
| Field | Type | Description |
|
||||
|-------|------|-------------|
|
||||
| `tts.voice` | string | Voice file path or voice ID |
|
||||
| `tts.audio_path` | string | Path to audio files (for Vall-E) |
|
||||
| `tts.voice` | string | Default backend voice ID, speaker name, or reference path. A request `voice` takes precedence. |
|
||||
| `tts.audio_path` | string | Default reference-audio path for cloning backends. A request voice or saved Voice Library profile takes precedence. |
|
||||
| `tts.voice_cloning` | bool | Optional Voice Library capability override. Omit for automatic backend/variant detection; `true` opts in a verified custom-named variant and `false` rejects saved profile references. |
|
||||
|
||||
For example, a custom-named model on a known cloning backend can declare support explicitly while retaining a model-wide reference fallback:
|
||||
|
||||
```yaml
|
||||
name: private-voice-model
|
||||
backend: qwen3-tts-cpp
|
||||
parameters:
|
||||
model: private/qwen-talker-base.gguf
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
audio_path: voices/default-reference.wav
|
||||
```
|
||||
|
||||
`tts.voice_cloning: true` only overrides model-variant detection. It cannot enable cloning on a backend that does not implement LocalAI's reference-audio contract.
|
||||
|
||||
## Roles Configuration
|
||||
|
||||
|
||||
@@ -29,6 +29,112 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
|
||||
|
||||
Returns an `audio/wav` file.
|
||||
|
||||
## Voice Library
|
||||
|
||||
Administrators can manage reusable voice-cloning references from **Operate → Voice Library** in the LocalAI WebUI. The library replaces per-model filesystem and YAML setup for supported cloning backends:
|
||||
|
||||
1. Select **Create voice** and upload or record a clear reference clip.
|
||||
2. Enter the exact words spoken in the clip. The transcript is sent to backends that require reference text.
|
||||
3. Confirm that you have permission to clone the voice, then save the profile.
|
||||
4. Open **Text to Speech**, choose a model marked **Cloning ready**, and select the saved voice.
|
||||
|
||||
The browser converts uploads and recordings to mono, 24 kHz, 16-bit PCM WAV so the same profile works across compatible backends. Clips must be between 1 and 120 seconds and no larger than 50 MiB; 6–30 seconds of clean, single-speaker audio is recommended. Profile audio is private biometric source material: LocalAI stores it below its configured data path, serves previews only to authenticated TTS users, and never returns its filesystem path.
|
||||
|
||||
### Voice profile API
|
||||
|
||||
The WebUI uses the following endpoints. Creating and deleting profiles requires administrator access; listing profiles and playing previews requires access to the TTS feature.
|
||||
|
||||
| Method | Endpoint | Purpose |
|
||||
| --- | --- | --- |
|
||||
| `GET` | `/api/voice-profiles` | List saved profiles and their public metadata. |
|
||||
| `POST` | `/api/voice-profiles` | Create a profile from multipart form data or JSON with base64 audio. |
|
||||
| `GET` | `/api/voice-profiles/{id}/audio` | Stream the authenticated WAV preview, including range requests. |
|
||||
| `DELETE` | `/api/voice-profiles/{id}` | Permanently delete a profile. |
|
||||
|
||||
For example, an administrator can create a profile without the WebUI:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/api/voice-profiles \
|
||||
-F 'name=Documentary narrator' \
|
||||
-F 'language=en-US' \
|
||||
-F 'transcript=The exact words spoken in this reference.' \
|
||||
-F 'consent_confirmed=true' \
|
||||
-F 'audio=@reference.wav;type=audio/wav'
|
||||
```
|
||||
|
||||
The response includes an opaque voice reference such as `localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000`. Pass that value as `voice` to either TTS-compatible endpoint:
|
||||
|
||||
```bash
|
||||
curl http://localhost:8080/v1/audio/speech \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{
|
||||
"model": "qwen3-tts-base",
|
||||
"input": "This sentence will use the saved voice.",
|
||||
"voice": "localai://voice-profiles/550e8400-e29b-41d4-a716-446655440000"
|
||||
}' --output speech.wav
|
||||
```
|
||||
|
||||
LocalAI resolves the opaque reference only for models that advertise voice-cloning support. Existing named speakers, backend-specific voice IDs, and explicit model YAML voice configuration remain available for models and advanced workflows that do not use the library.
|
||||
|
||||
The Voice Library uses the same server-side capability resolver for installed models and gallery recommendations. Administrators can inspect the currently configured galleries without maintaining a separate backend list:
|
||||
|
||||
```bash
|
||||
curl 'http://localhost:8080/api/models?capability=voice_cloning&items=20'
|
||||
```
|
||||
|
||||
Each returned model includes a non-null `voice_cloning` contract. Variant checks are applied before the model is returned, so TTS-only CustomVoice, VoiceDesign, or preset-prompt variants are not offered as reference-audio models. When the WebUI detects that no compatible model is installed, it uses this response to offer direct installation.
|
||||
|
||||
### Model configuration
|
||||
|
||||
Voice Library support is automatic for known backends and model variants. A custom model can override that detection with `tts.voice_cloning`:
|
||||
|
||||
```yaml
|
||||
name: private-qwen-base
|
||||
backend: qwen3-tts-cpp
|
||||
parameters:
|
||||
model: private/qwen-talker-checkpoint.gguf
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
# Optional: omit this for automatic backend and variant detection.
|
||||
voice_cloning: true
|
||||
# Optional model-wide fallback when a request does not select a saved profile.
|
||||
audio_path: voices/default-reference.wav
|
||||
options:
|
||||
- tokenizer:private/qwen-tokenizer.gguf
|
||||
```
|
||||
|
||||
`tts.voice_cloning` has three states:
|
||||
|
||||
| Value | Behavior |
|
||||
| --- | --- |
|
||||
| omitted | Detect support from the backend plus `name`, `parameters.model`, and compatibility options. This is recommended for gallery models. |
|
||||
| `true` | Advertise Voice Library support for a custom-named variant of a backend that LocalAI already knows can clone voices. This cannot add cloning to an unsupported backend. |
|
||||
| `false` | Hide the model from Voice Library compatibility results and reject `localai://voice-profiles/...` references for it. Backend-specific named voices and manual reference paths remain available. |
|
||||
|
||||
The older `options: ["voice_cloning:true"]` and `options: ["voice_cloning:false"]` spellings remain accepted for compatibility. Prefer `tts.voice_cloning`; generic options may be forwarded to a backend, whereas the typed field is consumed only by LocalAI.
|
||||
|
||||
Reference selection follows this order:
|
||||
|
||||
1. A request `voice`, including a saved `localai://voice-profiles/...` URI.
|
||||
2. The model's `tts.voice` default.
|
||||
3. The model's `tts.audio_path` reference-audio fallback.
|
||||
4. A backend-specific default voice or option.
|
||||
|
||||
When a saved profile is selected, LocalAI supplies both its private WAV and exact transcript for that request. It does not rewrite the model YAML or copy the recording into the model directory.
|
||||
|
||||
#### Supported backend and model variants
|
||||
|
||||
| Backend | Automatically compatible variants |
|
||||
| --- | --- |
|
||||
| `chatterbox`, `faster-qwen3-tts`, `fish-speech`, `neutts`, `omnivoice-cpp`, `pocket-tts`, `voxcpm` | Reference-audio cloning models served by these dedicated backends. |
|
||||
| `qwen-tts`, `qwen3-tts-cpp`, `vllm-omni` | Base or VoiceClone variants. CustomVoice and VoiceDesign variants are not raw reference-audio models. |
|
||||
| `vibevoice-cpp` | 1.5B reference-WAV variants. The realtime 0.5B preset-prompt model is excluded. |
|
||||
| `coqui` | XTTS and YourTTS variants. |
|
||||
| `crispasr` | F5-TTS variants. ASR, Piper, Orpheus, and other CrispASR model families are excluded. |
|
||||
|
||||
This table describes the built-in resolver, not a frontend allowlist. Gallery entries and installed configs are evaluated by the server, and `tts.voice_cloning` can make a verified custom filename explicit.
|
||||
|
||||
## Streaming TTS
|
||||
|
||||
LocalAI supports streaming TTS generation, allowing audio to be played as it's generated. This is useful for real-time applications and reduces latency.
|
||||
@@ -132,7 +238,7 @@ Future versions of LocalAI will expose additional control over audio generation
|
||||
|
||||
#### Setup
|
||||
|
||||
Install the `ace-step-turbo` model from the Model gallery or run `local-ai run models install ace-step-turbo`.
|
||||
Install the `ace-step-turbo` model from the Model gallery or run `local-ai models install ace-step-turbo`.
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -186,11 +292,11 @@ options:
|
||||
|
||||
### VibeVoice
|
||||
|
||||
[VibeVoice-Realtime](https://github.com/microsoft/VibeVoice) is a real-time text-to-speech model that generates natural-sounding speech with voice cloning capabilities.
|
||||
[VibeVoice-Realtime](https://github.com/microsoft/VibeVoice) is a real-time text-to-speech model that generates natural-sounding speech from precomputed voice presets.
|
||||
|
||||
#### Setup
|
||||
|
||||
Install the `vibevoice` model in the Model gallery or run `local-ai run models install vibevoice`.
|
||||
Install the `vibevoice` model in the Model gallery or run `local-ai models install vibevoice`.
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -203,9 +309,9 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
|
||||
}' | aplay
|
||||
```
|
||||
|
||||
#### Voice cloning
|
||||
#### Voice presets
|
||||
|
||||
VibeVoice supports voice cloning through voice preset files. You can configure a model with a specific voice:
|
||||
The Python `vibevoice` realtime 0.5B model uses `.pt` voice preset files. You can configure a model with a specific preset:
|
||||
|
||||
```yaml
|
||||
name: vibevoice
|
||||
@@ -220,6 +326,10 @@ tts:
|
||||
Then you can use the model:
|
||||
|
||||
```
|
||||
|
||||
{{% notice note %}}
|
||||
The realtime 0.5B preset model is not advertised to the Voice Library because it does not accept a raw reference WAV per request. For Voice Library profiles, use a `vibevoice-cpp` 1.5B reference-WAV model; LocalAI detects the 1.5B variant automatically, or a custom name can set `tts.voice_cloning: true`.
|
||||
{{% /notice %}}
|
||||
curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
|
||||
"model": "vibevoice",
|
||||
"input":"Hello!"
|
||||
@@ -232,7 +342,7 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
|
||||
|
||||
#### Setup
|
||||
|
||||
Install the `omnivoice-cpp` model in the Model gallery or run `local-ai run models install omnivoice-cpp`. A higher-quality BF16 variant is available as `omnivoice-cpp-hq` (the default `omnivoice-cpp` ships Q8_0 GGUFs).
|
||||
Install the `omnivoice-cpp` model in the Model gallery or run `local-ai models install omnivoice-cpp`. A higher-quality BF16 variant is available as `omnivoice-cpp-hq` (the default `omnivoice-cpp` ships Q8_0 GGUFs).
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -266,6 +376,7 @@ backend: omnivoice-cpp
|
||||
parameters:
|
||||
model: omnivoice-cpp/omnivoice-base-Q8_0.gguf
|
||||
tts:
|
||||
voice_cloning: true # optional explicit declaration; gallery models are auto-detected
|
||||
audio_path: "voices/my_reference.wav" # default cloning reference (or use tts.voice)
|
||||
options:
|
||||
- "tokenizer:omnivoice-cpp/omnivoice-tokenizer-Q8_0.gguf"
|
||||
@@ -308,7 +419,7 @@ A per-request `seed` can also be supplied through the `params` map alongside `re
|
||||
|
||||
#### Setup
|
||||
|
||||
Install the `pocket-tts` model in the Model gallery or run `local-ai run models install pocket-tts`.
|
||||
Install the `pocket-tts` model in the Model gallery or run `local-ai models install pocket-tts`.
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -335,6 +446,16 @@ tts:
|
||||
# Available built-in voices: alba, marius, javert, jean, fantine, cosette, eponine, azelma
|
||||
```
|
||||
|
||||
To make a reference recording the model-wide fallback, use `tts.audio_path`. The gallery model is detected automatically; `tts.voice_cloning` is only needed when you want an explicit declaration:
|
||||
|
||||
```yaml
|
||||
name: pocket-tts-clone
|
||||
backend: pocket-tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
audio_path: "voices/reference.wav"
|
||||
```
|
||||
|
||||
You can also pre-load a default voice for faster first generation:
|
||||
|
||||
```yaml
|
||||
@@ -359,7 +480,25 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
|
||||
|
||||
#### Setup
|
||||
|
||||
Install the `qwen-tts` model in the Model gallery or run `local-ai run models install qwen-tts`.
|
||||
Install the `qwen-tts` model in the Model gallery or run `local-ai models install qwen-tts`.
|
||||
|
||||
#### C++ / GGML gallery variants
|
||||
|
||||
For a native backend, install one of the Base variants `qwen3-tts-cpp`, `qwen3-tts-cpp-0.6b-base-q4`, `qwen3-tts-cpp-1.7b-base`, or `qwen3-tts-cpp-1.7b-base-q4`. These variants accept saved Voice Library profiles and are advertised automatically. Gallery entries containing `customvoice` or `voicedesign` provide their respective Qwen modes but are intentionally excluded from raw reference-audio cloning.
|
||||
|
||||
A private Qwen C++ Base conversion with an opaque filename can declare the capability explicitly. The tokenizer GGUF can sit beside the talker GGUF for automatic discovery:
|
||||
|
||||
```yaml
|
||||
name: company-narrator-engine
|
||||
backend: qwen3-tts-cpp
|
||||
parameters:
|
||||
model: qwen-private/talker.gguf
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
audio_path: voices/default-reference.wav # optional fallback
|
||||
```
|
||||
|
||||
#### Usage
|
||||
|
||||
@@ -476,6 +615,7 @@ backend: qwen-tts
|
||||
parameters:
|
||||
model: Qwen/Qwen3-TTS-12Hz-1.7B-Base
|
||||
tts:
|
||||
voice_cloning: true # optional for this Base model; useful when a private checkpoint has an opaque name
|
||||
audio_path: "path/to/reference_audio.wav" # Reference audio file
|
||||
options:
|
||||
- "ref_text:This is the transcript of the reference audio."
|
||||
@@ -544,10 +684,9 @@ The multi-voice mode is backward compatible with existing single-voice configura
|
||||
|
||||
You can also use a `config-file` to specify TTS models and their parameters.
|
||||
|
||||
In the following example we define a custom config to load the `xtts_v2` model, and specify a voice and language.
|
||||
In the following example, a custom config loads `xtts_v2` with a default cloning reference and language.
|
||||
|
||||
```yaml
|
||||
|
||||
name: xtts_v2
|
||||
backend: coqui
|
||||
parameters:
|
||||
@@ -555,9 +694,12 @@ parameters:
|
||||
model: tts_models/multilingual/multi-dataset/xtts_v2
|
||||
|
||||
tts:
|
||||
voice: Ana Florence
|
||||
voice_cloning: true
|
||||
audio_path: voices/reference.wav
|
||||
```
|
||||
|
||||
For XTTS/YourTTS, `tts.audio_path` is the default cloning reference and a saved Voice Library profile overrides it per request. Other Coqui model families are not advertised as Voice Library-compatible unless they match the supported variant rules or are explicitly verified with `tts.voice_cloning: true`.
|
||||
|
||||
With this config, you can now use the following curl command to generate a text-to-speech audio file:
|
||||
```bash
|
||||
curl -L http://localhost:8080/tts \
|
||||
|
||||
@@ -5076,6 +5076,8 @@
|
||||
backend: voxcpm
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: openbmb/VoxCPM1.5
|
||||
- name: neutts-air
|
||||
@@ -5096,6 +5098,8 @@
|
||||
backend: neutts
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: neuphonic/neutts-air
|
||||
- name: vllm-omni-z-image-turbo
|
||||
@@ -5428,8 +5432,9 @@
|
||||
- https://github.com/microsoft/VibeVoice
|
||||
description: |
|
||||
VibeVoice Realtime 0.5B (C++ / GGML, Q8_0) - native C++ port of Microsoft VibeVoice
|
||||
via the vibevoice-cpp backend. 24kHz mono TTS with voice cloning from a single
|
||||
reference voice prompt. Default voice prompt: en-Carter_man.
|
||||
via the vibevoice-cpp backend. 24kHz mono TTS with a selectable precomputed
|
||||
voice prompt. Default voice prompt: en-Carter_man. This realtime variant does
|
||||
not accept raw Voice Library reference WAVs.
|
||||
license: mit
|
||||
icon: https://github.com/microsoft/VibeVoice/raw/main/Figures/VibeVoice_logo_white.png
|
||||
tags:
|
||||
@@ -5442,7 +5447,6 @@
|
||||
- gguf
|
||||
- ggml
|
||||
- 0.5b
|
||||
- voice-cloning
|
||||
- quantized
|
||||
last_checked: "2026-04-30"
|
||||
overrides:
|
||||
@@ -5791,6 +5795,8 @@
|
||||
known_usecases:
|
||||
- tts
|
||||
name: omnivoice-cpp
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: omnivoice-cpp/omnivoice-base-Q8_0.gguf
|
||||
options:
|
||||
@@ -5824,6 +5830,8 @@
|
||||
known_usecases:
|
||||
- tts
|
||||
name: omnivoice-cpp-hq
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: omnivoice-cpp-hq/omnivoice-base-BF16.gguf
|
||||
options:
|
||||
@@ -6141,6 +6149,8 @@
|
||||
backend: fish-speech
|
||||
known_usecases:
|
||||
- tts
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: fishaudio/s2-pro
|
||||
- name: qwen3-asr-1.7b
|
||||
@@ -7164,6 +7174,7 @@
|
||||
tags:
|
||||
- text-to-speech
|
||||
- tts
|
||||
- voice-cloning
|
||||
size: 236MB
|
||||
- name: qwen3-vl-30b-a3b-instruct
|
||||
url: github:mudler/LocalAI/gallery/qwen3.yaml@master
|
||||
@@ -9029,7 +9040,7 @@
|
||||
urls:
|
||||
- https://github.com/resemble-ai/chatterbox
|
||||
description: |
|
||||
Chatterbox, Resemble AI's first production-grade open source TTS model. Licensed under MIT, Chatterbox has been benchmarked against leading closed-source systems like ElevenLabs, and is consistently preferred in side-by-side evaluations.
|
||||
Chatterbox, Resemble AI's first production-grade open source TTS model. Supports zero-shot voice cloning from a reference WAV, including saved LocalAI Voice Library profiles. Licensed under MIT, Chatterbox has been benchmarked against leading closed-source systems like ElevenLabs, and is consistently preferred in side-by-side evaluations.
|
||||
license: mit
|
||||
icon: https://private-user-images.githubusercontent.com/660224/448166653-bd8c5f03-e91d-4ee5-b680-57355da204d1.png
|
||||
tags:
|
||||
@@ -9037,12 +9048,15 @@
|
||||
- dia
|
||||
- gpu
|
||||
- text-to-speech
|
||||
- voice-cloning
|
||||
size: 3.2GB
|
||||
overrides:
|
||||
backend: chatterbox
|
||||
known_usecases:
|
||||
- tts
|
||||
name: chatterbox
|
||||
tts:
|
||||
voice_cloning: true
|
||||
- name: dia
|
||||
url: github:mudler/LocalAI/gallery/virtual.yaml@master
|
||||
urls:
|
||||
@@ -37184,7 +37198,7 @@
|
||||
description: |
|
||||
F5-TTS v1 Base (SWivid, MIT) text-to-speech synthesized through the CrispASR backend. A 22-layer DiT flow-matching model with a built-in Vocos vocoder in a single self-contained GGUF (no separate codec). Auto-detected by CrispASR and runs end-to-end on CPU, producing 24 kHz mono audio.
|
||||
|
||||
F5-TTS is a voice-cloning model with no built-in speaker: you must supply a reference clip and its transcript. Add `voice:<reference.wav>` and `voice_text:<transcript of the reference>` to the model's `options` (paths resolve against the model directory) before synthesizing; without a reference the model cannot generate audio.
|
||||
F5-TTS is a voice-cloning model with no built-in speaker: you must supply a reference clip and its exact transcript. Select a saved LocalAI Voice Library profile per request, or configure `voice:<reference.wav>` plus `voice_text:<transcript>` in the model's `options` as a model-wide fallback; without a reference the model cannot generate audio.
|
||||
|
||||
Default GGUF size ~945 MB (f16). Quantization below f16 is not recommended for flow-matching models. Synthesis runs a 32-step ODE solver and is compute-heavy on CPU (expect long generation times without a GPU-enabled CrispASR build).
|
||||
tags:
|
||||
@@ -37198,6 +37212,8 @@
|
||||
known_usecases:
|
||||
- tts
|
||||
name: f5-tts-crispasr
|
||||
tts:
|
||||
voice_cloning: true
|
||||
parameters:
|
||||
model: f5-tts-v1-base-f16.gguf
|
||||
files:
|
||||
|
||||
@@ -13,6 +13,9 @@ config_file: |-
|
||||
|
||||
# TTS configuration
|
||||
tts:
|
||||
# Advertise saved Voice Library profiles. A per-request saved profile
|
||||
# overrides voice/audio_path/default_voice without changing this YAML.
|
||||
voice_cloning: true
|
||||
# Voice selection - can be:
|
||||
# 1. Built-in voice name (e.g., "alba", "marius", "javert", "jean", "fantine", "cosette", "eponine", "azelma")
|
||||
# 2. HuggingFace URL (e.g., "hf://kyutai/tts-voices/alba-mackenna/casual.wav")
|
||||
|
||||
@@ -80,6 +80,11 @@ type LocalAIClient interface {
|
||||
// exposed over MCP — admins use the Settings UI for binary files.
|
||||
SetBranding(ctx context.Context, req SetBrandingRequest) (*Branding, error)
|
||||
|
||||
// ---- Voice profile library ----
|
||||
ListVoiceProfiles(ctx context.Context) ([]VoiceProfile, error)
|
||||
CreateVoiceProfile(ctx context.Context, req CreateVoiceProfileRequest) (*VoiceProfile, error)
|
||||
DeleteVoiceProfile(ctx context.Context, id string) error
|
||||
|
||||
// ---- Usage / billing ----
|
||||
|
||||
// GetUsageStats returns aggregated token usage. In single-user
|
||||
|
||||
@@ -42,20 +42,23 @@ var toolToHTTPRoute = map[string]string{
|
||||
ToolGetMiddlewareStatus: "GET /api/middleware/status",
|
||||
ToolGetRouterDecisions: "GET /api/router/decisions",
|
||||
ToolListAliases: "GET /api/aliases",
|
||||
ToolListVoiceProfiles: "GET /api/voice-profiles",
|
||||
|
||||
// Mutating tools.
|
||||
ToolInstallModel: "POST /models/apply",
|
||||
ToolImportModelURI: "POST /models/import-uri",
|
||||
ToolDeleteModel: "POST /models/delete/:name",
|
||||
ToolEditModelConfig: "PATCH /api/models/config-json/:name",
|
||||
ToolReloadModels: "POST /models/reload",
|
||||
ToolLoadModel: "POST /backend/load",
|
||||
ToolInstallBackend: "POST /backends/apply",
|
||||
ToolUpgradeBackend: "POST /backends/upgrade/:name",
|
||||
ToolToggleModelState: "PUT /models/toggle-state/:name/:action",
|
||||
ToolToggleModelPinned: "PUT /models/toggle-pinned/:name/:action",
|
||||
ToolSetBranding: "POST /api/settings (instance_name, instance_tagline)",
|
||||
ToolSetAlias: "PATCH /api/models/config-json/:name (swap) or POST /models/import (create)",
|
||||
ToolInstallModel: "POST /models/apply",
|
||||
ToolImportModelURI: "POST /models/import-uri",
|
||||
ToolDeleteModel: "POST /models/delete/:name",
|
||||
ToolEditModelConfig: "PATCH /api/models/config-json/:name",
|
||||
ToolReloadModels: "POST /models/reload",
|
||||
ToolLoadModel: "POST /backend/load",
|
||||
ToolInstallBackend: "POST /backends/apply",
|
||||
ToolUpgradeBackend: "POST /backends/upgrade/:name",
|
||||
ToolToggleModelState: "PUT /models/toggle-state/:name/:action",
|
||||
ToolToggleModelPinned: "PUT /models/toggle-pinned/:name/:action",
|
||||
ToolSetBranding: "POST /api/settings (instance_name, instance_tagline)",
|
||||
ToolSetAlias: "PATCH /api/models/config-json/:name (swap) or POST /models/import (create)",
|
||||
ToolCreateVoiceProfile: "POST /api/voice-profiles",
|
||||
ToolDeleteVoiceProfile: "DELETE /api/voice-profiles/:id",
|
||||
}
|
||||
|
||||
// allKnownTools is the union of expectedFullCatalog (defined in
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package localaitools
|
||||
|
||||
import "github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
|
||||
// DTOs for the LocalAIClient interface. Where the same shape already exists
|
||||
// elsewhere (config.Gallery, gallery.Metadata, schema.KnownBackend,
|
||||
// vram.EstimateResult) we surface that type directly via the interface
|
||||
@@ -145,6 +147,28 @@ type SetBrandingRequest struct {
|
||||
InstanceTagline *string `json:"instance_tagline,omitempty" jsonschema:"Optional short subtitle shown beneath the instance name. Pass an empty string to clear."`
|
||||
}
|
||||
|
||||
// VoiceProfile is the same path-free shape returned by the REST library.
|
||||
// Keeping the service type avoids REST/MCP field drift.
|
||||
type VoiceProfile = voiceprofile.Profile
|
||||
|
||||
// CreateVoiceProfileRequest is the MCP/JSON form of profile creation. Audio
|
||||
// must be a base64-encoded 16-bit PCM WAV (mono 24 kHz is recommended for
|
||||
// portability); the service enforces the same 50 MiB and duration limits as
|
||||
// the browser upload route.
|
||||
type CreateVoiceProfileRequest struct {
|
||||
Name string `json:"name" jsonschema:"Display name for the reusable voice profile."`
|
||||
Description string `json:"description,omitempty" jsonschema:"Optional note describing tone, source, or intended use."`
|
||||
Language string `json:"language,omitempty" jsonschema:"Optional BCP-47-style language tag such as en-US."`
|
||||
Transcript string `json:"transcript" jsonschema:"Exact transcript of the words spoken in the reference clip."`
|
||||
AudioBase64 string `json:"audio_base64" jsonschema:"Base64-encoded 16-bit PCM WAV reference, preferably mono 24 kHz, 1-120 seconds and at most 50 MiB decoded."`
|
||||
ConsentConfirmed bool `json:"consent_confirmed" jsonschema:"Must be true to confirm authorization to clone this voice."`
|
||||
}
|
||||
|
||||
// DeleteVoiceProfileRequest identifies the profile to remove.
|
||||
type DeleteVoiceProfileRequest struct {
|
||||
ID string `json:"id" jsonschema:"Opaque voice profile UUID returned by list_voice_profiles."`
|
||||
}
|
||||
|
||||
// UsageStatsQuery is the input for get_usage_stats. UserID is optional;
|
||||
// when empty the tool returns the calling user's own usage in auth-on
|
||||
// mode, or the synthetic local user's usage in single-user no-auth
|
||||
|
||||
@@ -47,6 +47,9 @@ type fakeClient struct {
|
||||
toggleModelPinned func(string, modeladmin.Action) error
|
||||
getBranding func() (*Branding, error)
|
||||
setBranding func(SetBrandingRequest) (*Branding, error)
|
||||
listVoiceProfiles func() ([]VoiceProfile, error)
|
||||
createVoiceProfile func(CreateVoiceProfileRequest) (*VoiceProfile, error)
|
||||
deleteVoiceProfile func(string) error
|
||||
getUsageStats func(UsageStatsQuery) (*UsageStats, error)
|
||||
getPIIEvents func(PIIEventsQuery) ([]PIIEvent, error)
|
||||
getMiddlewareStatus func() (*MiddlewareStatus, error)
|
||||
@@ -266,6 +269,30 @@ func (f *fakeClient) SetBranding(_ context.Context, req SetBrandingRequest) (*Br
|
||||
return &Branding{InstanceName: "LocalAI"}, nil
|
||||
}
|
||||
|
||||
func (f *fakeClient) ListVoiceProfiles(_ context.Context) ([]VoiceProfile, error) {
|
||||
f.record("ListVoiceProfiles", nil)
|
||||
if f.listVoiceProfiles != nil {
|
||||
return f.listVoiceProfiles()
|
||||
}
|
||||
return []VoiceProfile{}, nil
|
||||
}
|
||||
|
||||
func (f *fakeClient) CreateVoiceProfile(_ context.Context, req CreateVoiceProfileRequest) (*VoiceProfile, error) {
|
||||
f.record("CreateVoiceProfile", req)
|
||||
if f.createVoiceProfile != nil {
|
||||
return f.createVoiceProfile(req)
|
||||
}
|
||||
return &VoiceProfile{ID: "00000000-0000-0000-0000-000000000001", Name: req.Name}, nil
|
||||
}
|
||||
|
||||
func (f *fakeClient) DeleteVoiceProfile(_ context.Context, id string) error {
|
||||
f.record("DeleteVoiceProfile", id)
|
||||
if f.deleteVoiceProfile != nil {
|
||||
return f.deleteVoiceProfile(id)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeClient) GetUsageStats(_ context.Context, q UsageStatsQuery) (*UsageStats, error) {
|
||||
f.record("GetUsageStats", q)
|
||||
if f.getUsageStats != nil {
|
||||
|
||||
@@ -552,6 +552,33 @@ func (c *Client) SetBranding(ctx context.Context, req localaitools.SetBrandingRe
|
||||
return c.GetBranding(ctx)
|
||||
}
|
||||
|
||||
// ---- Voice profile library ----
|
||||
|
||||
func (c *Client) ListVoiceProfiles(ctx context.Context) ([]localaitools.VoiceProfile, error) {
|
||||
var response struct {
|
||||
Data []localaitools.VoiceProfile `json:"data"`
|
||||
}
|
||||
if err := c.do(ctx, http.MethodGet, routeVoiceProfiles, nil, &response); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return response.Data, nil
|
||||
}
|
||||
|
||||
func (c *Client) CreateVoiceProfile(ctx context.Context, req localaitools.CreateVoiceProfileRequest) (*localaitools.VoiceProfile, error) {
|
||||
var profile localaitools.VoiceProfile
|
||||
if err := c.do(ctx, http.MethodPost, routeVoiceProfiles, req, &profile); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteVoiceProfile(ctx context.Context, id string) error {
|
||||
if id == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
return c.do(ctx, http.MethodDelete, routeVoiceProfileDelete(id), nil, nil)
|
||||
}
|
||||
|
||||
// ---- Usage / billing ----
|
||||
|
||||
func (c *Client) GetUsageStats(ctx context.Context, q localaitools.UsageStatsQuery) (*localaitools.UsageStats, error) {
|
||||
|
||||
@@ -32,6 +32,7 @@ const (
|
||||
routePIIEvents = "/api/pii/events"
|
||||
routeMiddleware = "/api/middleware/status"
|
||||
routeRouterDecisions = "/api/router/decisions"
|
||||
routeVoiceProfiles = "/api/voice-profiles"
|
||||
)
|
||||
|
||||
func routeJobStatus(jobID string) string {
|
||||
@@ -57,3 +58,7 @@ func routeToggleModelState(name, action string) string {
|
||||
func routeToggleModelPinned(name, action string) string {
|
||||
return fmt.Sprintf("/models/toggle-pinned/%s/%s", url.PathEscape(name), url.PathEscape(action))
|
||||
}
|
||||
|
||||
func routeVoiceProfileDelete(id string) string {
|
||||
return "/api/voice-profiles/" + url.PathEscape(id)
|
||||
}
|
||||
|
||||
@@ -6,11 +6,13 @@ package inproc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/mudler/LocalAI/core/backend"
|
||||
@@ -24,6 +26,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/routing/billing"
|
||||
"github.com/mudler/LocalAI/core/services/routing/pii"
|
||||
"github.com/mudler/LocalAI/core/services/routing/router"
|
||||
"github.com/mudler/LocalAI/core/services/voiceprofile"
|
||||
"github.com/mudler/LocalAI/internal"
|
||||
localaitools "github.com/mudler/LocalAI/pkg/mcp/localaitools"
|
||||
"github.com/mudler/LocalAI/pkg/model"
|
||||
@@ -39,11 +42,12 @@ import (
|
||||
// distributed-aware, ModelConfigLoader manages on-disk YAML, etc.), so this
|
||||
// layer just translates between MCP DTOs and service signatures.
|
||||
type Client struct {
|
||||
AppConfig *config.ApplicationConfig
|
||||
SystemState *system.SystemState
|
||||
ConfigLoader *config.ModelConfigLoader
|
||||
ModelLoader *model.ModelLoader
|
||||
Gallery *galleryop.GalleryService
|
||||
AppConfig *config.ApplicationConfig
|
||||
SystemState *system.SystemState
|
||||
ConfigLoader *config.ModelConfigLoader
|
||||
ModelLoader *model.ModelLoader
|
||||
Gallery *galleryop.GalleryService
|
||||
VoiceProfiles *voiceprofile.Store
|
||||
|
||||
// StatsRecorder and FallbackUser are optional — they back the
|
||||
// get_usage_stats tool. nil StatsRecorder makes the tool return an
|
||||
@@ -73,12 +77,13 @@ type Client struct {
|
||||
// fields (StatsRecorder, FallbackUser) which gate get_usage_stats.
|
||||
func New(appConfig *config.ApplicationConfig, systemState *system.SystemState, cl *config.ModelConfigLoader, ml *model.ModelLoader, gs *galleryop.GalleryService) *Client {
|
||||
return &Client{
|
||||
AppConfig: appConfig,
|
||||
SystemState: systemState,
|
||||
ConfigLoader: cl,
|
||||
ModelLoader: ml,
|
||||
Gallery: gs,
|
||||
modelAdmin: modeladmin.NewConfigService(cl, appConfig),
|
||||
AppConfig: appConfig,
|
||||
SystemState: systemState,
|
||||
ConfigLoader: cl,
|
||||
ModelLoader: ml,
|
||||
Gallery: gs,
|
||||
VoiceProfiles: voiceprofile.NewStore(appConfig.DataPath),
|
||||
modelAdmin: modeladmin.NewConfigService(cl, appConfig),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -571,6 +576,59 @@ func (c *Client) SetBranding(_ context.Context, req localaitools.SetBrandingRequ
|
||||
return c.currentBranding(), nil
|
||||
}
|
||||
|
||||
// ---- Voice profile library ----
|
||||
|
||||
func (c *Client) voiceProfileStore() (*voiceprofile.Store, error) {
|
||||
if c.VoiceProfiles != nil {
|
||||
return c.VoiceProfiles, nil
|
||||
}
|
||||
if c.AppConfig == nil {
|
||||
return nil, errors.New("voice profile store is unavailable")
|
||||
}
|
||||
c.VoiceProfiles = voiceprofile.NewStore(c.AppConfig.DataPath)
|
||||
return c.VoiceProfiles, nil
|
||||
}
|
||||
|
||||
func (c *Client) ListVoiceProfiles(ctx context.Context) ([]localaitools.VoiceProfile, error) {
|
||||
store, err := c.voiceProfileStore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return store.List(ctx)
|
||||
}
|
||||
|
||||
func (c *Client) CreateVoiceProfile(ctx context.Context, req localaitools.CreateVoiceProfileRequest) (*localaitools.VoiceProfile, error) {
|
||||
if req.AudioBase64 == "" {
|
||||
return nil, errors.New("audio_base64 is required")
|
||||
}
|
||||
if base64.StdEncoding.DecodedLen(len(req.AudioBase64)) > int(voiceprofile.MaxAudioBytes) {
|
||||
return nil, voiceprofile.ErrAudioTooLarge
|
||||
}
|
||||
store, err := c.voiceProfileStore()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
profile, err := store.Create(ctx, voiceprofile.CreateInput{
|
||||
Name: req.Name,
|
||||
Description: req.Description,
|
||||
Language: req.Language,
|
||||
Transcript: req.Transcript,
|
||||
ConsentConfirmed: req.ConsentConfirmed,
|
||||
}, base64.NewDecoder(base64.StdEncoding, strings.NewReader(req.AudioBase64)))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &profile, nil
|
||||
}
|
||||
|
||||
func (c *Client) DeleteVoiceProfile(ctx context.Context, id string) error {
|
||||
store, err := c.voiceProfileStore()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return store.Delete(ctx, id)
|
||||
}
|
||||
|
||||
// ---- helpers ----
|
||||
|
||||
// sendModelOp pushes op onto ch but bails if ctx is cancelled before the
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
These rules are non-negotiable. The user trusts you to operate their server without unintended changes.
|
||||
|
||||
1. **Confirm before mutating.** Before calling any of these tools — `install_model`, `import_model_uri`, `delete_model`, `install_backend`, `upgrade_backend`, `edit_model_config`, `reload_models`, `load_model`, `toggle_model_state`, `toggle_model_pinned` — first state in plain language what you are about to do (which tool, which target, which arguments) and wait for the user's explicit confirmation in the next turn. "Yes", "do it", "go ahead", "proceed" all count as confirmation. Anything else does not.
|
||||
1. **Confirm before mutating.** Before calling any of these tools — `install_model`, `import_model_uri`, `delete_model`, `install_backend`, `upgrade_backend`, `edit_model_config`, `reload_models`, `load_model`, `toggle_model_state`, `toggle_model_pinned`, `create_voice_profile`, `delete_voice_profile` — first state in plain language what you are about to do (which tool, which target, which arguments) and wait for the user's explicit confirmation in the next turn. "Yes", "do it", "go ahead", "proceed" all count as confirmation. Anything else does not.
|
||||
|
||||
2. **Disambiguate before mutating.** If the user's request is ambiguous (several gallery candidates match, the model name has multiple installed versions, the backend has variants), present the candidates as a numbered list and ask the user to pick before calling any mutating tool.
|
||||
|
||||
|
||||
@@ -14,6 +14,7 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the
|
||||
- `vram_estimate` — Estimate VRAM use for a model under a given config.
|
||||
- `system_info` — LocalAI version, paths, distributed flag, loaded models, installed backends.
|
||||
- `list_nodes` — List federated worker nodes (only useful in distributed mode).
|
||||
- `list_voice_profiles` — List reusable voice-cloning profiles and their stable TTS voice URIs.
|
||||
|
||||
## Mutating (require user confirmation per safety rule 1)
|
||||
|
||||
@@ -27,3 +28,5 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the
|
||||
- `load_model` — Pre-load a model into memory so the first request pays no cold-start cost. For a realtime pipeline model, every sub-model (VAD, transcription, LLM, TTS, sound_detection, voice_recognition) is loaded. Inverse of stopping a model.
|
||||
- `toggle_model_state` — Enable or disable a model (`action`: `enable` or `disable`).
|
||||
- `toggle_model_pinned` — Pin or unpin a model (`action`: `pin` or `unpin`).
|
||||
- `create_voice_profile` — Save a consent-confirmed base64 PCM-WAV reference and exact transcript for reuse in TTS.
|
||||
- `delete_voice_profile` — Permanently delete a saved voice profile by UUID.
|
||||
|
||||
@@ -49,6 +49,7 @@ func NewServer(client LocalAIClient, opts Options) *mcp.Server {
|
||||
registerSystemTools(srv, client, opts)
|
||||
registerStateTools(srv, client, opts)
|
||||
registerBrandingTools(srv, client, opts)
|
||||
registerVoiceProfileTools(srv, client, opts)
|
||||
registerUsageTools(srv, client, opts)
|
||||
registerPIITools(srv, client, opts)
|
||||
registerMiddlewareTools(srv, client, opts)
|
||||
|
||||
@@ -92,6 +92,7 @@ var expectedFullCatalog = sortedStrings(
|
||||
ToolListInstalledModels,
|
||||
ToolListKnownBackends,
|
||||
ToolListNodes,
|
||||
ToolListVoiceProfiles,
|
||||
ToolLoadModel,
|
||||
ToolReloadModels,
|
||||
ToolSetAlias,
|
||||
@@ -101,6 +102,8 @@ var expectedFullCatalog = sortedStrings(
|
||||
ToolToggleModelState,
|
||||
ToolUpgradeBackend,
|
||||
ToolVRAMEstimate,
|
||||
ToolCreateVoiceProfile,
|
||||
ToolDeleteVoiceProfile,
|
||||
)
|
||||
|
||||
// expectedReadOnlyCatalog is the tool set when DisableMutating=true. Sorted.
|
||||
@@ -119,6 +122,7 @@ var expectedReadOnlyCatalog = sortedStrings(
|
||||
ToolListInstalledModels,
|
||||
ToolListKnownBackends,
|
||||
ToolListNodes,
|
||||
ToolListVoiceProfiles,
|
||||
ToolSystemInfo,
|
||||
ToolVRAMEstimate,
|
||||
)
|
||||
@@ -160,6 +164,7 @@ var _ = Describe("Tool dispatch", func() {
|
||||
{ToolListKnownBackends, struct{}{}, "ListKnownBackends"},
|
||||
{ToolSystemInfo, struct{}{}, "SystemInfo"},
|
||||
{ToolListNodes, struct{}{}, "ListNodes"},
|
||||
{ToolListVoiceProfiles, struct{}{}, "ListVoiceProfiles"},
|
||||
{ToolInstallModel, InstallModelRequest{ModelName: "test/foo"}, "InstallModel"},
|
||||
{ToolImportModelURI, ImportModelURIRequest{URI: "Qwen/Qwen3-4B-GGUF"}, "ImportModelURI"},
|
||||
{ToolDeleteModel, map[string]any{"name": "foo"}, "DeleteModel"},
|
||||
@@ -172,6 +177,8 @@ var _ = Describe("Tool dispatch", func() {
|
||||
{ToolToggleModelPinned, map[string]any{"name": "foo", "action": "pin"}, "ToggleModelPinned"},
|
||||
{ToolSetAlias, map[string]any{"name": "gpt-4", "target": "real"}, "SetAlias"},
|
||||
{ToolListAliases, struct{}{}, "ListAliases"},
|
||||
{ToolCreateVoiceProfile, CreateVoiceProfileRequest{Name: "Narrator", Transcript: "Reference words", AudioBase64: "UklGRg==", ConsentConfirmed: true}, "CreateVoiceProfile"},
|
||||
{ToolDeleteVoiceProfile, DeleteVoiceProfileRequest{ID: "00000000-0000-0000-0000-000000000001"}, "DeleteVoiceProfile"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
@@ -226,6 +233,7 @@ var _ = Describe("Argument validation", func() {
|
||||
{"delete_model rejects missing name (schema)", ToolDeleteModel, map[string]any{}, "missing properties"},
|
||||
{"toggle_model_state rejects unknown action", ToolToggleModelState, map[string]any{"name": "foo", "action": "noop"}, "action must be one of"},
|
||||
{"edit_model_config rejects empty patch", ToolEditModelConfig, map[string]any{"name": "foo", "patch": map[string]any{}}, "patch is required"},
|
||||
{"create_voice_profile requires consent", ToolCreateVoiceProfile, CreateVoiceProfileRequest{Name: "Voice", Transcript: "words", AudioBase64: "UklGRg=="}, "consent_confirmed must be true"},
|
||||
}
|
||||
|
||||
for _, c := range cases {
|
||||
|
||||
@@ -23,21 +23,24 @@ const (
|
||||
ToolGetPIIEvents = "get_pii_events"
|
||||
ToolGetMiddlewareStatus = "get_middleware_status"
|
||||
ToolGetRouterDecisions = "get_router_decisions"
|
||||
ToolListVoiceProfiles = "list_voice_profiles"
|
||||
|
||||
// Mutating tools — guarded by Options.DisableMutating and the
|
||||
// LLM-side safety prompt (see prompts/10_safety.md).
|
||||
ToolInstallModel = "install_model"
|
||||
ToolImportModelURI = "import_model_uri"
|
||||
ToolDeleteModel = "delete_model"
|
||||
ToolEditModelConfig = "edit_model_config"
|
||||
ToolReloadModels = "reload_models"
|
||||
ToolLoadModel = "load_model"
|
||||
ToolInstallBackend = "install_backend"
|
||||
ToolUpgradeBackend = "upgrade_backend"
|
||||
ToolToggleModelState = "toggle_model_state"
|
||||
ToolToggleModelPinned = "toggle_model_pinned"
|
||||
ToolSetBranding = "set_branding"
|
||||
ToolSetAlias = "set_alias"
|
||||
ToolInstallModel = "install_model"
|
||||
ToolImportModelURI = "import_model_uri"
|
||||
ToolDeleteModel = "delete_model"
|
||||
ToolEditModelConfig = "edit_model_config"
|
||||
ToolReloadModels = "reload_models"
|
||||
ToolLoadModel = "load_model"
|
||||
ToolInstallBackend = "install_backend"
|
||||
ToolUpgradeBackend = "upgrade_backend"
|
||||
ToolToggleModelState = "toggle_model_state"
|
||||
ToolToggleModelPinned = "toggle_model_pinned"
|
||||
ToolSetBranding = "set_branding"
|
||||
ToolSetAlias = "set_alias"
|
||||
ToolCreateVoiceProfile = "create_voice_profile"
|
||||
ToolDeleteVoiceProfile = "delete_voice_profile"
|
||||
|
||||
// ToolListAliases is read-only but lives here so the alias tools stay
|
||||
// grouped; the catalog tests assert its read-only placement.
|
||||
|
||||
54
pkg/mcp/localaitools/tools_voice_profiles.go
Normal file
54
pkg/mcp/localaitools/tools_voice_profiles.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package localaitools
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/modelcontextprotocol/go-sdk/mcp"
|
||||
)
|
||||
|
||||
func registerVoiceProfileTools(s *mcp.Server, client LocalAIClient, opts Options) {
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: ToolListVoiceProfiles,
|
||||
Description: "List reusable voice-cloning profiles. Returns stable voice URI values suitable for TTSRequest.voice; filesystem paths are never exposed.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
|
||||
profiles, err := client.ListVoiceProfiles(ctx)
|
||||
if err != nil {
|
||||
return errorResult(err), nil, nil
|
||||
}
|
||||
return jsonResult(profiles), nil, nil
|
||||
})
|
||||
|
||||
if opts.DisableMutating {
|
||||
return
|
||||
}
|
||||
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: ToolCreateVoiceProfile,
|
||||
Description: "Create a reusable voice-cloning profile from a base64 16-bit PCM WAV (mono 24 kHz recommended) and its exact transcript. consent_confirmed must be true. Requires user confirmation per safety rule 1.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, args CreateVoiceProfileRequest) (*mcp.CallToolResult, any, error) {
|
||||
if args.Name == "" || args.Transcript == "" || args.AudioBase64 == "" {
|
||||
return errorResultf("name, transcript, and audio_base64 are required"), nil, nil
|
||||
}
|
||||
if !args.ConsentConfirmed {
|
||||
return errorResultf("consent_confirmed must be true"), nil, nil
|
||||
}
|
||||
profile, err := client.CreateVoiceProfile(ctx, args)
|
||||
if err != nil {
|
||||
return errorResult(err), nil, nil
|
||||
}
|
||||
return jsonResult(profile), nil, nil
|
||||
})
|
||||
|
||||
mcp.AddTool(s, &mcp.Tool{
|
||||
Name: ToolDeleteVoiceProfile,
|
||||
Description: "Permanently delete a reusable voice-cloning profile by opaque UUID. Requires user confirmation per safety rule 1.",
|
||||
}, func(ctx context.Context, _ *mcp.CallToolRequest, args DeleteVoiceProfileRequest) (*mcp.CallToolResult, any, error) {
|
||||
if args.ID == "" {
|
||||
return errorResultf("id is required"), nil, nil
|
||||
}
|
||||
if err := client.DeleteVoiceProfile(ctx, args.ID); err != nil {
|
||||
return errorResult(err), nil, nil
|
||||
}
|
||||
return jsonResult(map[string]any{"deleted": true, "id": args.ID}), nil, nil
|
||||
})
|
||||
}
|
||||
262
swagger/docs.go
262
swagger/docs.go
@@ -1319,6 +1319,176 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles": {
|
||||
"get": {
|
||||
"description": "List saved voice-cloning references without exposing filesystem paths.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "List voice profiles",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/localai.VoiceProfileListResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"description": "Save a consent-confirmed PCM WAV reference clip and exact transcript for voice cloning. Admin-only.",
|
||||
"consumes": [
|
||||
"multipart/form-data",
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Create a voice profile",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Display name",
|
||||
"name": "name",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Optional description",
|
||||
"name": "description",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Optional language tag",
|
||||
"name": "language",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Exact transcript of the reference clip",
|
||||
"name": "transcript",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "boolean",
|
||||
"description": "Confirms authorization to clone the voice",
|
||||
"name": "consent_confirmed",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"description": "16-bit PCM WAV, preferably mono 24 kHz, 1-120 seconds, up to 50 MiB",
|
||||
"name": "audio",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Created",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/voiceprofile.Profile"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "Request Entity Too Large",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles/{id}": {
|
||||
"delete": {
|
||||
"description": "Permanently remove a saved voice-cloning profile. Admin-only.",
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Delete a voice profile",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Voice profile UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": "No Content"
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles/{id}/audio": {
|
||||
"get": {
|
||||
"description": "Stream the saved reference WAV for an authenticated TTS user.",
|
||||
"produces": [
|
||||
"audio/x-wav"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Preview voice profile audio",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Voice profile UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/audio/transform": {
|
||||
"post": {
|
||||
"description": "Runs an audio-in / audio-out transform conditioned on an optional auxiliary reference signal. Concrete transforms include AEC + noise suppression + dereverberation (LocalVQE), voice conversion (reference = target speaker), and pitch shifting. The backend determines the operation; pass model-specific tuning via repeated ` + "`" + `params[\u003ckey\u003e]=\u003cvalue\u003e` + "`" + ` form fields.",
|
||||
@@ -3781,6 +3951,17 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"localai.VoiceProfileListResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/voiceprofile.Profile"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.BackendLogLine": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -3949,6 +4130,21 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.APIError": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"param": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.AnthropicContentBlock": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -4489,6 +4685,14 @@ const docTemplate = `{
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.ErrorResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {
|
||||
"$ref": "#/definitions/schema.APIError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.FaceAnalysis": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -6930,6 +7134,64 @@ const docTemplate = `{
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"voiceprofile.AudioMetadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bit_depth": {
|
||||
"type": "integer"
|
||||
},
|
||||
"channels": {
|
||||
"type": "integer"
|
||||
},
|
||||
"duration_ms": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"sample_rate": {
|
||||
"type": "integer"
|
||||
},
|
||||
"size_bytes": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"voiceprofile.Profile": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audio": {
|
||||
"$ref": "#/definitions/voiceprofile.AudioMetadata"
|
||||
},
|
||||
"consent_confirmed_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"transcript": {
|
||||
"type": "string"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"voice": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"securityDefinitions": {
|
||||
|
||||
@@ -1316,6 +1316,176 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles": {
|
||||
"get": {
|
||||
"description": "List saved voice-cloning references without exposing filesystem paths.",
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "List voice profiles",
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/localai.VoiceProfileListResponse"
|
||||
}
|
||||
},
|
||||
"500": {
|
||||
"description": "Internal Server Error",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"post": {
|
||||
"description": "Save a consent-confirmed PCM WAV reference clip and exact transcript for voice cloning. Admin-only.",
|
||||
"consumes": [
|
||||
"multipart/form-data",
|
||||
"application/json"
|
||||
],
|
||||
"produces": [
|
||||
"application/json"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Create a voice profile",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Display name",
|
||||
"name": "name",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Optional description",
|
||||
"name": "description",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Optional language tag",
|
||||
"name": "language",
|
||||
"in": "formData"
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Exact transcript of the reference clip",
|
||||
"name": "transcript",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "boolean",
|
||||
"description": "Confirms authorization to clone the voice",
|
||||
"name": "consent_confirmed",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
},
|
||||
{
|
||||
"type": "file",
|
||||
"description": "16-bit PCM WAV, preferably mono 24 kHz, 1-120 seconds, up to 50 MiB",
|
||||
"name": "audio",
|
||||
"in": "formData",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"201": {
|
||||
"description": "Created",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/voiceprofile.Profile"
|
||||
}
|
||||
},
|
||||
"400": {
|
||||
"description": "Bad Request",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
},
|
||||
"413": {
|
||||
"description": "Request Entity Too Large",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles/{id}": {
|
||||
"delete": {
|
||||
"description": "Permanently remove a saved voice-cloning profile. Admin-only.",
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Delete a voice profile",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Voice profile UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"204": {
|
||||
"description": "No Content"
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/api/voice-profiles/{id}/audio": {
|
||||
"get": {
|
||||
"description": "Stream the saved reference WAV for an authenticated TTS user.",
|
||||
"produces": [
|
||||
"audio/x-wav"
|
||||
],
|
||||
"tags": [
|
||||
"audio",
|
||||
"voice-profiles"
|
||||
],
|
||||
"summary": "Preview voice profile audio",
|
||||
"parameters": [
|
||||
{
|
||||
"type": "string",
|
||||
"description": "Voice profile UUID",
|
||||
"name": "id",
|
||||
"in": "path",
|
||||
"required": true
|
||||
}
|
||||
],
|
||||
"responses": {
|
||||
"200": {
|
||||
"description": "OK",
|
||||
"schema": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"404": {
|
||||
"description": "Not Found",
|
||||
"schema": {
|
||||
"$ref": "#/definitions/schema.ErrorResponse"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"/audio/transform": {
|
||||
"post": {
|
||||
"description": "Runs an audio-in / audio-out transform conditioned on an optional auxiliary reference signal. Concrete transforms include AEC + noise suppression + dereverberation (LocalVQE), voice conversion (reference = target speaker), and pitch shifting. The backend determines the operation; pass model-specific tuning via repeated `params[\u003ckey\u003e]=\u003cvalue\u003e` form fields.",
|
||||
@@ -3778,6 +3948,17 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"localai.VoiceProfileListResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"data": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/definitions/voiceprofile.Profile"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"model.BackendLogLine": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -3946,6 +4127,21 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.APIError": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"code": {},
|
||||
"message": {
|
||||
"type": "string"
|
||||
},
|
||||
"param": {
|
||||
"type": "string"
|
||||
},
|
||||
"type": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.AnthropicContentBlock": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -4486,6 +4682,14 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.ErrorResponse": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"error": {
|
||||
"$ref": "#/definitions/schema.APIError"
|
||||
}
|
||||
}
|
||||
},
|
||||
"schema.FaceAnalysis": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -6927,6 +7131,64 @@
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
},
|
||||
"voiceprofile.AudioMetadata": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"bit_depth": {
|
||||
"type": "integer"
|
||||
},
|
||||
"channels": {
|
||||
"type": "integer"
|
||||
},
|
||||
"duration_ms": {
|
||||
"type": "integer"
|
||||
},
|
||||
"mime_type": {
|
||||
"type": "string"
|
||||
},
|
||||
"sample_rate": {
|
||||
"type": "integer"
|
||||
},
|
||||
"size_bytes": {
|
||||
"type": "integer"
|
||||
}
|
||||
}
|
||||
},
|
||||
"voiceprofile.Profile": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"audio": {
|
||||
"$ref": "#/definitions/voiceprofile.AudioMetadata"
|
||||
},
|
||||
"consent_confirmed_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"created_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"description": {
|
||||
"type": "string"
|
||||
},
|
||||
"id": {
|
||||
"type": "string"
|
||||
},
|
||||
"language": {
|
||||
"type": "string"
|
||||
},
|
||||
"name": {
|
||||
"type": "string"
|
||||
},
|
||||
"transcript": {
|
||||
"type": "string"
|
||||
},
|
||||
"updated_at": {
|
||||
"type": "string"
|
||||
},
|
||||
"voice": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
"securityDefinitions": {
|
||||
|
||||
@@ -390,6 +390,13 @@ definitions:
|
||||
>= 1.
|
||||
type: integer
|
||||
type: object
|
||||
localai.VoiceProfileListResponse:
|
||||
properties:
|
||||
data:
|
||||
items:
|
||||
$ref: '#/definitions/voiceprofile.Profile'
|
||||
type: array
|
||||
type: object
|
||||
model.BackendLogLine:
|
||||
properties:
|
||||
stream:
|
||||
@@ -504,6 +511,16 @@ definitions:
|
||||
start:
|
||||
type: number
|
||||
type: object
|
||||
schema.APIError:
|
||||
properties:
|
||||
code: {}
|
||||
message:
|
||||
type: string
|
||||
param:
|
||||
type: string
|
||||
type:
|
||||
type: string
|
||||
type: object
|
||||
schema.AnthropicContentBlock:
|
||||
properties:
|
||||
content: {}
|
||||
@@ -872,6 +889,11 @@ definitions:
|
||||
vocal_language:
|
||||
type: string
|
||||
type: object
|
||||
schema.ErrorResponse:
|
||||
properties:
|
||||
error:
|
||||
$ref: '#/definitions/schema.APIError'
|
||||
type: object
|
||||
schema.FaceAnalysis:
|
||||
properties:
|
||||
age:
|
||||
@@ -2602,6 +2624,44 @@ definitions:
|
||||
description: Webhook endpoint URL
|
||||
type: string
|
||||
type: object
|
||||
voiceprofile.AudioMetadata:
|
||||
properties:
|
||||
bit_depth:
|
||||
type: integer
|
||||
channels:
|
||||
type: integer
|
||||
duration_ms:
|
||||
type: integer
|
||||
mime_type:
|
||||
type: string
|
||||
sample_rate:
|
||||
type: integer
|
||||
size_bytes:
|
||||
type: integer
|
||||
type: object
|
||||
voiceprofile.Profile:
|
||||
properties:
|
||||
audio:
|
||||
$ref: '#/definitions/voiceprofile.AudioMetadata'
|
||||
consent_confirmed_at:
|
||||
type: string
|
||||
created_at:
|
||||
type: string
|
||||
description:
|
||||
type: string
|
||||
id:
|
||||
type: string
|
||||
language:
|
||||
type: string
|
||||
name:
|
||||
type: string
|
||||
transcript:
|
||||
type: string
|
||||
updated_at:
|
||||
type: string
|
||||
voice:
|
||||
type: string
|
||||
type: object
|
||||
info:
|
||||
contact:
|
||||
name: LocalAI
|
||||
@@ -3493,6 +3553,124 @@ paths:
|
||||
summary: Clear API traces
|
||||
tags:
|
||||
- monitoring
|
||||
/api/voice-profiles:
|
||||
get:
|
||||
description: List saved voice-cloning references without exposing filesystem
|
||||
paths.
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
$ref: '#/definitions/localai.VoiceProfileListResponse'
|
||||
"500":
|
||||
description: Internal Server Error
|
||||
schema:
|
||||
$ref: '#/definitions/schema.ErrorResponse'
|
||||
summary: List voice profiles
|
||||
tags:
|
||||
- audio
|
||||
- voice-profiles
|
||||
post:
|
||||
consumes:
|
||||
- multipart/form-data
|
||||
- application/json
|
||||
description: Save a consent-confirmed PCM WAV reference clip and exact transcript
|
||||
for voice cloning. Admin-only.
|
||||
parameters:
|
||||
- description: Display name
|
||||
in: formData
|
||||
name: name
|
||||
required: true
|
||||
type: string
|
||||
- description: Optional description
|
||||
in: formData
|
||||
name: description
|
||||
type: string
|
||||
- description: Optional language tag
|
||||
in: formData
|
||||
name: language
|
||||
type: string
|
||||
- description: Exact transcript of the reference clip
|
||||
in: formData
|
||||
name: transcript
|
||||
required: true
|
||||
type: string
|
||||
- description: Confirms authorization to clone the voice
|
||||
in: formData
|
||||
name: consent_confirmed
|
||||
required: true
|
||||
type: boolean
|
||||
- description: 16-bit PCM WAV, preferably mono 24 kHz, 1-120 seconds, up to
|
||||
50 MiB
|
||||
in: formData
|
||||
name: audio
|
||||
required: true
|
||||
type: file
|
||||
produces:
|
||||
- application/json
|
||||
responses:
|
||||
"201":
|
||||
description: Created
|
||||
schema:
|
||||
$ref: '#/definitions/voiceprofile.Profile'
|
||||
"400":
|
||||
description: Bad Request
|
||||
schema:
|
||||
$ref: '#/definitions/schema.ErrorResponse'
|
||||
"413":
|
||||
description: Request Entity Too Large
|
||||
schema:
|
||||
$ref: '#/definitions/schema.ErrorResponse'
|
||||
summary: Create a voice profile
|
||||
tags:
|
||||
- audio
|
||||
- voice-profiles
|
||||
/api/voice-profiles/{id}:
|
||||
delete:
|
||||
description: Permanently remove a saved voice-cloning profile. Admin-only.
|
||||
parameters:
|
||||
- description: Voice profile UUID
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
responses:
|
||||
"204":
|
||||
description: No Content
|
||||
"404":
|
||||
description: Not Found
|
||||
schema:
|
||||
$ref: '#/definitions/schema.ErrorResponse'
|
||||
summary: Delete a voice profile
|
||||
tags:
|
||||
- audio
|
||||
- voice-profiles
|
||||
/api/voice-profiles/{id}/audio:
|
||||
get:
|
||||
description: Stream the saved reference WAV for an authenticated TTS user.
|
||||
parameters:
|
||||
- description: Voice profile UUID
|
||||
in: path
|
||||
name: id
|
||||
required: true
|
||||
type: string
|
||||
produces:
|
||||
- audio/x-wav
|
||||
responses:
|
||||
"200":
|
||||
description: OK
|
||||
schema:
|
||||
type: string
|
||||
"404":
|
||||
description: Not Found
|
||||
schema:
|
||||
$ref: '#/definitions/schema.ErrorResponse'
|
||||
summary: Preview voice profile audio
|
||||
tags:
|
||||
- audio
|
||||
- voice-profiles
|
||||
/audio/transform:
|
||||
post:
|
||||
consumes:
|
||||
|
||||
Reference in New Issue
Block a user