feat(audio): list available TTS voices

Clients cannot discover the named voices that an installed TTS model accepts without consulting backend-specific documentation. Expose voice metadata through the audio API and let custom model configs declare their own catalog.

Assisted-by: Codex:gpt-5
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto committed 2026-09-11 21:55:21 +00:00
1 parent 21e81434b3
commit d55474a149
14 files changed
+486 -2

No files matched your search

+47 -1
View File
@@ -244,6 +244,8 @@ type BackendCapability struct {
// contract. Model variants that share a backend may narrow this further;
// use VoiceCloningForModel for UI/API decisions.
VoiceCloning *VoiceCloningCapability
// TTSVoices lists named voices built into the backend.
TTSVoices []TTSVoice
// Description is a human-readable summary of the backend.
Description string
}
@@ -263,6 +265,22 @@ func referenceVoiceCloning() *VoiceCloningCapability {
}
}
// TTSVoicesForModel returns model-specific metadata or the backend's built-in
// catalog. The returned slice is safe for callers to modify.
func TTSVoicesForModel(cfg *ModelConfig) []TTSVoice {
if cfg == nil {
return nil
}
if len(cfg.TTSConfig.Voices) > 0 {
return slices.Clone(cfg.TTSConfig.Voices)
}
capability := GetBackendCapability(cfg.Backend)
if capability == nil {
return nil
}
return slices.Clone(capability.TTSVoices)
}
// BackendCapabilities maps each backend name (as used in model configs and gallery
// entries) to its verified capabilities. This is the single source of truth for
// what each backend supports.
@@ -587,7 +605,35 @@ var BackendCapabilities = map[string]BackendCapability{
PossibleUsecases: []string{UsecaseTTS},
DefaultUsecases: []string{UsecaseTTS},
VoiceCloning: referenceVoiceCloning(),
Description: "Pocket TTS — lightweight text-to-speech",
TTSVoices: []TTSVoice{
{Name: "juergen", Language: "de_DE", Gender: "male"},
{Name: "alba", Language: "en_US", Gender: "female"},
{Name: "bill_boerst", Language: "en_US", Gender: "male"},
{Name: "charles", Language: "en_US", Gender: "male"},
{Name: "george", Language: "en_US", Gender: "male"},
{Name: "javert", Language: "en_US", Gender: "male"},
{Name: "jean", Language: "en_US", Gender: "male"},
{Name: "marius", Language: "en_US", Gender: "male"},
{Name: "michael", Language: "en_US", Gender: "male"},
{Name: "paul", Language: "en_US", Gender: "male"},
{Name: "peter_yearsley", Language: "en_US", Gender: "male"},
{Name: "stuart_bell", Language: "en_US", Gender: "male"},
{Name: "anna", Language: "en_US", Gender: "female"},
{Name: "azelma", Language: "en_US", Gender: "female"},
{Name: "caro_davy", Language: "en_US", Gender: "female"},
{Name: "cosette", Language: "en_US", Gender: "female"},
{Name: "eponine", Language: "en_US", Gender: "female"},
{Name: "eve", Language: "en_US", Gender: "female"},
{Name: "fantine", Language: "en_US", Gender: "female"},
{Name: "jane", Language: "en_US", Gender: "female"},
{Name: "mary", Language: "en_US", Gender: "female"},
{Name: "vera", Language: "en_US", Gender: "female"},
{Name: "lola", Language: "es_ES", Gender: "female"},
{Name: "estelle", Language: "fr_FR", Gender: "female"},
{Name: "giovanni", Language: "it_IT", Gender: "male"},
{Name: "rafael", Language: "pt_PT", Gender: "male"},
},
Description: "Pocket TTS — lightweight text-to-speech",
},
"qwen-tts": {
GRPCMethods: []GRPCMethod{MethodTTS},
+22
View File
@@ -297,6 +297,28 @@ var _ = Describe("VoiceCloningForModel", func() {
)
})
var _ = Describe("TTSVoicesForModel", func() {
It("returns the built-in Pocket TTS voice catalog", func() {
voices := TTSVoicesForModel(&ModelConfig{Name: "pocket", Backend: "pocket-tts"})
Expect(voices).To(ContainElement(TTSVoice{Name: "alba", Language: "en_US", Gender: "female"}))
Expect(voices).To(ContainElement(TTSVoice{Name: "giovanni", Language: "it_IT", Gender: "male"}))
})
It("resolves the catalog for pinned backend variants", func() {
voices := TTSVoicesForModel(&ModelConfig{Name: "pocket", Backend: "cuda12-pocket-tts"})
Expect(voices).To(ContainElement(TTSVoice{Name: "alba", Language: "en_US", Gender: "female"}))
})
It("prefers model-specific voice metadata", func() {
configured := []TTSVoice{{Name: "custom", Language: "en_GB"}}
voices := TTSVoicesForModel(&ModelConfig{
Backend: "pocket-tts",
TTSConfig: TTSConfig{Voices: configured},
})
Expect(voices).To(Equal(configured))
})
})
// llama.cpp serves Qwen3-TTS as well as the text LLMs it is known for, so the
// backend has to advertise TTS. That advertisement is what makes narrowing
// mandatory: the per-backend switch in VoiceCloningForModel ends in a
+7
View File
@@ -893,6 +893,13 @@ func DefaultRegistry() map[string]FieldMetaOverride {
Component: "input",
Order: 91,
},
"tts.voices": {
Section: "tts",
Label: "Named Voices",
Description: "Named voices that this model accepts. Each entry requires a name and can include language and gender metadata.",
Component: "json-editor",
Order: 92,
},
// --- Diffusers ---
"diffusers.pipeline_type": {
+11
View File
@@ -39,6 +39,17 @@ type TTSConfig struct {
// A pointer preserves the distinction between an explicit false and the
// default automatic behavior.
VoiceCloning *bool `yaml:"voice_cloning,omitempty" json:"voice_cloning,omitempty"`
// Voices describes named voices accepted by this model. Backends with a
// built-in catalog supply defaults when this list is empty.
Voices []TTSVoice `yaml:"voices,omitempty" json:"voices,omitempty"`
}
// TTSVoice describes one named voice accepted by a text-to-speech model.
type TTSVoice struct {
Name string `yaml:"name" json:"name"`
Language string `yaml:"language,omitempty" json:"language,omitempty"`
Gender string `yaml:"gender,omitempty" json:"gender,omitempty"`
}
// @Description ModelConfig represents a model configuration
+2
View File
@@ -58,6 +58,8 @@ var RouteFeatureRegistry = []RouteFeature{
{"POST", "/v1/audio/speech", FeatureAudioSpeech},
{"POST", "/audio/speech", FeatureAudioSpeech},
{"POST", "/tts", FeatureAudioSpeech},
{"GET", "/v1/audio/voices", FeatureAudioSpeech},
{"GET", "/audio/voices", FeatureAudioSpeech},
{"POST", "/v1/text-to-speech/:voice-id", FeatureAudioSpeech},
{"GET", "/api/voice-profiles", FeatureAudioSpeech},
{"GET", "/api/voice-profiles/:id/audio", FeatureAudioSpeech},
@@ -40,7 +40,7 @@ var instructionDefs = []instructionDef{
Name: "audio",
Description: "Text-to-speech, voice activity detection, transcription, speaker diarization, sound classification, and sound generation",
Tags: []string{"audio"},
Intro: "Diarization (/v1/audio/diarization) returns speaker-labelled time segments. Backends with native ASR-diarization (vibevoice-cpp) can also emit per-segment text via include_text=true; backends with a dedicated pipeline (sherpa-onnx + pyannote) emit segmentation only. Response formats: json (default), verbose_json (adds speakers summary + text), rttm (NIST format). Sound classification (/v1/audio/classification) returns scored AudioSet sound-event tags (audio tagging via the ced backend); top_k and threshold control the returned set.",
Intro: "GET /v1/audio/voices lists named voices for installed TTS models and accepts an optional model filter. Diarization (/v1/audio/diarization) returns speaker-labelled time segments. Backends with native ASR-diarization (vibevoice-cpp) can also emit per-segment text via include_text=true; backends with a dedicated pipeline (sherpa-onnx + pyannote) emit segmentation only. Response formats: json (default), verbose_json (adds speakers summary + text), rttm (NIST format). Sound classification (/v1/audio/classification) returns scored AudioSet sound-event tags (audio tagging via the ced backend); top_k and threshold control the returned set.",
},
{
Name: "voice-library",
+99
View File
@@ -0,0 +1,99 @@
package localai
import (
"net/http"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/http/auth"
"github.com/mudler/LocalAI/core/schema"
"gorm.io/gorm"
)
// TTSModelVoices groups named voices by installed model.
type TTSModelVoices struct {
Model string `json:"model"`
Voices []config.TTSVoice `json:"voices"`
}
// TTSVoicesResponse is returned by the TTS voice discovery endpoint.
type TTSVoicesResponse struct {
Data []TTSModelVoices `json:"data"`
}
// TTSVoicesEndpoint lists named voices advertised by installed model configs.
//
// @Summary List text-to-speech voices
// @Description List named voices and their language and gender metadata. Use the optional model query parameter to filter the response.
// @Tags audio
// @Produce json
// @Param model query string false "Installed model name"
// @Success 200 {object} TTSVoicesResponse
// @Failure 404 {object} schema.ErrorResponse
// @Router /v1/audio/voices [get]
func TTSVoicesEndpoint(loader *config.ModelConfigLoader, databases ...*gorm.DB) echo.HandlerFunc {
var authDB *gorm.DB
if len(databases) > 0 {
authDB = databases[0]
}
return func(c echo.Context) error {
allowed, err := ttsVoiceModelAllowlist(c, authDB)
if err != nil {
return c.JSON(http.StatusInternalServerError, schema.ErrorResponse{Error: &schema.APIError{
Code: http.StatusInternalServerError, Message: "failed to check permissions", Type: "server_error",
}})
}
modelName := c.QueryParam("model")
if modelName != "" {
cfg, ok := loader.GetModelConfig(modelName)
if !ok || (allowed != nil && !allowed[modelName]) {
return c.JSON(http.StatusNotFound, schema.ErrorResponse{Error: &schema.APIError{
Code: http.StatusNotFound, Message: "model not found", Type: "not_found",
}})
}
return c.JSON(http.StatusOK, TTSVoicesResponse{Data: []TTSModelVoices{{
Model: cfg.Name, Voices: ttsVoicesForConfig(loader, &cfg),
}}})
}
response := TTSVoicesResponse{Data: []TTSModelVoices{}}
for _, cfg := range loader.GetAllModelsConfigs() {
if allowed != nil && !allowed[cfg.Name] {
continue
}
voices := ttsVoicesForConfig(loader, &cfg)
if len(voices) == 0 {
continue
}
response.Data = append(response.Data, TTSModelVoices{Model: cfg.Name, Voices: voices})
}
return c.JSON(http.StatusOK, response)
}
}
func ttsVoicesForConfig(loader *config.ModelConfigLoader, cfg *config.ModelConfig) []config.TTSVoice {
resolved, isAlias, err := loader.ResolveAlias(cfg)
if err == nil && isAlias {
return config.TTSVoicesForModel(resolved)
}
return config.TTSVoicesForModel(cfg)
}
func ttsVoiceModelAllowlist(c echo.Context, db *gorm.DB) (map[string]bool, error) {
if db == nil {
return nil, nil
}
user := auth.GetUser(c)
if user == nil || user.Role == auth.RoleAdmin {
return nil, nil
}
permissions, err := auth.GetCachedUserPermissions(c, db, user.ID)
if err != nil || !permissions.AllowedModels.Enabled {
return nil, err
}
allowed := make(map[string]bool, len(permissions.AllowedModels.Models))
for _, model := range permissions.AllowedModels.Models {
allowed[model] = true
}
return allowed, nil
}
@@ -0,0 +1,66 @@
package localai_test
import (
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/config"
. "github.com/mudler/LocalAI/core/http/endpoints/localai"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("TTSVoicesEndpoint", func() {
var loader *config.ModelConfigLoader
BeforeEach(func() {
dir, err := os.MkdirTemp("", "localai-tts-voices-test")
Expect(err).NotTo(HaveOccurred())
DeferCleanup(os.RemoveAll, dir)
Expect(os.WriteFile(filepath.Join(dir, "pocket.yaml"), []byte("name: pocket\nbackend: pocket-tts\n"), 0o600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(dir, "custom.yaml"), []byte("name: custom\nbackend: custom\nknown_usecases: [tts]\ntts:\n voices:\n - name: narrator\n language: en_GB\n"), 0o600)).To(Succeed())
Expect(os.WriteFile(filepath.Join(dir, "pocket-alias.yaml"), []byte("name: pocket-alias\nalias: pocket\n"), 0o600)).To(Succeed())
loader = config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir)).To(Succeed())
})
It("returns the target catalog under an alias name", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=pocket-alias", nil))
Expect(rec.Code).To(Equal(http.StatusOK))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"pocket-alias"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"alba"`))
})
It("lists voice metadata for installed TTS models", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))
rec := httptest.NewRecorder()
e.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/v1/audio/voices", nil))
Expect(rec.Code).To(Equal(http.StatusOK))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"custom"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"narrator"`))
Expect(rec.Body.String()).To(ContainSubstring(`"model":"pocket"`))
Expect(rec.Body.String()).To(ContainSubstring(`"name":"alba"`))
})
It("filters by model and rejects an unknown model", func() {
e := echo.New()
e.GET("/v1/audio/voices", TTSVoicesEndpoint(loader))
found := httptest.NewRecorder()
e.ServeHTTP(found, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=pocket", nil))
Expect(found.Code).To(Equal(http.StatusOK))
Expect(found.Body.String()).To(ContainSubstring(`"model":"pocket"`))
Expect(found.Body.String()).NotTo(ContainSubstring(`"model":"custom"`))
missing := httptest.NewRecorder()
e.ServeHTTP(missing, httptest.NewRequest(http.MethodGet, "/v1/audio/voices?model=missing", nil))
Expect(missing.Code).To(Equal(http.StatusNotFound))
})
})
+4
View File
@@ -328,6 +328,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"vram_estimate": "/api/models/vram-estimate",
"model_load_status": "/api/models/:id/load-status",
"tts": "/tts",
"tts_voices": "/v1/audio/voices",
"voice_profiles": "/api/voice-profiles",
"transcription": "/v1/audio/transcriptions",
"image_generation": "/v1/images/generations",
@@ -345,6 +346,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"transcription": "/v1/audio/transcriptions",
"diarization": "/v1/audio/diarization",
"sound_classification": "/v1/audio/classification",
"tts_voices": "/v1/audio/voices",
"image_generation": "/v1/images/generations",
},
"config_management": map[string]string{
@@ -366,6 +368,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
},
"ai_functions": map[string]string{
"tts": "/tts",
"tts_voices": "/v1/audio/voices",
"voice_profiles": "/api/voice-profiles",
"vad": "/vad",
"video": "/video",
@@ -414,6 +417,7 @@ func RegisterLocalAIRoutes(router *echo.Echo,
"p2p": appConfig.P2PToken != "",
"tracing": true,
"voice_profiles": true,
"tts_voices": true,
},
})
})
+2
View File
@@ -238,6 +238,8 @@ func RegisterOpenAIRoutes(app *echo.Echo,
app.POST("/v1/audio/speech", audioSpeechHandler, audioSpeechMiddleware...)
app.POST("/audio/speech", audioSpeechHandler, audioSpeechMiddleware...)
app.GET("/v1/audio/voices", localai.TTSVoicesEndpoint(application.ModelConfigLoader(), application.AuthDB()))
app.GET("/audio/voices", localai.TTSVoicesEndpoint(application.ModelConfigLoader(), application.AuthDB()))
// images
imageHandler := openai.ImageEndpoint(application.ModelConfigLoader(), application.ModelLoader(), application.ApplicationConfig())
+31
View File
@@ -30,6 +30,37 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
Returns an `audio/wav` file.
## List available voices
Use `GET /v1/audio/voices` to list named voices for installed TTS models:
```bash
curl http://localhost:8080/v1/audio/voices
```
Add the `model` query parameter to return one installed model:
```bash
curl 'http://localhost:8080/v1/audio/voices?model=pocket-tts'
```
Each voice can include `language` and `gender` metadata. LocalAI supplies the
built-in Pocket TTS catalog. Other models can declare their catalog in YAML:
```yaml
name: custom-tts
backend: custom
known_usecases: [tts]
tts:
voices:
- name: narrator
language: en_GB
gender: female
```
LocalAI returns `404` when the requested model is not installed. Models without
voice metadata do not appear in the unfiltered response.
## Voice Library
Administrators can manage reusable voice-cloning references from **Operate → Voice Library** in the LocalAI WebUI. The library replaces per-model filesystem and YAML setup for supported cloning backends:
+73
View File
@@ -2908,6 +2908,40 @@ const docTemplate = `{
}
}
},
"/v1/audio/voices": {
"get": {
"description": "List named voices and their language and gender metadata. Use the optional model query parameter to filter the response.",
"produces": [
"application/json"
],
"tags": [
"audio"
],
"summary": "List text-to-speech voices",
"parameters": [
{
"type": "string",
"description": "Installed model name",
"name": "model",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/localai.TTSVoicesResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/schema.ErrorResponse"
}
}
}
}
},
"/v1/chat/completions": {
"post": {
"tags": [
@@ -4071,6 +4105,20 @@ const docTemplate = `{
}
}
},
"config.TTSVoice": {
"type": "object",
"properties": {
"gender": {
"type": "string"
},
"language": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"functions.Function": {
"type": "object",
"properties": {
@@ -4606,6 +4654,31 @@ const docTemplate = `{
}
}
},
"localai.TTSModelVoices": {
"type": "object",
"properties": {
"model": {
"type": "string"
},
"voices": {
"type": "array",
"items": {
"$ref": "#/definitions/config.TTSVoice"
}
}
}
},
"localai.TTSVoicesResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/localai.TTSModelVoices"
}
}
}
},
"localai.UpdateMaxReplicasPerModelRequest": {
"type": "object",
"properties": {
+73
View File
@@ -2905,6 +2905,40 @@
}
}
},
"/v1/audio/voices": {
"get": {
"description": "List named voices and their language and gender metadata. Use the optional model query parameter to filter the response.",
"produces": [
"application/json"
],
"tags": [
"audio"
],
"summary": "List text-to-speech voices",
"parameters": [
{
"type": "string",
"description": "Installed model name",
"name": "model",
"in": "query"
}
],
"responses": {
"200": {
"description": "OK",
"schema": {
"$ref": "#/definitions/localai.TTSVoicesResponse"
}
},
"404": {
"description": "Not Found",
"schema": {
"$ref": "#/definitions/schema.ErrorResponse"
}
}
}
}
},
"/v1/chat/completions": {
"post": {
"tags": [
@@ -4068,6 +4102,20 @@
}
}
},
"config.TTSVoice": {
"type": "object",
"properties": {
"gender": {
"type": "string"
},
"language": {
"type": "string"
},
"name": {
"type": "string"
}
}
},
"functions.Function": {
"type": "object",
"properties": {
@@ -4603,6 +4651,31 @@
}
}
},
"localai.TTSModelVoices": {
"type": "object",
"properties": {
"model": {
"type": "string"
},
"voices": {
"type": "array",
"items": {
"$ref": "#/definitions/config.TTSVoice"
}
}
}
},
"localai.TTSVoicesResponse": {
"type": "object",
"properties": {
"data": {
"type": "array",
"items": {
"$ref": "#/definitions/localai.TTSModelVoices"
}
}
}
},
"localai.UpdateMaxReplicasPerModelRequest": {
"type": "object",
"properties": {
+48
View File
@@ -33,6 +33,15 @@ definitions:
description: NotBefore is an RFC3339 timestamp. Empty disables the time check.
type: string
type: object
config.TTSVoice:
properties:
gender:
type: string
language:
type: string
name:
type: string
type: object
functions.Function:
properties:
description:
@@ -431,6 +440,22 @@ definitions:
success:
type: boolean
type: object
localai.TTSModelVoices:
properties:
model:
type: string
voices:
items:
$ref: '#/definitions/config.TTSVoice'
type: array
type: object
localai.TTSVoicesResponse:
properties:
data:
items:
$ref: '#/definitions/localai.TTSModelVoices'
type: array
type: object
localai.UpdateMaxReplicasPerModelRequest:
properties:
value:
@@ -4982,6 +5007,29 @@ paths:
summary: Transcribes audio into the input language.
tags:
- audio
/v1/audio/voices:
get:
description: List named voices and their language and gender metadata. Use the
optional model query parameter to filter the response.
parameters:
- description: Installed model name
in: query
name: model
type: string
produces:
- application/json
responses:
"200":
description: OK
schema:
$ref: '#/definitions/localai.TTSVoicesResponse'
"404":
description: Not Found
schema:
$ref: '#/definitions/schema.ErrorResponse'
summary: List text-to-speech voices
tags:
- audio
/v1/chat/completions:
post:
parameters: