From 2f3dd404b5c6fa0edcdeabf03c96f8ea95fb229d Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sat, 1 Aug 2026 09:12:54 +0200 Subject: [PATCH] feat(import): route MLX TTS models to mlx-audio (#11267) Detect text-to-speech MLX repositories during model import and emit a TTS-ready mlx-audio configuration. Expose mlx-audio in the backend preference dropdown for repositories without complete metadata. Assisted-by: Codex:gpt-5 Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/gallery/importers/mlx.go | 24 +++++++++--- core/gallery/importers/mlx_test.go | 42 +++++++++++++++++++++ core/http/endpoints/localai/backend.go | 1 + core/http/endpoints/localai/backend_test.go | 1 + docs/content/getting-started/models.md | 6 +++ 5 files changed, 69 insertions(+), 5 deletions(-) diff --git a/core/gallery/importers/mlx.go b/core/gallery/importers/mlx.go index 2698fe72f..1f23b48dc 100644 --- a/core/gallery/importers/mlx.go +++ b/core/gallery/importers/mlx.go @@ -3,6 +3,7 @@ package importers import ( "encoding/json" "path/filepath" + "slices" "strings" "github.com/mudler/LocalAI/core/config" @@ -31,7 +32,7 @@ func (i *MLXImporter) Match(details Details) bool { } b, ok := preferencesMap["backend"].(string) - if ok && b == "mlx" || b == "mlx-vlm" { + if ok && slices.Contains([]string{"mlx", "mlx-vlm", "mlx-audio"}, b) { return true } @@ -71,19 +72,32 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { // (issue #10269). Send them to the mlx-vlm backend, which applies the // processor-aware chat template. backend := "mlx" - if details.HuggingFace != nil && details.HuggingFace.PipelineTag == "image-text-to-text" { - backend = "mlx-vlm" + usecases := []string{config.UsecaseChat} + useTokenizerTemplate := true + if details.HuggingFace != nil { + switch details.HuggingFace.PipelineTag { + case "image-text-to-text": + backend = "mlx-vlm" + case "text-to-speech": + backend = "mlx-audio" + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } // An explicit backend preference always wins. b, ok := preferencesMap["backend"].(string) if ok { backend = b + if backend == "mlx-audio" { + usecases = []string{config.UsecaseTTS} + useTokenizerTemplate = false + } } modelConfig := config.ModelConfig{ Name: name, Description: description, - KnownUsecaseStrings: []string{config.UsecaseChat}, + KnownUsecaseStrings: usecases, Backend: backend, PredictionOptions: schema.PredictionOptions{ BasicModelRequest: schema.BasicModelRequest{ @@ -91,7 +105,7 @@ func (i *MLXImporter) Import(details Details) (gallery.ModelConfig, error) { }, }, TemplateConfig: config.TemplateConfig{ - UseTokenizerTemplate: true, + UseTokenizerTemplate: useTokenizerTemplate, }, } diff --git a/core/gallery/importers/mlx_test.go b/core/gallery/importers/mlx_test.go index 2eeaef3fb..9c3464557 100644 --- a/core/gallery/importers/mlx_test.go +++ b/core/gallery/importers/mlx_test.go @@ -48,6 +48,16 @@ var _ = Describe("MLXImporter", func() { Expect(result).To(BeTrue()) }) + It("should match when backend preference is mlx-audio", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://example.com/model", + Preferences: preferences, + } + + Expect(importer.Match(details)).To(BeTrue()) + }) + It("should not match when URI does not contain mlx-community/ and no backend preference", func() { details := importers.Details{ URI: "https://huggingface.co/other-org/test-model", @@ -123,6 +133,21 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should configure explicit mlx-audio imports for text-to-speech", func() { + preferences := json.RawMessage(`{"backend": "mlx-audio"}`) + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + Preferences: preferences, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should auto-route vision-language models to the mlx-vlm backend", func() { // gemma-4 E4B and similar VLMs declare pipeline_tag // "image-text-to-text" on HuggingFace. The text-only mlx-lm @@ -143,6 +168,23 @@ var _ = Describe("MLXImporter", func() { Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-vlm")) }) + It("should auto-route text-to-speech models to the mlx-audio backend", func() { + details := importers.Details{ + URI: "https://huggingface.co/mlx-community/Kokoro-82M-4bit", + HuggingFace: &hfapi.ModelDetails{ + ModelID: "mlx-community/Kokoro-82M-4bit", + PipelineTag: "text-to-speech", + }, + } + + modelConfig, err := importer.Import(details) + + Expect(err).ToNot(HaveOccurred()) + Expect(modelConfig.ConfigFile).To(ContainSubstring("backend: mlx-audio")) + Expect(modelConfig.ConfigFile).To(ContainSubstring("- tts")) + Expect(modelConfig.ConfigFile).ToNot(ContainSubstring("use_tokenizer_template: true")) + }) + It("should keep text-only models on the plain mlx backend", func() { details := importers.Details{ URI: "https://huggingface.co/mlx-community/Llama-3.2-1B-Instruct-4bit", diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index 3d4a25b33..221e2829a 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -38,6 +38,7 @@ var knownPrefOnlyBackends = []schema.KnownBackend{ {Name: "whisperx", Modality: "asr", AutoDetect: false, Description: "WhisperX transcription (preference-only)"}, {Name: "crispasr", Modality: "asr", AutoDetect: false, Description: "CrispASR multi-architecture transcription (preference-only)"}, // TTS + {Name: "mlx-audio", Modality: "tts", AutoDetect: false, Description: "MLX-Audio text-to-speech models (auto-detected; pref-only fallback)"}, {Name: "kokoros", Modality: "tts", AutoDetect: false, Description: "Kokoros TTS (preference-only)"}, {Name: "qwen-tts", Modality: "tts", AutoDetect: false, Description: "Qwen TTS (preference-only)"}, {Name: "qwen3-tts-cpp", Modality: "tts", AutoDetect: false, Description: "Qwen3 TTS C++ (preference-only)"}, diff --git a/core/http/endpoints/localai/backend_test.go b/core/http/endpoints/localai/backend_test.go index 3c43fb60c..04a486354 100644 --- a/core/http/endpoints/localai/backend_test.go +++ b/core/http/endpoints/localai/backend_test.go @@ -152,6 +152,7 @@ var _ = Describe("Backend Endpoints", func() { expectPrefOnly("tinygrad", "text") expectPrefOnly("trl", "text") expectPrefOnly("mlx-vlm", "text") + expectPrefOnly("mlx-audio", "tts") expectPrefOnly("whisperx", "asr") expectPrefOnly("crispasr", "asr") expectPrefOnly("kokoros", "tts") diff --git a/docs/content/getting-started/models.md b/docs/content/getting-started/models.md index d3345a346..1bd612358 100644 --- a/docs/content/getting-started/models.md +++ b/docs/content/getting-started/models.md @@ -80,6 +80,12 @@ The WebUI provides a powerful model import interface that supports both simple a - Custom preferences 5. Click "Import Model" to start the import process +Repositories under `mlx-community` are imported with the native MLX backend. +LocalAI uses Hugging Face's pipeline metadata to select `mlx-vlm` for +vision-language models and `mlx-audio` for text-to-speech models; other MLX +repositories use `mlx`. An explicit backend selection in the import form always +overrides this automatic routing. + ### Advanced Import Mode For full control over model configuration: