feat(audio): add YuE2 music generation (#12617)

* feat(audio): forward sound generation params

Pass string-valued sound generation parameters from the public request to
the audio backend. Typed fields retain precedence over colliding generic
parameters, and source audio remains unavailable in the public schema.

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(gallery): add YuE2 music model

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* docs(audio): document YuE2 sound generation

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(audio): reserve source input parameter

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/backend.proto

@@ -907,6 +907,7 @@ message SoundGenerationRequest {
   //
   // Empty means "no identity supplied" and backends MUST skip the check.
   string ModelIdentity = 18;
+  map<string, string> params = 19;
 }
 
 message TokenizationResponse {

backend/cpp/audio-cpp/generation_request.cpp

@@ -170,6 +170,17 @@ build_sound_generation_request(const backend::SoundGenerationRequest &request,
         task.audio_input = std::move(*source_audio);
     }
 
+    // Generic options are copied first so typed request fields remain authoritative
+    // when callers provide the same key in both places.
+    for (const auto &param : request.params()) {
+        // src is reserved for trusted, decoded source audio and must not cross
+        // the generic option boundary as a caller-controlled filesystem path.
+        if (param.first == "src") {
+            continue;
+        }
+        task.options[param.first] = param.second;
+    }
+
     // WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in
     // the pinned upstream: ace_step, heartmula and stable_audio. Every key below
     // was grepped against find_option/parse_*_option in src/ and include/ rather

backend/cpp/audio-cpp/generation_request_ctest.cpp

@@ -441,6 +441,14 @@ static void test_sound_generation_full() {
     request.set_language("en");
     request.set_timesignature("4/4");
     request.set_instrumental(true);
+    (*request.mutable_params())["style"] = "cinematic";
+    (*request.mutable_params())["cot"] = "true";
+    (*request.mutable_params())["src"] = "/attacker-controlled/input.wav";
+    // Generic values lose collisions to typed fields, and src is transported as
+    // audio_input rather than leaking its filesystem path into engine options.
+    (*request.mutable_params())["lyrics"] = "generic lyrics";
+    (*request.mutable_params())["duration_seconds"] = "999";
+    request.set_src("/private/input.wav");
 
     const auto task = build_sound_generation_request(request, clip(48000, 2));
 
@@ -477,6 +485,16 @@ static void test_sound_generation_full() {
           "sound: instrumental forwarded");
     check(option_or(task.options, "language", "") == "en",
           "sound: language forwarded as an option alias");
+    check(option_or(task.options, "style", "") == "cinematic",
+          "sound: generic style parameter forwarded");
+    check(option_or(task.options, "cot", "") == "true",
+          "sound: generic cot parameter forwarded");
+    check(option_or(task.options, "lyrics", "") == "first line\nsecond line",
+          "sound: typed lyrics overrides a colliding generic parameter");
+    check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0,
+          "sound: typed duration overrides a colliding generic parameter");
+    check(!has_key(task.options, "src"),
+          "sound: generic params cannot cross the reserved source-input boundary");
     check(task.text_input->language == "en",
           "sound: language reaches the transcript, which is what ace_step reads");
     check(task.audio_input.has_value() &&

core/backend/model_identity_modalities_test.go

@@ -48,6 +48,7 @@ type recordingBackend struct {
 	// SoundGenerationRequest carries both `model` (staged path) and
 	// `ModelIdentity`; the spec below asserts they are populated independently.
 	soundGenModelField string
+	soundGenParams     map[string]string
 }
 
 func newRecordingBackend() *recordingBackend {
@@ -71,6 +72,7 @@ func (r *recordingBackend) GenerateImage(_ context.Context, in *pb.GenerateImage
 func (r *recordingBackend) SoundGeneration(_ context.Context, in *pb.SoundGenerationRequest, _ ...ggrpc.CallOption) (*pb.Result, error) {
 	r.record("SoundGeneration", in)
 	r.soundGenModelField = in.GetModel()
+	r.soundGenParams = in.GetParams()
 	return &pb.Result{Success: true}, nil
 }
 
@@ -321,11 +323,13 @@ var _ = Describe("modality requests carry the model identity", func() {
 	// comparing it would reject valid requests. Both must arrive populated.
 	It("SoundGeneration", func() {
 		appCfg.GeneratedContentDir = GinkgoT().TempDir()
-		_, _, err := backend.SoundGeneration(ctx, "hello", nil, nil, nil, nil, nil, nil, "", "", nil, "", "", "", nil, loader, appCfg, cfg)
+		params := map[string]string{"style": "jazz", "cot": "true"}
+		_, _, err := backend.SoundGeneration(ctx, "hello", nil, nil, nil, nil, nil, nil, "", "", nil, "", "", "", nil, params, loader, appCfg, cfg)
 		Expect(err).ToNot(HaveOccurred())
 		expectIdentity("SoundGeneration")
 		Expect(rec.soundGenModelField).To(Equal(identityModelFile),
 			"the staged `model` field must still be sent, separately from the identity")
+		Expect(rec.soundGenParams).To(Equal(params))
 	})
 
 	It("ModelAudioTransform", func() {

core/backend/soundgeneration.go

@@ -30,6 +30,7 @@ func SoundGeneration(
 	language string,
 	timesignature string,
 	instrumental *bool,
+	params map[string]string,
 	loader *model.ModelLoader,
 	appConfig *config.ApplicationConfig,
 	modelConfig config.ModelConfig,
@@ -75,6 +76,7 @@ func SoundGeneration(
 		Temperature:   temperature,
 		Src:           sourceFile,
 		SrcDivisor:    sourceDivisor,
+		Params:        params,
 	}
 	if think != nil {
 		req.Think = think

core/cli/soundgeneration.go

@@ -100,7 +100,7 @@ func (t *SoundGenerationCMD) Run(ctx *cliContext.Context) error {
 	filePath, _, err := backend.SoundGeneration(context.Background(), text,
 		parseToFloat32Ptr(t.Duration), parseToFloat32Ptr(t.Temperature), &t.DoSample,
 		inputFile, parseToInt32Ptr(t.InputFileSampleDivisor),
-		nil, "", "", nil, "", "", "", nil,
+		nil, "", "", nil, "", "", "", nil, nil,
 		ml, opts, options)
 
 	if err != nil {

core/gallery/yue2_entry_test.go

@@ -0,0 +1,60 @@
+// SPDX-License-Identifier: MIT
+
+package gallery_test
+
+import (
+	"strings"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/gallery"
+)
+
+var _ = Describe("gallery/index.yaml Yue2 entry", func() {
+	It("installs the complete Q8_0 and F16 package as an audio-cpp model directory", func() {
+		entries, err := loadGalleryIndex()
+		Expect(err).ToNot(HaveOccurred())
+
+		models := make([]*gallery.GalleryModel, 0, len(entries))
+		for i := range entries {
+			models = append(models, &entries[i])
+		}
+		entry := gallery.FindGalleryElement(models, "audio-cpp-yue2-3b")
+		Expect(entry).ToNot(BeNil())
+		Expect(entry.License).To(Equal("cc-by-nc-4.0"))
+		Expect(entry.Tags).To(ContainElements("audio-cpp", "yue2", "music-generation", "text-to-audio", "sound-generation", "gguf", "quantized"))
+		Expect(entry.Description).To(ContainSubstring("Q8_0"))
+		Expect(entry.Description).To(ContainSubstring("F16 VAE"))
+
+		Expect(entry.Overrides).To(HaveKeyWithValue("backend", "audio-cpp"))
+		Expect(entry.Overrides).To(HaveKeyWithValue("name", "audio-cpp-yue2-3b"))
+		Expect(entry.Overrides["known_usecases"]).To(ConsistOf("sound_generation"))
+		Expect(entry.Overrides["parameters"]).To(HaveKeyWithValue("model", "audio-cpp/yue2-3b"))
+		Expect(entry.Overrides["options"]).To(ConsistOf(
+			"family:yue2",
+			"task:gen",
+			"backend:best",
+			"session.yue2.model_gguf:yue2-3b-q8_0.gguf",
+			"session.yue2.vae_gguf:yue2-vae-f16.gguf",
+		))
+
+		wantFiles := []gallery.File{
+			{Filename: "audio-cpp/yue2-3b/yue2-3b-q8_0.gguf", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/yue2-3b-q8_0.gguf", SHA256: "52be345b5155bf9ef0b2411e7df8d1d2b8e13b6d977a2f70c35612ec69b1f591"},
+			{Filename: "audio-cpp/yue2-3b/yue2-vae-f16.gguf", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/yue2-vae-f16.gguf", SHA256: "d4f4a05d8f291ae820cd1e43609da3fa91b56465810091a2b08c3350b751719d"},
+			{Filename: "audio-cpp/yue2-3b/sidecars/yue2-model-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-model-config.json", SHA256: "ad3477bbef890bf98ae196c1e4b44779494a6231c4ab66f32708eabadf265329"},
+			{Filename: "audio-cpp/yue2-3b/sidecars/yue2-generation-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-generation-config.json", SHA256: "203830cebde6e3644eb291925362990d198e66c3b6006cd11f1aaa21904bcc61"},
+			{Filename: "audio-cpp/yue2-3b/sidecars/yue2-qwen.tiktoken", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-qwen.tiktoken", SHA256: "b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186"},
+			{Filename: "audio-cpp/yue2-3b/sidecars/yue2-vae-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-vae-config.json", SHA256: "f0191bb9694009956de44e0c361a6f1334760be4c8f848e599bde242a54a0970"},
+		}
+		Expect(entry.AdditionalFiles).To(ConsistOf(wantFiles))
+
+		sidecars := 0
+		for _, file := range entry.AdditionalFiles {
+			if strings.Contains(file.Filename, "/sidecars/") {
+				sidecars++
+			}
+		}
+		Expect(sidecars).To(Equal(4))
+	})
+})

core/http/endpoints/elevenlabs/soundgeneration.go

@@ -49,7 +49,7 @@ func SoundGenerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader
 			nil, nil,
 			input.Think, input.Caption, input.Lyrics, bpm, input.Keyscale,
 			language, input.Timesignature,
-			input.Instrumental,
+			input.Instrumental, input.Params,
 			ml, appConfig, *cfg)
 		if err != nil {
 			return err

core/schema/elevenlabs.go

@@ -40,7 +40,8 @@ type ElevenLabsSoundGenerationRequest struct {
 	VocalLanguage string `json:"vocal_language,omitempty" yaml:"vocal_language,omitempty"`
 	Timesignature string `json:"timesignature,omitempty" yaml:"timesignature,omitempty"`
 	// Simple mode: use text as description; optional instrumental / vocal_language
-	Instrumental *bool `json:"instrumental,omitempty" yaml:"instrumental,omitempty"`
+	Instrumental *bool             `json:"instrumental,omitempty" yaml:"instrumental,omitempty"`
+	Params       map[string]string `json:"params,omitempty" yaml:"params,omitempty"`
 }
 
 func (elttsr *ElevenLabsTTSRequest) ModelName(s *string) string {

core/schema/elevenlabs_test.go

@@ -0,0 +1,34 @@
+package schema_test
+
+import (
+	"encoding/json"
+	"reflect"
+	"strings"
+
+	"github.com/mudler/LocalAI/core/schema"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("ElevenLabs sound generation request", func() {
+	It("accepts string parameters", func() {
+		var request schema.ElevenLabsSoundGenerationRequest
+		Expect(json.Unmarshal([]byte(`{"text":"song","params":{"style":"jazz","cot":"true"}}`), &request)).To(Succeed())
+		Expect(request.Params).To(Equal(map[string]string{"style": "jazz", "cot": "true"}))
+	})
+
+	It("rejects non-string parameter values", func() {
+		var request schema.ElevenLabsSoundGenerationRequest
+		err := json.Unmarshal([]byte(`{"text":"song","params":{"cot":true}}`), &request)
+		Expect(err).To(MatchError(ContainSubstring("cannot unmarshal bool into Go struct field ElevenLabsSoundGenerationRequest.params of type string")))
+	})
+
+	It("does not expose source audio", func() {
+		requestType := reflect.TypeOf(schema.ElevenLabsSoundGenerationRequest{})
+		for i := 0; i < requestType.NumField(); i++ {
+			field := requestType.Field(i)
+			Expect(strings.Split(field.Tag.Get("json"), ",")[0]).ToNot(Equal("src"))
+		}
+	})
+})

docs/content/features/audio-cpp.md

@@ -243,6 +243,11 @@ voice conversion from the same weights.
   streaming speech requests. If you supply both parameters, `reference_text`
   takes precedence. For direct requests with a reference file in `voice`, supply
   its transcript in one of these parameters.
+- **YuE2**: the `audio-cpp-yue2-3b` gallery entry installs a model
+  directory containing the Q8_0 main model, the F16 VAE, and four required
+  sidecar files. Each request requires a non-empty `params.style` string. Set
+  the `params.cot` string to `off`, `melody`, or `full`. The model and GGUF
+  package are licensed under CC-BY-NC-4.0 for non-commercial use.
 - **Supertonic**: use the `orig` GGUF package, whose weights are f32. The f16 package was
   observed to reach `ggml_concat` with mismatched operand types and take the backend
   process down with `SIGABRT` on the first request, rather than returning an error.

docs/content/features/text-to-audio.md

@@ -411,6 +411,28 @@ The `/v1/sound-generation` endpoint is compatible with the [ElevenLabs sound gen
 | `vocal_language`    | `string` | No       | Vocal language (fallback if `language` is empty) |
 | `timesignature`     | `string` | No       | Time signature (for example `4`)                |
 | `instrumental`      | `bool`   | No       | Generate instrumental audio (no vocals)         |
+| `params`            | `object` | No       | Backend-specific options with string keys and string values |
+
+Use `params` to pass backend-specific string options that do not have fixed request
+fields. Every value in `params` must be a string. Existing fixed fields remain
+authoritative when a fixed field and `params` contain the same option.
+
+Install the YuE2 gallery model and generate a song:
+
+```bash
+local-ai models install audio-cpp-yue2-3b
+
+curl http://localhost:8080/v1/sound-generation \
+  -H "Content-Type: application/json" \
+  -d '{
+    "model_id": "audio-cpp-yue2-3b",
+    "lyrics": "[Verse]\nMorning light across the bay\nCarry every cloud away",
+    "params": {
+      "style": "acoustic folk, warm vocals, gentle guitar",
+      "cot": "off"
+    }
+  }' --output yue2.wav
+```
 
 Error responses: `400` for a missing or invalid model or request parameters, and `500` for a backend error during sound generation.
 

gallery/index.yaml

@@ -64144,6 +64144,63 @@
     - model: gemma-4-26b-a4b-it-heretic-apex-i-balanced
     - model: gemma-4-26b-a4b-it-heretic-apex-i-compact
     - model: gemma-4-26b-a4b-it-heretic-apex-i-mini
+- name: audio-cpp-yue2-3b
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/m-a-p/YuE2-3B
+    - https://huggingface.co/audio-cpp/Yue2-3B-GGUF
+    - https://github.com/0xShug0/audio.cpp
+  description: |
+    Yue2-3B is a text-to-music model that generates songs from lyrics and a
+    style prompt, with optional symbolic ABC melody or score conditioning. This
+    audio.cpp package uses the published Q8_0 main model and F16 VAE pairing and
+    preserves the model, generation, tokenizer, and VAE sidecars it requires.
+
+    The model and GGUF package are licensed CC-BY-NC-4.0 for non-commercial use.
+    Review the upstream license before installation or deployment.
+  license: cc-by-nc-4.0
+  tags:
+    - audio-cpp
+    - yue2
+    - music-generation
+    - text-to-audio
+    - sound-generation
+    - gguf
+    - ggml
+    - quantized
+  last_checked: "2026-10-10"
+  overrides:
+    backend: audio-cpp
+    known_usecases:
+      - sound_generation
+    name: audio-cpp-yue2-3b
+    options:
+      - family:yue2
+      - task:gen
+      - backend:best
+      - session.yue2.model_gguf:yue2-3b-q8_0.gguf
+      - session.yue2.vae_gguf:yue2-vae-f16.gguf
+    parameters:
+      model: audio-cpp/yue2-3b
+  files:
+    - filename: audio-cpp/yue2-3b/yue2-3b-q8_0.gguf
+      sha256: 52be345b5155bf9ef0b2411e7df8d1d2b8e13b6d977a2f70c35612ec69b1f591
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/yue2-3b-q8_0.gguf
+    - filename: audio-cpp/yue2-3b/yue2-vae-f16.gguf
+      sha256: d4f4a05d8f291ae820cd1e43609da3fa91b56465810091a2b08c3350b751719d
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/yue2-vae-f16.gguf
+    - filename: audio-cpp/yue2-3b/sidecars/yue2-model-config.json
+      sha256: ad3477bbef890bf98ae196c1e4b44779494a6231c4ab66f32708eabadf265329
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-model-config.json
+    - filename: audio-cpp/yue2-3b/sidecars/yue2-generation-config.json
+      sha256: 203830cebde6e3644eb291925362990d198e66c3b6006cd11f1aaa21904bcc61
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-generation-config.json
+    - filename: audio-cpp/yue2-3b/sidecars/yue2-qwen.tiktoken
+      sha256: b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-qwen.tiktoken
+    - filename: audio-cpp/yue2-3b/sidecars/yue2-vae-config.json
+      sha256: f0191bb9694009956de44e0c361a6f1334760be4c8f848e599bde242a54a0970
+      uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-vae-config.json
 - name: audio-cpp-indextts-2.5
   url: github:mudler/LocalAI/gallery/virtual.yaml@master
   urls:

swagger/docs.go

@@ -6355,6 +6355,12 @@ const docTemplate = `{
                 "model_id": {
                     "type": "string"
                 },
+                "params": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
                 "prompt_influence": {
                     "type": "number"
                 },

swagger/swagger.json

@@ -6352,6 +6352,12 @@
                 "model_id": {
                     "type": "string"
                 },
+                "params": {
+                    "type": "object",
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
                 "prompt_influence": {
                     "type": "number"
                 },

swagger/swagger.yaml

@@ -1324,6 +1324,10 @@ definitions:
         type: string
       model_id:
         type: string
+      params:
+        additionalProperties:
+          type: string
+        type: object
       prompt_influence:
         type: number
       text: