From 08871b1a6ae69241ff4d484e1050f1c6e6fdbd1c Mon Sep 17 00:00:00 2001 From: mudler-agent Date: Sun, 11 Oct 2026 00:26:10 +0200 Subject: [PATCH] feat(audio): add YuE2 music generation (#12617) * feat(audio): forward sound generation params Pass string-valued sound generation parameters from the public request to the audio backend. Typed fields retain precedence over colliding generic parameters, and source audio remains unavailable in the public schema. Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto * feat(gallery): add YuE2 music model Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto * docs(audio): document YuE2 sound generation Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto * fix(audio): reserve source input parameter Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- backend/backend.proto | 1 + backend/cpp/audio-cpp/generation_request.cpp | 11 ++++ .../audio-cpp/generation_request_ctest.cpp | 18 ++++++ .../backend/model_identity_modalities_test.go | 6 +- core/backend/soundgeneration.go | 2 + core/cli/soundgeneration.go | 2 +- core/gallery/yue2_entry_test.go | 60 +++++++++++++++++++ .../endpoints/elevenlabs/soundgeneration.go | 2 +- core/schema/elevenlabs.go | 3 +- core/schema/elevenlabs_test.go | 34 +++++++++++ docs/content/features/audio-cpp.md | 5 ++ docs/content/features/text-to-audio.md | 22 +++++++ gallery/index.yaml | 57 ++++++++++++++++++ swagger/docs.go | 6 ++ swagger/swagger.json | 6 ++ swagger/swagger.yaml | 4 ++ 16 files changed, 235 insertions(+), 4 deletions(-) create mode 100644 core/gallery/yue2_entry_test.go create mode 100644 core/schema/elevenlabs_test.go diff --git a/backend/backend.proto b/backend/backend.proto index 614f006e9..8a7ef8b17 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -907,6 +907,7 @@ message SoundGenerationRequest { // // Empty means "no identity supplied" and backends MUST skip the check. string ModelIdentity = 18; + map params = 19; } message TokenizationResponse { diff --git a/backend/cpp/audio-cpp/generation_request.cpp b/backend/cpp/audio-cpp/generation_request.cpp index c0aa2b95b..f09d14284 100644 --- a/backend/cpp/audio-cpp/generation_request.cpp +++ b/backend/cpp/audio-cpp/generation_request.cpp @@ -170,6 +170,17 @@ build_sound_generation_request(const backend::SoundGenerationRequest &request, task.audio_input = std::move(*source_audio); } + // Generic options are copied first so typed request fields remain authoritative + // when callers provide the same key in both places. + for (const auto ¶m : request.params()) { + // src is reserved for trusted, decoded source audio and must not cross + // the generic option boundary as a caller-controlled filesystem path. + if (param.first == "src") { + continue; + } + task.options[param.first] = param.second; + } + // WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in // the pinned upstream: ace_step, heartmula and stable_audio. Every key below // was grepped against find_option/parse_*_option in src/ and include/ rather diff --git a/backend/cpp/audio-cpp/generation_request_ctest.cpp b/backend/cpp/audio-cpp/generation_request_ctest.cpp index a2fcce0e3..32c905090 100644 --- a/backend/cpp/audio-cpp/generation_request_ctest.cpp +++ b/backend/cpp/audio-cpp/generation_request_ctest.cpp @@ -441,6 +441,14 @@ static void test_sound_generation_full() { request.set_language("en"); request.set_timesignature("4/4"); request.set_instrumental(true); + (*request.mutable_params())["style"] = "cinematic"; + (*request.mutable_params())["cot"] = "true"; + (*request.mutable_params())["src"] = "/attacker-controlled/input.wav"; + // Generic values lose collisions to typed fields, and src is transported as + // audio_input rather than leaking its filesystem path into engine options. + (*request.mutable_params())["lyrics"] = "generic lyrics"; + (*request.mutable_params())["duration_seconds"] = "999"; + request.set_src("/private/input.wav"); const auto task = build_sound_generation_request(request, clip(48000, 2)); @@ -477,6 +485,16 @@ static void test_sound_generation_full() { "sound: instrumental forwarded"); check(option_or(task.options, "language", "") == "en", "sound: language forwarded as an option alias"); + check(option_or(task.options, "style", "") == "cinematic", + "sound: generic style parameter forwarded"); + check(option_or(task.options, "cot", "") == "true", + "sound: generic cot parameter forwarded"); + check(option_or(task.options, "lyrics", "") == "first line\nsecond line", + "sound: typed lyrics overrides a colliding generic parameter"); + check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0, + "sound: typed duration overrides a colliding generic parameter"); + check(!has_key(task.options, "src"), + "sound: generic params cannot cross the reserved source-input boundary"); check(task.text_input->language == "en", "sound: language reaches the transcript, which is what ace_step reads"); check(task.audio_input.has_value() && diff --git a/core/backend/model_identity_modalities_test.go b/core/backend/model_identity_modalities_test.go index 9f83f6da8..cc77a87b8 100644 --- a/core/backend/model_identity_modalities_test.go +++ b/core/backend/model_identity_modalities_test.go @@ -48,6 +48,7 @@ type recordingBackend struct { // SoundGenerationRequest carries both `model` (staged path) and // `ModelIdentity`; the spec below asserts they are populated independently. soundGenModelField string + soundGenParams map[string]string } func newRecordingBackend() *recordingBackend { @@ -71,6 +72,7 @@ func (r *recordingBackend) GenerateImage(_ context.Context, in *pb.GenerateImage func (r *recordingBackend) SoundGeneration(_ context.Context, in *pb.SoundGenerationRequest, _ ...ggrpc.CallOption) (*pb.Result, error) { r.record("SoundGeneration", in) r.soundGenModelField = in.GetModel() + r.soundGenParams = in.GetParams() return &pb.Result{Success: true}, nil } @@ -321,11 +323,13 @@ var _ = Describe("modality requests carry the model identity", func() { // comparing it would reject valid requests. Both must arrive populated. It("SoundGeneration", func() { appCfg.GeneratedContentDir = GinkgoT().TempDir() - _, _, err := backend.SoundGeneration(ctx, "hello", nil, nil, nil, nil, nil, nil, "", "", nil, "", "", "", nil, loader, appCfg, cfg) + params := map[string]string{"style": "jazz", "cot": "true"} + _, _, err := backend.SoundGeneration(ctx, "hello", nil, nil, nil, nil, nil, nil, "", "", nil, "", "", "", nil, params, loader, appCfg, cfg) Expect(err).ToNot(HaveOccurred()) expectIdentity("SoundGeneration") Expect(rec.soundGenModelField).To(Equal(identityModelFile), "the staged `model` field must still be sent, separately from the identity") + Expect(rec.soundGenParams).To(Equal(params)) }) It("ModelAudioTransform", func() { diff --git a/core/backend/soundgeneration.go b/core/backend/soundgeneration.go index 82cbe8492..98fb64201 100644 --- a/core/backend/soundgeneration.go +++ b/core/backend/soundgeneration.go @@ -30,6 +30,7 @@ func SoundGeneration( language string, timesignature string, instrumental *bool, + params map[string]string, loader *model.ModelLoader, appConfig *config.ApplicationConfig, modelConfig config.ModelConfig, @@ -75,6 +76,7 @@ func SoundGeneration( Temperature: temperature, Src: sourceFile, SrcDivisor: sourceDivisor, + Params: params, } if think != nil { req.Think = think diff --git a/core/cli/soundgeneration.go b/core/cli/soundgeneration.go index 3eb6cfa4b..76d97461d 100644 --- a/core/cli/soundgeneration.go +++ b/core/cli/soundgeneration.go @@ -100,7 +100,7 @@ func (t *SoundGenerationCMD) Run(ctx *cliContext.Context) error { filePath, _, err := backend.SoundGeneration(context.Background(), text, parseToFloat32Ptr(t.Duration), parseToFloat32Ptr(t.Temperature), &t.DoSample, inputFile, parseToInt32Ptr(t.InputFileSampleDivisor), - nil, "", "", nil, "", "", "", nil, + nil, "", "", nil, "", "", "", nil, nil, ml, opts, options) if err != nil { diff --git a/core/gallery/yue2_entry_test.go b/core/gallery/yue2_entry_test.go new file mode 100644 index 000000000..0cdc1c78b --- /dev/null +++ b/core/gallery/yue2_entry_test.go @@ -0,0 +1,60 @@ +// SPDX-License-Identifier: MIT + +package gallery_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +var _ = Describe("gallery/index.yaml Yue2 entry", func() { + It("installs the complete Q8_0 and F16 package as an audio-cpp model directory", func() { + entries, err := loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + + models := make([]*gallery.GalleryModel, 0, len(entries)) + for i := range entries { + models = append(models, &entries[i]) + } + entry := gallery.FindGalleryElement(models, "audio-cpp-yue2-3b") + Expect(entry).ToNot(BeNil()) + Expect(entry.License).To(Equal("cc-by-nc-4.0")) + Expect(entry.Tags).To(ContainElements("audio-cpp", "yue2", "music-generation", "text-to-audio", "sound-generation", "gguf", "quantized")) + Expect(entry.Description).To(ContainSubstring("Q8_0")) + Expect(entry.Description).To(ContainSubstring("F16 VAE")) + + Expect(entry.Overrides).To(HaveKeyWithValue("backend", "audio-cpp")) + Expect(entry.Overrides).To(HaveKeyWithValue("name", "audio-cpp-yue2-3b")) + Expect(entry.Overrides["known_usecases"]).To(ConsistOf("sound_generation")) + Expect(entry.Overrides["parameters"]).To(HaveKeyWithValue("model", "audio-cpp/yue2-3b")) + Expect(entry.Overrides["options"]).To(ConsistOf( + "family:yue2", + "task:gen", + "backend:best", + "session.yue2.model_gguf:yue2-3b-q8_0.gguf", + "session.yue2.vae_gguf:yue2-vae-f16.gguf", + )) + + wantFiles := []gallery.File{ + {Filename: "audio-cpp/yue2-3b/yue2-3b-q8_0.gguf", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/yue2-3b-q8_0.gguf", SHA256: "52be345b5155bf9ef0b2411e7df8d1d2b8e13b6d977a2f70c35612ec69b1f591"}, + {Filename: "audio-cpp/yue2-3b/yue2-vae-f16.gguf", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/yue2-vae-f16.gguf", SHA256: "d4f4a05d8f291ae820cd1e43609da3fa91b56465810091a2b08c3350b751719d"}, + {Filename: "audio-cpp/yue2-3b/sidecars/yue2-model-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-model-config.json", SHA256: "ad3477bbef890bf98ae196c1e4b44779494a6231c4ab66f32708eabadf265329"}, + {Filename: "audio-cpp/yue2-3b/sidecars/yue2-generation-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-generation-config.json", SHA256: "203830cebde6e3644eb291925362990d198e66c3b6006cd11f1aaa21904bcc61"}, + {Filename: "audio-cpp/yue2-3b/sidecars/yue2-qwen.tiktoken", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-qwen.tiktoken", SHA256: "b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186"}, + {Filename: "audio-cpp/yue2-3b/sidecars/yue2-vae-config.json", URI: "huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-vae-config.json", SHA256: "f0191bb9694009956de44e0c361a6f1334760be4c8f848e599bde242a54a0970"}, + } + Expect(entry.AdditionalFiles).To(ConsistOf(wantFiles)) + + sidecars := 0 + for _, file := range entry.AdditionalFiles { + if strings.Contains(file.Filename, "/sidecars/") { + sidecars++ + } + } + Expect(sidecars).To(Equal(4)) + }) +}) diff --git a/core/http/endpoints/elevenlabs/soundgeneration.go b/core/http/endpoints/elevenlabs/soundgeneration.go index eb9152e43..fc081edb0 100644 --- a/core/http/endpoints/elevenlabs/soundgeneration.go +++ b/core/http/endpoints/elevenlabs/soundgeneration.go @@ -49,7 +49,7 @@ func SoundGenerationEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader nil, nil, input.Think, input.Caption, input.Lyrics, bpm, input.Keyscale, language, input.Timesignature, - input.Instrumental, + input.Instrumental, input.Params, ml, appConfig, *cfg) if err != nil { return err diff --git a/core/schema/elevenlabs.go b/core/schema/elevenlabs.go index f8790a333..1906cca6f 100644 --- a/core/schema/elevenlabs.go +++ b/core/schema/elevenlabs.go @@ -40,7 +40,8 @@ type ElevenLabsSoundGenerationRequest struct { VocalLanguage string `json:"vocal_language,omitempty" yaml:"vocal_language,omitempty"` Timesignature string `json:"timesignature,omitempty" yaml:"timesignature,omitempty"` // Simple mode: use text as description; optional instrumental / vocal_language - Instrumental *bool `json:"instrumental,omitempty" yaml:"instrumental,omitempty"` + Instrumental *bool `json:"instrumental,omitempty" yaml:"instrumental,omitempty"` + Params map[string]string `json:"params,omitempty" yaml:"params,omitempty"` } func (elttsr *ElevenLabsTTSRequest) ModelName(s *string) string { diff --git a/core/schema/elevenlabs_test.go b/core/schema/elevenlabs_test.go new file mode 100644 index 000000000..3527bbc52 --- /dev/null +++ b/core/schema/elevenlabs_test.go @@ -0,0 +1,34 @@ +package schema_test + +import ( + "encoding/json" + "reflect" + "strings" + + "github.com/mudler/LocalAI/core/schema" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("ElevenLabs sound generation request", func() { + It("accepts string parameters", func() { + var request schema.ElevenLabsSoundGenerationRequest + Expect(json.Unmarshal([]byte(`{"text":"song","params":{"style":"jazz","cot":"true"}}`), &request)).To(Succeed()) + Expect(request.Params).To(Equal(map[string]string{"style": "jazz", "cot": "true"})) + }) + + It("rejects non-string parameter values", func() { + var request schema.ElevenLabsSoundGenerationRequest + err := json.Unmarshal([]byte(`{"text":"song","params":{"cot":true}}`), &request) + Expect(err).To(MatchError(ContainSubstring("cannot unmarshal bool into Go struct field ElevenLabsSoundGenerationRequest.params of type string"))) + }) + + It("does not expose source audio", func() { + requestType := reflect.TypeOf(schema.ElevenLabsSoundGenerationRequest{}) + for i := 0; i < requestType.NumField(); i++ { + field := requestType.Field(i) + Expect(strings.Split(field.Tag.Get("json"), ",")[0]).ToNot(Equal("src")) + } + }) +}) diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md index b122b3a3c..30fe25d93 100644 --- a/docs/content/features/audio-cpp.md +++ b/docs/content/features/audio-cpp.md @@ -243,6 +243,11 @@ voice conversion from the same weights. streaming speech requests. If you supply both parameters, `reference_text` takes precedence. For direct requests with a reference file in `voice`, supply its transcript in one of these parameters. +- **YuE2**: the `audio-cpp-yue2-3b` gallery entry installs a model + directory containing the Q8_0 main model, the F16 VAE, and four required + sidecar files. Each request requires a non-empty `params.style` string. Set + the `params.cot` string to `off`, `melody`, or `full`. The model and GGUF + package are licensed under CC-BY-NC-4.0 for non-commercial use. - **Supertonic**: use the `orig` GGUF package, whose weights are f32. The f16 package was observed to reach `ggml_concat` with mismatched operand types and take the backend process down with `SIGABRT` on the first request, rather than returning an error. diff --git a/docs/content/features/text-to-audio.md b/docs/content/features/text-to-audio.md index 36b9139a9..f5fd4bdd5 100644 --- a/docs/content/features/text-to-audio.md +++ b/docs/content/features/text-to-audio.md @@ -411,6 +411,28 @@ The `/v1/sound-generation` endpoint is compatible with the [ElevenLabs sound gen | `vocal_language` | `string` | No | Vocal language (fallback if `language` is empty) | | `timesignature` | `string` | No | Time signature (for example `4`) | | `instrumental` | `bool` | No | Generate instrumental audio (no vocals) | +| `params` | `object` | No | Backend-specific options with string keys and string values | + +Use `params` to pass backend-specific string options that do not have fixed request +fields. Every value in `params` must be a string. Existing fixed fields remain +authoritative when a fixed field and `params` contain the same option. + +Install the YuE2 gallery model and generate a song: + +```bash +local-ai models install audio-cpp-yue2-3b + +curl http://localhost:8080/v1/sound-generation \ + -H "Content-Type: application/json" \ + -d '{ + "model_id": "audio-cpp-yue2-3b", + "lyrics": "[Verse]\nMorning light across the bay\nCarry every cloud away", + "params": { + "style": "acoustic folk, warm vocals, gentle guitar", + "cot": "off" + } + }' --output yue2.wav +``` Error responses: `400` for a missing or invalid model or request parameters, and `500` for a backend error during sound generation. diff --git a/gallery/index.yaml b/gallery/index.yaml index 73b875194..3b8decb30 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -64144,6 +64144,63 @@ - model: gemma-4-26b-a4b-it-heretic-apex-i-balanced - model: gemma-4-26b-a4b-it-heretic-apex-i-compact - model: gemma-4-26b-a4b-it-heretic-apex-i-mini +- name: audio-cpp-yue2-3b + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/m-a-p/YuE2-3B + - https://huggingface.co/audio-cpp/Yue2-3B-GGUF + - https://github.com/0xShug0/audio.cpp + description: | + Yue2-3B is a text-to-music model that generates songs from lyrics and a + style prompt, with optional symbolic ABC melody or score conditioning. This + audio.cpp package uses the published Q8_0 main model and F16 VAE pairing and + preserves the model, generation, tokenizer, and VAE sidecars it requires. + + The model and GGUF package are licensed CC-BY-NC-4.0 for non-commercial use. + Review the upstream license before installation or deployment. + license: cc-by-nc-4.0 + tags: + - audio-cpp + - yue2 + - music-generation + - text-to-audio + - sound-generation + - gguf + - ggml + - quantized + last_checked: "2026-10-10" + overrides: + backend: audio-cpp + known_usecases: + - sound_generation + name: audio-cpp-yue2-3b + options: + - family:yue2 + - task:gen + - backend:best + - session.yue2.model_gguf:yue2-3b-q8_0.gguf + - session.yue2.vae_gguf:yue2-vae-f16.gguf + parameters: + model: audio-cpp/yue2-3b + files: + - filename: audio-cpp/yue2-3b/yue2-3b-q8_0.gguf + sha256: 52be345b5155bf9ef0b2411e7df8d1d2b8e13b6d977a2f70c35612ec69b1f591 + uri: huggingface://audio-cpp/Yue2-3B-GGUF/yue2-3b-q8_0.gguf + - filename: audio-cpp/yue2-3b/yue2-vae-f16.gguf + sha256: d4f4a05d8f291ae820cd1e43609da3fa91b56465810091a2b08c3350b751719d + uri: huggingface://audio-cpp/Yue2-3B-GGUF/yue2-vae-f16.gguf + - filename: audio-cpp/yue2-3b/sidecars/yue2-model-config.json + sha256: ad3477bbef890bf98ae196c1e4b44779494a6231c4ab66f32708eabadf265329 + uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-model-config.json + - filename: audio-cpp/yue2-3b/sidecars/yue2-generation-config.json + sha256: 203830cebde6e3644eb291925362990d198e66c3b6006cd11f1aaa21904bcc61 + uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-generation-config.json + - filename: audio-cpp/yue2-3b/sidecars/yue2-qwen.tiktoken + sha256: b2b1b8dfb5cc5f024bafc373121c6aba3f66f9a5a0269e243470a1de16a33186 + uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-qwen.tiktoken + - filename: audio-cpp/yue2-3b/sidecars/yue2-vae-config.json + sha256: f0191bb9694009956de44e0c361a6f1334760be4c8f848e599bde242a54a0970 + uri: huggingface://audio-cpp/Yue2-3B-GGUF/sidecars/yue2-vae-config.json - name: audio-cpp-indextts-2.5 url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: diff --git a/swagger/docs.go b/swagger/docs.go index f8e1f0756..aa0d16d3e 100644 --- a/swagger/docs.go +++ b/swagger/docs.go @@ -6355,6 +6355,12 @@ const docTemplate = `{ "model_id": { "type": "string" }, + "params": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "prompt_influence": { "type": "number" }, diff --git a/swagger/swagger.json b/swagger/swagger.json index 1f98a5070..89bc82bf9 100644 --- a/swagger/swagger.json +++ b/swagger/swagger.json @@ -6352,6 +6352,12 @@ "model_id": { "type": "string" }, + "params": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "prompt_influence": { "type": "number" }, diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml index d663d42ce..296c61ba0 100644 --- a/swagger/swagger.yaml +++ b/swagger/swagger.yaml @@ -1324,6 +1324,10 @@ definitions: type: string model_id: type: string + params: + additionalProperties: + type: string + type: object prompt_influence: type: number text: