feat(audio): add YuE2 music generation (#12617)

* feat(audio): forward sound generation params

Pass string-valued sound generation parameters from the public request to
the audio backend. Typed fields retain precedence over colliding generic
parameters, and source audio remains unavailable in the public schema.

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(gallery): add YuE2 music model

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* docs(audio): document YuE2 sound generation

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(audio): reserve source input parameter

Assisted-by: nib:gpt-5.6-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
mudler-agentandEttore Di Giacinto authored and GitHub committed 2026-10-11 00:26:10 +02:00
1 parent 9e73596a1b
commit 08871b1a6a
16 files changed
+235 -4

No files matched your search

@@ -170,6 +170,17 @@ build_sound_generation_request(const backend::SoundGenerationRequest &request,
task.audio_input = std::move(*source_audio);
}
// Generic options are copied first so typed request fields remain authoritative
// when callers provide the same key in both places.
for (const auto &param : request.params()) {
// src is reserved for trusted, decoded source audio and must not cross
// the generic option boundary as a caller-controlled filesystem path.
if (param.first == "src") {
continue;
}
task.options[param.first] = param.second;
}
// WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in
// the pinned upstream: ace_step, heartmula and stable_audio. Every key below
// was grepped against find_option/parse_*_option in src/ and include/ rather
@@ -441,6 +441,14 @@ static void test_sound_generation_full() {
request.set_language("en");
request.set_timesignature("4/4");
request.set_instrumental(true);
(*request.mutable_params())["style"] = "cinematic";
(*request.mutable_params())["cot"] = "true";
(*request.mutable_params())["src"] = "/attacker-controlled/input.wav";
// Generic values lose collisions to typed fields, and src is transported as
// audio_input rather than leaking its filesystem path into engine options.
(*request.mutable_params())["lyrics"] = "generic lyrics";
(*request.mutable_params())["duration_seconds"] = "999";
request.set_src("/private/input.wav");
const auto task = build_sound_generation_request(request, clip(48000, 2));
@@ -477,6 +485,16 @@ static void test_sound_generation_full() {
"sound: instrumental forwarded");
check(option_or(task.options, "language", "") == "en",
"sound: language forwarded as an option alias");
check(option_or(task.options, "style", "") == "cinematic",
"sound: generic style parameter forwarded");
check(option_or(task.options, "cot", "") == "true",
"sound: generic cot parameter forwarded");
check(option_or(task.options, "lyrics", "") == "first line\nsecond line",
"sound: typed lyrics overrides a colliding generic parameter");
check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0,
"sound: typed duration overrides a colliding generic parameter");
check(!has_key(task.options, "src"),
"sound: generic params cannot cross the reserved source-input boundary");
check(task.text_input->language == "en",
"sound: language reaches the transcript, which is what ace_step reads");
check(task.audio_input.has_value() &&