mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
feat(audio): add YuE2 music generation (#12617)
* feat(audio): forward sound generation params Pass string-valued sound generation parameters from the public request to the audio backend. Typed fields retain precedence over colliding generic parameters, and source audio remains unavailable in the public schema. Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(gallery): add YuE2 music model Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * docs(audio): document YuE2 sound generation Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(audio): reserve source input parameter Assisted-by: nib:gpt-5.6-sol Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
1 parent
9e73596a1b
commit
08871b1a6a
16 files changed
+235
-4
No files matched your search
@@ -170,6 +170,17 @@ build_sound_generation_request(const backend::SoundGenerationRequest &request,
|
||||
task.audio_input = std::move(*source_audio);
|
||||
}
|
||||
|
||||
// Generic options are copied first so typed request fields remain authoritative
|
||||
// when callers provide the same key in both places.
|
||||
for (const auto ¶m : request.params()) {
|
||||
// src is reserved for trusted, decoded source audio and must not cross
|
||||
// the generic option boundary as a caller-controlled filesystem path.
|
||||
if (param.first == "src") {
|
||||
continue;
|
||||
}
|
||||
task.options[param.first] = param.second;
|
||||
}
|
||||
|
||||
// WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in
|
||||
// the pinned upstream: ace_step, heartmula and stable_audio. Every key below
|
||||
// was grepped against find_option/parse_*_option in src/ and include/ rather
|
||||
|
||||
@@ -441,6 +441,14 @@ static void test_sound_generation_full() {
|
||||
request.set_language("en");
|
||||
request.set_timesignature("4/4");
|
||||
request.set_instrumental(true);
|
||||
(*request.mutable_params())["style"] = "cinematic";
|
||||
(*request.mutable_params())["cot"] = "true";
|
||||
(*request.mutable_params())["src"] = "/attacker-controlled/input.wav";
|
||||
// Generic values lose collisions to typed fields, and src is transported as
|
||||
// audio_input rather than leaking its filesystem path into engine options.
|
||||
(*request.mutable_params())["lyrics"] = "generic lyrics";
|
||||
(*request.mutable_params())["duration_seconds"] = "999";
|
||||
request.set_src("/private/input.wav");
|
||||
|
||||
const auto task = build_sound_generation_request(request, clip(48000, 2));
|
||||
|
||||
@@ -477,6 +485,16 @@ static void test_sound_generation_full() {
|
||||
"sound: instrumental forwarded");
|
||||
check(option_or(task.options, "language", "") == "en",
|
||||
"sound: language forwarded as an option alias");
|
||||
check(option_or(task.options, "style", "") == "cinematic",
|
||||
"sound: generic style parameter forwarded");
|
||||
check(option_or(task.options, "cot", "") == "true",
|
||||
"sound: generic cot parameter forwarded");
|
||||
check(option_or(task.options, "lyrics", "") == "first line\nsecond line",
|
||||
"sound: typed lyrics overrides a colliding generic parameter");
|
||||
check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0,
|
||||
"sound: typed duration overrides a colliding generic parameter");
|
||||
check(!has_key(task.options, "src"),
|
||||
"sound: generic params cannot cross the reserved source-input boundary");
|
||||
check(task.text_input->language == "en",
|
||||
"sound: language reaches the transcript, which is what ace_step reads");
|
||||
check(task.audio_input.has_value() &&
|
||||
|
||||
Reference in new issue
Block a user