feat(sound): expand Simple prompts with chat (#12772)

* feat(sound): expand simple prompts with chat models

Allow callers to select an installed chat model for sound prompt
expansion. Validate its JSON before audio generation and preserve
explicit instrumental and language controls. Keep backend defaults
when expansion is omitted, including native Python ACE-Step sampling.

Assisted-by: Codex:gpt-6
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(sound): map expansion by loaded audio family

Map neutral expanded fields where the resolved audio family is known.
Keep unmarked requests unchanged and reject unsupported expansion before
inference. Clear YuE2 and HeartMuLa transcripts to prevent lyric fallback.

Assisted-by: Codex:gpt-6.1-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* feat(sound): choose a prompt expansion model

Require an explicit Simple choice and remember it separately from chat.
Clear unavailable choices instead of silently switching language models.
Preserve the choice in history and restore it when editing a past result.
Keep Backend default and Advanced requests compatible with existing APIs.

Document family support and align the published optional request schema.
Verify selection, request fields, persistence and history in the browser.

Assisted-by: Codex:gpt-6.1-sol
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(sound): authorize both generation models

Check the resolved audio model and explicit expansion model before
loading either. Permission lookup failures stop generation.

Assisted-by: Codex:gpt-6
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

* fix(sound): enforce JSON during prompt expansion

Constrain expansion output with the existing JSON grammar before decoding.
Require a llama-cpp chat model so unsupported backends cannot ignore it.
Keep strict field validation and fail before generating audio.

Assisted-by: Codex:gpt-6
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

---------

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>

backend/cpp/audio-cpp/generation_request.cpp

@@ -1,6 +1,7 @@
 #include "generation_request.h"
 
 #include <filesystem>
+#include <stdexcept>
 #include <string>
 #include <system_error>
 #include <utility>
@@ -153,7 +154,8 @@ build_tts_request(const backend::TTSRequest &request,
 
 engine::runtime::TaskRequest
 build_sound_generation_request(const backend::SoundGenerationRequest &request,
-                               std::optional<engine::runtime::AudioBuffer> source_audio) {
+                               std::optional<engine::runtime::AudioBuffer> source_audio,
+                               const std::string &family) {
     engine::runtime::TaskRequest task;
 
     engine::runtime::Transcript transcript;
@@ -266,6 +268,27 @@ build_sound_generation_request(const backend::SoundGenerationRequest &request,
         // vocal_language. No AudioGeneration family reads a "language" option.
         task.options["language"] = request.language();
     }
+    if (request.expanded()) {
+        if (family != "yue2" && family != "heartmula" &&
+            family != "ace_step" && family != "stable_audio") {
+            throw std::invalid_argument(
+                "audio-cpp: prompt expansion is unsupported for family '" + family + "'");
+        }
+        if (family == "heartmula" &&
+            (request.instrumental() || request.lyrics().empty())) {
+            throw std::invalid_argument(
+                "audio-cpp: heartmula prompt expansion requires non-empty lyrics and does not support instrumental requests");
+        }
+        // Expanded fields are authoritative even when empty. In particular,
+        // YuE2 falls back from empty lyrics to transcript text upstream.
+        task.options["lyrics"] = request.lyrics();
+        if (family == "yue2" || family == "heartmula") {
+            task.options[family == "yue2" ? "style" : "tags"] = request.caption();
+            task.text_input.reset();
+        } else if (family == "ace_step" || task.text_input->text.empty()) {
+            task.text_input->text = request.caption();
+        }
+    }
     return task;
 }
 

backend/cpp/audio-cpp/generation_request.h

@@ -70,9 +70,12 @@ build_tts_request(const backend::TTSRequest &request,
 
 // `source_audio` is SoundGenerationRequest.src already read, present exactly
 // when the field was set and non-empty. Same reasoning as above.
+// family comes from LoadedModel::family(), never from the request model name.
+// Marked expansions reject families without a verified field mapping.
 engine::runtime::TaskRequest
 build_sound_generation_request(const backend::SoundGenerationRequest &request,
-                               std::optional<engine::runtime::AudioBuffer> source_audio);
+                               std::optional<engine::runtime::AudioBuffer> source_audio,
+                               const std::string &family = "");
 
 // Lifts a text-conditioned transform route's text out of the request params
 // into TaskRequest.text_input, and reports whether it set one.

backend/cpp/audio-cpp/generation_request_ctest.cpp

@@ -22,6 +22,7 @@
 #include <filesystem>
 #include <fstream>
 #include <string>
+#include <stdexcept>
 #include <unordered_map>
 
 static int failures = 0;
@@ -639,7 +640,74 @@ static void test_transform_text_preserves_other_inputs() {
           "transform text: the source audio is untouched");
 }
 
+static void test_expanded_families() {
+    backend::SoundGenerationRequest request;
+    request.set_expanded(true);
+    request.set_caption("warm jazz");
+    request.set_lyrics("sing this");
+    request.set_bpm(123);
+    request.set_language("en");
+    request.set_text("raw description");
+    (*request.mutable_params())["tags"] = "old tags";
+    auto task = build_sound_generation_request(request, std::nullopt, "heartmula");
+    check(!task.text_input && task.options.at("tags") == "warm jazz" &&
+              task.options.at("lyrics") == "sing this", "expanded heartmula: tags and lyrics, no transcript");
+    task = build_sound_generation_request(request, std::nullopt, "ace_step");
+    check(task.text_input && task.text_input->text == "warm jazz" &&
+              task.text_input->language == "en" && task.options.at("bpm") == "123" &&
+              task.options.at("lyrics") == "sing this", "expanded ace_step: caption transcript and metadata");
+    request.set_text("");
+    task = build_sound_generation_request(request, std::nullopt, "stable_audio");
+    check(task.text_input && task.text_input->text == "warm jazz", "expanded stable_audio: caption fills empty transcript");
+    request.set_text("explicit transcript");
+    task = build_sound_generation_request(request, std::nullopt, "stable_audio");
+    check(task.text_input && task.text_input->text == "explicit transcript", "expanded stable_audio: preserve nonempty transcript");
+    for (const auto &family : {"unknown", "", "heartmula"}) {
+        if (std::string(family) == "heartmula") request.set_lyrics("");
+        bool rejected = false;
+        try { build_sound_generation_request(request, std::nullopt, family); }
+        catch (const std::invalid_argument &) { rejected = true; }
+        check(rejected, std::string("expanded rejects unsupported or empty lyrics: ") + family);
+    }
+    request.set_lyrics("sing this");
+    request.set_instrumental(true);
+    bool rejected = false;
+    try { build_sound_generation_request(request, std::nullopt, "heartmula"); }
+    catch (const std::invalid_argument &) { rejected = true; }
+    check(rejected, "expanded heartmula: instrumental rejected even with lyrics");
+    request.set_expanded(false);
+    (*request.mutable_params())["style"] = "";
+    task = build_sound_generation_request(request, std::nullopt, "unknown");
+    check(task.text_input && task.text_input->text == "explicit transcript" &&
+              task.options.at("style").empty(), "unmarked: unknown family and empty explicit style unchanged");
+    request.mutable_params()->erase("style");
+    task = build_sound_generation_request(request, std::nullopt, "unknown");
+    check(task.options.at("style") == "warm jazz", "unmarked: legacy caption style fallback unchanged");
+}
+
+static void test_expanded_yue2() {
+    backend::SoundGenerationRequest request;
+    request.set_expanded(true);
+    request.set_text("raw description");
+    request.set_caption("soft piano");
+    request.set_lyrics("");
+    (*request.mutable_params())["style"] = "conflicting style";
+    (*request.mutable_params())["lyrics"] = "conflicting lyrics";
+    const auto task = build_sound_generation_request(request, std::nullopt, "yue2");
+    check(!task.text_input.has_value(), "expanded yue2: raw text cannot become lyrics");
+    check(option_or(task.options, "style", "") == "soft piano",
+          "expanded yue2: caption overrides conflicting style");
+    check(option_or(task.options, "lyrics", "missing").empty(),
+          "expanded yue2: empty lyrics override params");
+    request.set_caption("");
+    const auto empty = build_sound_generation_request(request, std::nullopt, "yue2");
+    check(empty.options.at("style").empty() && !empty.text_input,
+          "expanded yue2: empty caption clears params style without raw fallback");
+}
+
 int main() {
+    test_expanded_families();
+    test_expanded_yue2();
     test_voice_is_reference_file();
     test_tts_shape();
     test_tts_plain();

backend/cpp/audio-cpp/grpc-server.cpp

@@ -46,6 +46,7 @@
 #include <mutex>
 #include <optional>
 #include <set>
+#include <stdexcept>
 #include <string>
 #include <thread>
 #include <utility>
@@ -1316,7 +1317,7 @@ public:
                     request->src(), kVoiceReferenceSampleRate);
             }
             const auto task = audiocpp_backend::build_sound_generation_request(
-                *request, std::move(source));
+                *request, std::move(source), model->family());
 
             audiocpp_backend::LaneEntry lane = model->acquire(0);
             const auto session = model->session_for(
@@ -1334,6 +1335,10 @@ public:
             result->set_success(true);
             result->set_message(request->dst());
             return GStatus::OK;
+        } catch (const std::invalid_argument &err) {
+            result->set_success(false);
+            result->set_message(err.what());
+            return GStatus(grpc::StatusCode::INVALID_ARGUMENT, err.what());
         } catch (const std::exception &err) {
             result->set_success(false);
             result->set_message(err.what());