diff --git a/backend/cpp/audio-cpp/CMakeLists.txt b/backend/cpp/audio-cpp/CMakeLists.txt index b4ecb28b4..740d74c32 100644 --- a/backend/cpp/audio-cpp/CMakeLists.txt +++ b/backend/cpp/audio-cpp/CMakeLists.txt @@ -131,6 +131,7 @@ add_executable(${TARGET} transcript_assembly.cpp result_map.cpp stem_selection.cpp + generation_request.cpp inference_lane.cpp ) @@ -211,6 +212,25 @@ if(AUDIO_CPP_GRPC_BUILD_TESTS) target_compile_options(result_map_ctest PRIVATE -Wall -Wextra -Wpedantic) add_test(NAME result_map COMMAND result_map_ctest) + # Same shape as result_map_ctest: generation_request touches only the plain + # structs in engine/framework/runtime/session.h plus the generated protobuf + # messages, so the headers are all it needs and no engine_runtime is linked. + add_executable(generation_request_ctest + generation_request_ctest.cpp + generation_request.cpp) + target_include_directories(generation_request_ctest PRIVATE + "${AUDIO_CPP_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}" + # session.h reaches ggml.h through core/backend.h, and this target links + # no ggml, so it has to name the include directory itself. + "${AUDIO_CPP_DIR}/external/ggml/include") + target_link_libraries(generation_request_ctest PRIVATE + hw_grpc_proto + protobuf::libprotobuf + Threads::Threads) + target_compile_options(generation_request_ctest PRIVATE -Wall -Wextra -Wpedantic) + add_test(NAME generation_request COMMAND generation_request_ctest) + add_executable(audio_io_ctest audio_io_ctest.cpp audio_io.cpp) diff --git a/backend/cpp/audio-cpp/generation_request.cpp b/backend/cpp/audio-cpp/generation_request.cpp new file mode 100644 index 000000000..6dbc7489e --- /dev/null +++ b/backend/cpp/audio-cpp/generation_request.cpp @@ -0,0 +1,227 @@ +#include "generation_request.h" + +#include +#include +#include +#include + +namespace audiocpp_backend { +namespace { + +const char *bool_option(bool value) { return value ? "true" : "false"; } + +} // namespace + +bool voice_is_reference_file(const std::string &voice) { + if (voice.empty()) { + return false; + } + std::error_code ec; + return std::filesystem::is_regular_file(std::filesystem::path(voice), ec); +} + +engine::runtime::TaskRequest +build_tts_request(const backend::TTSRequest &request, + std::optional reference_audio) { + engine::runtime::TaskRequest task; + + // The Transcript, not an option, is where every TTS family reads its + // language: chatterbox normalises request.text_input->language into its + // voice-clone config, qwen3_tts reads it as out.language, ace_step turns it + // into vocal_language. Upstream's own HTTP server does the same and sets no + // language option at all (app/server/runtime.cpp build_speech_request). + engine::runtime::Transcript transcript; + transcript.text = request.text(); + if (request.has_language()) { + transcript.language = request.language(); + } + task.text_input = std::move(transcript); + + engine::runtime::VoiceCondition condition; + bool condition_used = false; + + if (reference_audio.has_value()) { + // A clip: VoiceReference::audio, at the file's own rate and channel + // count. See kVoiceReferenceSampleRate in grpc-server.cpp for why it is + // not folded first. + engine::runtime::VoiceReference reference; + reference.audio = std::move(*reference_audio); + condition.speaker = std::move(reference); + condition_used = true; + } else if (!request.voice().empty()) { + // A named preset. cached_voice_id is the channel that actually lands: + // supertonic (options.voice), pocket_tts (voice_config.preset_name), + // voxcpm2, vibevoice, fish_audio (its saved-reference lookup) and + // qwen3_tts CustomVoice all read request.voice->speaker->cached_voice_id, + // and upstream's own server puts a non-preset `voice` body field in + // exactly this slot (app/server/runtime.cpp build_speech_request). + engine::runtime::VoiceReference reference; + reference.cached_voice_id = request.voice(); + condition.speaker = std::move(reference); + condition_used = true; + // Forward-tolerant alias only. A bare "voice" REQUEST OPTION is read by + // no family in the pinned upstream: grepping find_option for it returns + // nothing. It is sent so a family adopting the name later works with no + // change here, not because it does anything today. + task.options["voice"] = request.voice(); + } + + if (request.has_instructions() && !request.instructions().empty()) { + // "instruct" is the key upstream itself maps the OpenAI `instructions` + // body field onto (app/server/runtime.cpp: request.options["instruct"] + // = value->as_string()), and it is read: qwen3_tts VoiceDesign and + // CustomVoice both take find_option(options, {"instruct"}) first, and + // omnivoice reads it in resolve_instruct. + task.options["instruct"] = request.instructions(); + // "caption" is irodori_tts's name for the same thing, read in its + // make_request and documented as the voice-design caption for the 600M + // VoiceDesign model (docs/tts.md). Without this, that family's voice + // design cannot be driven from this RPC at all. + task.options["caption"] = request.instructions(); + // The proto field's own name, forwarded for the same forward-tolerant + // reason as "voice" above and with the same honest accounting: NO family + // in the pinned upstream reads a request option called "instructions". + task.options["instructions"] = request.instructions(); + + engine::runtime::StyleCondition style; + // "instruct", not "instructions". This tag IS read, and only under that + // spelling: omnivoice and qwen3_tts both fall back to + // request.voice->style->tags.find("instruct") when the option is absent. + // Spelling it "instructions" here would have made the whole + // StyleCondition dead weight. + style.tags["instruct"] = request.instructions(); + if (request.has_language()) { + style.language = request.language(); + } + condition.style = std::move(style); + condition_used = true; + } + + if (condition_used) { + task.voice = std::move(condition); + } + + if (request.has_language() && !request.language().empty()) { + // Forward-tolerant alias, exactly as in build_transcription_request. The + // families that read a "language" request option are the ASR ones + // (nemotron_asr, hviske_asr, vibevoice_asr, higgs_audio_stt), none of + // which this RPC can route to; pocket_tts reads one but from its + // ModelLoadRequest at load time, not from here. The Transcript above is + // what actually carries the language to a TTS family. + task.options["language"] = request.language(); + } + + // LAST, so an explicit params entry wins over anything derived above. That + // matters for "caption": a caller who sets params[caption] has named the + // exact string they want, and it must not be overwritten by `instructions`. + for (const auto ¶m : request.params()) { + task.options[param.first] = param.second; + } + return task; +} + +engine::runtime::TaskRequest +build_sound_generation_request(const backend::SoundGenerationRequest &request, + std::optional source_audio) { + engine::runtime::TaskRequest task; + + engine::runtime::Transcript transcript; + transcript.text = request.text(); + if (request.has_language()) { + transcript.language = request.language(); + } + task.text_input = std::move(transcript); + + // src is the input clip for the editing routes. ace_step's repaint, cover + // and edit routes need it; stable_audio uses it as init_audio or + // inpaint_audio; heartmula refuses it outright. + if (source_audio.has_value()) { + task.audio_input = std::move(*source_audio); + } + + // WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in + // the pinned upstream: ace_step, heartmula and stable_audio. Every key below + // was grepped against find_option/parse_*_option in src/ and include/ rather + // than assumed, because a key nobody reads is not a feature and shipping one + // while implying it works is the mistake this comment exists to prevent. + // + // Unknown REQUEST options cannot turn a valid request into an error: + // families look theirs up by name and ignore the rest, and the unknown-key + // refusals upstream does have are on SESSION options, which arrive at load + // time. So a forward-tolerant alias is free; it is just not a feature. + + if (request.has_duration()) { + // duration_seconds is the key that works, and it works everywhere: + // ace_step (request_parser.cpp), heartmula (session.cpp, which also + // refuses a non-positive value) and stable_audio (request.cpp) all read + // it. This is the SoundGeneration analogue of Task 9's return_timestamps. + task.options["duration_seconds"] = std::to_string(request.duration()); + // The proto field's own name. Read by exactly one family, omnivoice, and + // omnivoice advertises Tts rather than AudioGeneration, so this RPC can + // never route to it: DEAD here, kept only as a forward-tolerant alias. + task.options["duration"] = std::to_string(request.duration()); + } + if (request.has_temperature()) { + // Read by heartmula. ace_step's sampling temperature is a different, + // narrower knob it calls lm_temperature (it drives the caption/thinking + // LM, not the audio diffusion), so this is deliberately NOT mapped onto + // it; a caller who wants it sets it through the request options that + // reach ace_step by name. stable_audio has no temperature at all. + task.options["temperature"] = std::to_string(request.temperature()); + } + if (request.has_sample()) { + // do_sample is read widely upstream, but only by TTS and ASR families + // (chatterbox, index_tts2, miotts, moss, qwen3_tts, vibevoice, + // hviske_asr, voxtral_realtime). NO AudioGeneration family reads it, so + // it is dead on this route. + task.options["do_sample"] = bool_option(request.sample()); + } + if (request.has_src_divisor()) { + // Read by nobody, anywhere in the pinned upstream. Forwarded because the + // proto documents it as part of this request and a family adopting it + // then works unchanged. + task.options["src_divisor"] = std::to_string(request.src_divisor()); + } + if (request.has_think()) { + // "thinking" is the key ace_step actually reads (request_parser.cpp), + // which is why it is sent alongside the proto's own "think". "think" on + // its own is read by nobody. + task.options["thinking"] = bool_option(request.think()); + task.options["think"] = bool_option(request.think()); + } + if (request.has_caption()) { + // Read only by irodori_tts, which advertises Tts/VoiceCloning/VoiceDesign + // and not AudioGeneration, so it is unreachable from this RPC: DEAD here. + task.options["caption"] = request.caption(); + } + if (request.has_lyrics()) { + // Read by ace_step and heartmula. + task.options["lyrics"] = request.lyrics(); + } + if (request.has_bpm()) { + // Read by ace_step. + task.options["bpm"] = std::to_string(request.bpm()); + } + if (request.has_keyscale()) { + // Read by ace_step. + task.options["keyscale"] = request.keyscale(); + } + if (request.has_timesignature()) { + // Read by ace_step. + task.options["timesignature"] = request.timesignature(); + } + if (request.has_instrumental()) { + // Read by nobody: "instrumental" appears in the pinned upstream only as + // a roformer STEM NAME, never as a request option. A forward-tolerant + // alias and nothing more. + task.options["instrumental"] = bool_option(request.instrumental()); + } + if (request.has_language() && !request.language().empty()) { + // Alias again: the Transcript above is what ace_step reads as + // vocal_language. No AudioGeneration family reads a "language" option. + task.options["language"] = request.language(); + } + return task; +} + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/generation_request.h b/backend/cpp/audio-cpp/generation_request.h new file mode 100644 index 000000000..b98891994 --- /dev/null +++ b/backend/cpp/audio-cpp/generation_request.h @@ -0,0 +1,65 @@ +#pragma once + +// Builds the engine::runtime::TaskRequest for the two audio-PRODUCING offline +// RPCs, TTS and SoundGeneration, and answers the one filesystem question TTS +// routing depends on. +// +// It is a unit of its own rather than a pair of statics in grpc-server.cpp so +// that it can be tested: grpc-server.cpp has a main() and cannot be linked into +// a test binary, and everything here is a pure function of its arguments once +// the file read has been lifted out (which is why the reference clip arrives as +// an already-read buffer rather than a path). TTSStream reuses build_tts_request +// unchanged. +// +// Only the plain structs in engine/framework/runtime/session.h are touched, so +// this compiles against the header without linking engine_runtime, the same way +// result_map does. + +#include "backend.pb.h" + +#include "engine/framework/runtime/session.h" + +#include +#include + +namespace audiocpp_backend { + +// True when TTSRequest.voice names an existing regular file, in which case it +// is a speaker reference clip and routing prefers VoiceCloning; false when it is +// a named preset (or empty). +// +// The overload is LocalAI's, not this backend's: `voice` is the OpenAI speech +// field and different LocalAI backends have always read it both ways. Deciding +// it from the filesystem needs no new option and matches how somebody actually +// configures a cloning family, which is by pointing at a clip. +// +// A DIRECTORY is deliberately not a reference: is_regular_file, not exists. A +// directory named as a voice cannot be read as a WAV, and treating it as a +// reference would turn a preset typo into "cannot read /x as WAV" instead of +// letting it travel as the preset name it looks like. +// +// The error_code overload is used so an unreadable parent directory answers +// false rather than throwing. That is the right answer here: the name is then +// passed on as a preset, and if it really was meant to be a clip the family +// refuses a request it cannot serve, which is a better message than a +// filesystem exception thrown while classifying a string. +bool voice_is_reference_file(const std::string &voice); + +// `reference_audio` is the already-read speaker clip, present exactly when +// voice_is_reference_file(request.voice()) was true. Passing it in rather than a +// path keeps this function pure and lets the caller do the read where the +// ordering rules (capability refusal first, then the lane) are enforced. +// +// It is taken BY VALUE and moved in: a reference clip is seconds of audio and +// the caller has no use for it afterwards. +engine::runtime::TaskRequest +build_tts_request(const backend::TTSRequest &request, + std::optional reference_audio); + +// `source_audio` is SoundGenerationRequest.src already read, present exactly +// when the field was set and non-empty. Same reasoning as above. +engine::runtime::TaskRequest +build_sound_generation_request(const backend::SoundGenerationRequest &request, + std::optional source_audio); + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/generation_request_ctest.cpp b/backend/cpp/audio-cpp/generation_request_ctest.cpp new file mode 100644 index 000000000..86c35525d --- /dev/null +++ b/backend/cpp/audio-cpp/generation_request_ctest.cpp @@ -0,0 +1,322 @@ +// Tests for the TTS and SoundGeneration request builders, and for the +// filesystem rule that decides whether TTSRequest.voice is a speaker reference +// clip or a named preset. +// +// NAMED _ctest AND NOT _test ON PURPOSE: see the note at the top of +// result_map_ctest.cpp. This file needs the generated protobuf messages and the +// audio.cpp include path, neither of which backend/cpp/run-unit-tests.sh +// provides, so it is built and run by ctest: +// +// make -C backend/cpp/audio-cpp test-engine +// +// The assertions on OPTION KEYS are the point of this file, not decoration. +// Every one of them names a key that was grepped against the pinned upstream: +// "instruct" is read, "instructions" is not; "duration_seconds" is read, +// "duration" is not. A rename that looks harmless is exactly the change that +// silently stops a family honouring the request, so the spellings are pinned +// here rather than left to a comment. + +#include "generation_request.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const std::string &name) { + if (!ok) { + failures++; + fprintf(stderr, "FAIL: %s\n", name.c_str()); + } else { + fprintf(stderr, "ok: %s\n", name.c_str()); + } +} + +using namespace audiocpp_backend; + +static bool has_key(const std::unordered_map &options, + const std::string &key) { + return options.find(key) != options.end(); +} + +static std::string option_or( + const std::unordered_map &options, + const std::string &key, const std::string &fallback) { + const auto it = options.find(key); + return it == options.end() ? fallback : it->second; +} + +static engine::runtime::AudioBuffer clip(int sample_rate, int channels) { + engine::runtime::AudioBuffer buffer; + buffer.sample_rate = sample_rate; + buffer.channels = channels; + // Distinguishable content, so a builder that swapped one buffer for another + // (or default-constructed one) is visible rather than merely a size change. + buffer.samples = {0.25f, -0.5f, 0.75f, -1.0f}; + return buffer; +} + +static void test_voice_is_reference_file() { + const auto dir = std::filesystem::temp_directory_path() / + "audiocpp-generation-request-ctest"; + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + + const auto file = dir / "reference.wav"; + { + std::ofstream out(file, std::ios::binary); + out << "not really a wav, but a regular file"; + } + const auto subdir = dir / "a-directory"; + std::filesystem::create_directories(subdir); + + check(!voice_is_reference_file(""), "voice_is_reference_file: empty"); + check(!voice_is_reference_file("alloy"), + "voice_is_reference_file: bare preset name"); + check(!voice_is_reference_file((dir / "absent.wav").string()), + "voice_is_reference_file: missing path"); + check(voice_is_reference_file(file.string()), + "voice_is_reference_file: existing regular file"); + // A directory is NOT a reference. exists() would say yes here and the read + // would then fail with "cannot read as WAV", which sends the operator + // after a file problem instead of a preset typo. + check(!voice_is_reference_file(subdir.string()), + "voice_is_reference_file: directory is not a reference"); + + std::filesystem::remove_all(dir); +} + +static void test_tts_plain() { + backend::TTSRequest request; + request.set_text("hello there"); + + const auto task = build_tts_request(request, std::nullopt); + + check(task.text_input.has_value() && task.text_input->text == "hello there", + "tts: text reaches the transcript"); + // No voice and no instructions means NO voice condition at all. A builder + // that always emitted one would make every family think a speaker was + // named, and chatterbox in particular refuses a prepare whose voice + // condition carries neither audio nor anything else it can use. + check(!task.voice.has_value(), "tts: no voice condition when nothing is set"); + check(task.options.empty(), "tts: no options when nothing is set"); + check(!task.audio_input.has_value(), "tts: no audio input"); +} + +static void test_tts_named_preset() { + backend::TTSRequest request; + request.set_text("hello"); + request.set_voice("alloy"); + + const auto task = build_tts_request(request, std::nullopt); + + check(task.voice.has_value() && task.voice->speaker.has_value(), + "tts preset: speaker condition present"); + check(task.voice->speaker->cached_voice_id.has_value() && + *task.voice->speaker->cached_voice_id == "alloy", + "tts preset: lands in cached_voice_id"); + // The clip slot must stay empty, or a cloning family would try to prepare + // conditionals from a default-constructed buffer. + check(!task.voice->speaker->audio.has_value(), + "tts preset: no reference audio"); + check(!task.voice->style.has_value(), "tts preset: no style condition"); + check(option_or(task.options, "voice", "") == "alloy", + "tts preset: forwarded as the voice option too"); +} + +static void test_tts_reference_clip() { + backend::TTSRequest request; + request.set_text("hello"); + request.set_voice("/tmp/reference.wav"); + + const auto task = build_tts_request(request, clip(44100, 2)); + + check(task.voice.has_value() && task.voice->speaker.has_value(), + "tts clip: speaker condition present"); + check(task.voice->speaker->audio.has_value(), + "tts clip: reference audio present"); + // Rate and channels survive untouched. This is the assertion that fails if + // anybody decides to fold the clip to 16 kHz mono on the way in. + check(task.voice->speaker->audio->sample_rate == 44100 && + task.voice->speaker->audio->channels == 2 && + task.voice->speaker->audio->samples.size() == 4, + "tts clip: rate, channels and samples pass through unchanged"); + // A clip is NOT also a cached voice id, and the path must not travel as a + // preset name: a family reading cached_voice_id would then look up a voice + // called "/tmp/reference.wav". + check(!task.voice->speaker->cached_voice_id.has_value(), + "tts clip: no cached_voice_id"); + check(!has_key(task.options, "voice"), "tts clip: no voice option"); +} + +static void test_tts_instructions() { + backend::TTSRequest request; + request.set_text("hello"); + request.set_instructions("a calm older man, speaking slowly"); + + const auto task = build_tts_request(request, std::nullopt); + + check(option_or(task.options, "instruct", "") == + "a calm older man, speaking slowly", + "tts instructions: instruct option is the one qwen3_tts reads"); + check(option_or(task.options, "caption", "") == + "a calm older man, speaking slowly", + "tts instructions: caption option is the one irodori_tts reads"); + check(option_or(task.options, "instructions", "") == + "a calm older man, speaking slowly", + "tts instructions: proto field name forwarded as an alias"); + check(task.voice.has_value() && task.voice->style.has_value(), + "tts instructions: style condition present"); + // "instruct", not "instructions". omnivoice and qwen3_tts both look this tag + // up by that exact name and by no other. + check(option_or(task.voice->style->tags, "instruct", "") == + "a calm older man, speaking slowly", + "tts instructions: style tag is spelled instruct"); + check(!has_key(task.voice->style->tags, "instructions"), + "tts instructions: style tag is NOT spelled instructions"); + // Instructions alone must not invent a speaker: has_voice_reference is what + // routing keys VoiceCloning off, and a speaker here would make a + // clone-capable family expect a clip it never received. + check(!task.voice->speaker.has_value(), + "tts instructions: no speaker without a voice"); +} + +static void test_tts_empty_instructions_are_not_instructions() { + backend::TTSRequest request; + request.set_text("hello"); + request.set_instructions(""); + + const auto task = build_tts_request(request, std::nullopt); + + // has_instructions() is true here, because the field was set. An empty + // string is still no instruction, and forwarding it would set an empty + // instruct option that qwen3_tts would prefer over its style tag fallback. + check(!has_key(task.options, "instruct"), + "tts: an empty instructions string sets no instruct option"); + check(!task.voice.has_value(), + "tts: an empty instructions string sets no voice condition"); +} + +static void test_tts_language_and_params() { + backend::TTSRequest request; + request.set_text("ciao"); + request.set_language("it"); + request.set_instructions("warm"); + (*request.mutable_params())["exaggeration"] = "0.7"; + // An explicit param must win over the value derived from instructions. + (*request.mutable_params())["caption"] = "explicitly chosen caption"; + + const auto task = build_tts_request(request, std::nullopt); + + check(task.text_input.has_value() && task.text_input->language == "it", + "tts language: reaches the transcript"); + check(option_or(task.options, "language", "") == "it", + "tts language: forwarded as an option alias"); + check(task.voice.has_value() && task.voice->style.has_value() && + task.voice->style->language.has_value() && + *task.voice->style->language == "it", + "tts language: reaches the style condition"); + check(option_or(task.options, "exaggeration", "") == "0.7", + "tts params: passed through verbatim"); + check(option_or(task.options, "caption", "") == "explicitly chosen caption", + "tts params: an explicit param overrides the derived caption"); +} + +static void test_sound_generation_minimal() { + backend::SoundGenerationRequest request; + request.set_text("a distant thunderstorm"); + + const auto task = build_sound_generation_request(request, std::nullopt); + + check(task.text_input.has_value() && + task.text_input->text == "a distant thunderstorm", + "sound: text reaches the transcript"); + // Unset optionals must emit NOTHING. Emitting a zero for an unset duration + // would make heartmula refuse the request ("duration_seconds must be + // positive") on a request that never mentioned a duration. + check(task.options.empty(), "sound: unset optionals emit no options"); + check(!task.audio_input.has_value(), "sound: no audio input without src"); + check(!task.voice.has_value(), "sound: no voice condition"); +} + +static void test_sound_generation_full() { + backend::SoundGenerationRequest request; + request.set_text("a slow blues in E"); + request.set_duration(30.0f); + request.set_temperature(0.8f); + request.set_sample(false); + request.set_src_divisor(4); + request.set_think(true); + request.set_caption("smoky bar recording"); + request.set_lyrics("first line\nsecond line"); + request.set_bpm(72); + request.set_keyscale("E minor"); + request.set_language("en"); + request.set_timesignature("4/4"); + request.set_instrumental(true); + + const auto task = build_sound_generation_request(request, clip(48000, 2)); + + // duration_seconds is the key every AudioGeneration family actually reads; + // "duration" rides along as an alias. Both spellings are pinned so a + // "cleanup" that keeps only the proto's own name is a test failure and not + // a silent loss of the duration. + check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0, + "sound: duration lands as duration_seconds"); + check(option_or(task.options, "duration", "").rfind("30.", 0) == 0, + "sound: duration also forwarded under its own name"); + check(option_or(task.options, "temperature", "").rfind("0.8", 0) == 0, + "sound: temperature forwarded"); + // Set to FALSE, so this also proves the key is written whenever the field is + // present rather than only when the value is truthy. + check(option_or(task.options, "do_sample", "") == "false", + "sound: sample=false is forwarded as do_sample=false"); + check(option_or(task.options, "src_divisor", "") == "4", + "sound: src_divisor forwarded"); + check(option_or(task.options, "thinking", "") == "true", + "sound: think lands as thinking, the key ace_step reads"); + check(option_or(task.options, "think", "") == "true", + "sound: think also forwarded under its own name"); + check(option_or(task.options, "caption", "") == "smoky bar recording", + "sound: caption forwarded"); + check(option_or(task.options, "lyrics", "") == "first line\nsecond line", + "sound: lyrics forwarded"); + check(option_or(task.options, "bpm", "") == "72", "sound: bpm forwarded"); + check(option_or(task.options, "keyscale", "") == "E minor", + "sound: keyscale forwarded"); + check(option_or(task.options, "timesignature", "") == "4/4", + "sound: timesignature forwarded"); + check(option_or(task.options, "instrumental", "") == "true", + "sound: instrumental forwarded"); + check(option_or(task.options, "language", "") == "en", + "sound: language forwarded as an option alias"); + check(task.text_input->language == "en", + "sound: language reaches the transcript, which is what ace_step reads"); + check(task.audio_input.has_value() && + task.audio_input->sample_rate == 48000 && + task.audio_input->channels == 2 && + task.audio_input->samples.size() == 4, + "sound: src passes through at its own rate and channel count"); +} + +int main() { + test_voice_is_reference_file(); + test_tts_plain(); + test_tts_named_preset(); + test_tts_reference_clip(); + test_tts_instructions(); + test_tts_empty_instructions_are_not_instructions(); + test_tts_language_and_params(); + test_sound_generation_minimal(); + test_sound_generation_full(); + + if (failures != 0) { + fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + fprintf(stderr, "all checks passed\n"); + return 0; +} diff --git a/backend/cpp/audio-cpp/grpc-server.cpp b/backend/cpp/audio-cpp/grpc-server.cpp index 57280cad3..027492222 100644 --- a/backend/cpp/audio-cpp/grpc-server.cpp +++ b/backend/cpp/audio-cpp/grpc-server.cpp @@ -6,8 +6,8 @@ // upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT. // // This commit adds LoadModel/Free/Status plus the AudioTranscription, VAD, -// Diarize and AudioTransform RPCs. The remaining audio RPCs land in later -// commits. +// Diarize, AudioTransform, TTS and SoundGeneration RPCs. The remaining audio +// RPCs land in later commits. #include "backend.pb.h" #include "backend.grpc.pb.h" @@ -15,6 +15,7 @@ #include "audio_io.h" #include "audio_units.h" #include "capability_routing.h" +#include "generation_request.h" #include "inference_lane.h" #include "loaded_model.h" #include "model_options.h" @@ -36,6 +37,7 @@ #include #include #include +#include #include #include #include @@ -178,6 +180,48 @@ constexpr int kSpeechSampleRate = 16000; // ordinary music file at its own rate. constexpr int kTransformSampleRate = 0; +// The rate TTS reads its speaker reference clip at, and the rate +// SoundGeneration reads its `src` editing clip at. Zero, i.e. the file's own +// rate and its own channel count, no resample and no downmix. +// +// Settled from upstream rather than reasoned about, and upstream settles it +// twice over: +// +// 1. Upstream does exactly this itself. Both its CLI and its HTTP server load +// a voice reference with minitts::cli::read_audio_buffer (app/cli/request.cpp), +// which is read_wav_f32 and then {wav.sample_rate, wav.channels, +// wav.samples} verbatim. app/server/runtime.cpp's build_speech_request +// puts that buffer straight into voice.speaker->audio, and its +// audio_input path (line 518) does the same for the editing clip. Every +// family that consumes a reference has therefore only ever been tested +// against native-rate, native-channel input. +// 2. Every consuming family converts it itself, and most of them with a +// BETTER resampler than read_audio_file's. Checked one at a time: +// chatterbox to_mono_audio, then 24 kHz, then 16 kHz derived from +// the 24 kHz path (conditionals.cpp), matching Python's +// prepare_conditionals ordering. +// index_tts2 mixdown_interleaved_to_mono_average, then a soxr or +// torchaudio sinc-hann resample to its mel rate and to +// 16 kHz (audio_features.cpp, waveform_22k). +// higgs_audio_tts mixdown, then sinc-hann to 24 kHz and 16 kHz. +// irodori_tts mixdown, then sinc-hann (codec.cpp). +// voxcpm2 mixdown, then soxr-or-linear (audiovae.cpp). +// omnivoice mixdown, then sinc-hann (audio_tokenizer.cpp). +// qwen3_tts convert_interleaved_audio_to_mono_linear_resampled. +// For the SoundGeneration side, ace_step resamples the LEFT and RIGHT +// channels SEPARATELY (pre_dit.cpp), and stable_audio resamples per +// channel too, so a mono downmix here would destroy input those two are +// built to consume as stereo. +// +// So folding to 16 kHz mono first would band-limit at 8 kHz through +// read_audio_file's LINEAR, unfiltered resampler, alias what is above it, and +// then hand that damaged signal to a good resampler for a second conversion. +// One conversion is the floor, and passing the file through unchanged is what +// keeps it at one. Same argument as kTransformSampleRate, same value, kept as a +// separate constant because the routes are different and a future per-model +// preferred-rate lookup would have to answer them separately. +constexpr int kVoiceReferenceSampleRate = 0; + // Parses ModelOptions.MainGPU into a device index. // // Not std::atoi: it returns 0 for anything unparseable, so "gpu1" or a device @@ -953,6 +997,166 @@ public: return to_status(err); } } + + // The first RPC that GENERATES audio from text, together with + // SoundGeneration below. AudioTransform already wrote files, but it + // transformed audio it was given; these two have no required input audio at + // all, which is why both carry a Result with a success flag rather than a + // response message describing what was found. + // + // TTSRequest.voice is overloaded across LocalAI backends, some reading it as + // a named preset and some as a path to a clip. The rule here is decided from + // the filesystem: an existing regular file is a speaker reference and + // routing prefers VoiceCloning, anything else is a preset. Instructions with + // no clip prefer VoiceDesign, and a clip outranks instructions when both are + // set. None of that is re-derived here; it is capability_routing's + // task_candidates, and this handler's job is only to describe the request + // truthfully in the RequestShape. + GStatus TTS(ServerContext *, const backend::TTSRequest *request, + backend::Result *result) override { + try { + GStatus refusal = GStatus::OK; + const auto model = snapshot_for(request, refusal); + if (model == nullptr) { + // Answered with the STATUS ALONE, leaving Result at its default. + // core/backend/tts.go checks the transport error before it looks + // at res.Success and returns on it, so the Result is never read + // on this path; and the router matches a stale route on the + // NOT_FOUND code plus the sentinel in the status message, which + // is where check_model_identity already put it. + return refusal; + } + + const bool voice_is_file = + audiocpp_backend::voice_is_reference_file(request->voice()); + audiocpp_backend::RequestShape shape; + shape.has_voice_reference = voice_is_file; + shape.has_instructions = + request->has_instructions() && !request->instructions().empty(); + shape.pinned_task = model->pinned_task(); + + // FIRST, before the lane and before any file read. A family that + // cannot synthesise at all is answering a question about itself, so + // it must not queue behind somebody else's run to be told no, and it + // must not blame the caller's reference clip for a decision that had + // nothing to do with it. Same ordering as Diarize and + // AudioTransform. + model->check_can_serve(audiocpp_backend::Rpc::Tts, shape); + + if (request->dst().empty()) { + throw audiocpp_backend::ConfigError( + "audio-cpp: TTS needs a dst output path"); + } + + std::optional reference; + if (voice_is_file) { + // Native rate, native channels: see kVoiceReferenceSampleRate. + reference = audiocpp_backend::read_audio_file( + request->voice(), kVoiceReferenceSampleRate); + } + // Read before the lane, like every other file-fed handler here, so + // a slow or large clip is not decoded while holding it. + const auto task = + audiocpp_backend::build_tts_request(*request, std::move(reference)); + + // Named local: LaneEntry is immovable and has to span both + // session_for and run_offline. + audiocpp_backend::LaneEntry lane = model->acquire(0); + const auto session = + model->session_for(audiocpp_backend::Rpc::Tts, shape, lane); + const auto task_result = + audiocpp_backend::run_offline(session, task, lane); + + if (!task_result.audio_output.has_value()) { + // A family that routed successfully and then produced no audio + // is a broken family rather than a wrong request, but from the + // caller's side the actionable fact is that this model does not + // do this, so CapabilityError. Same judgement as AudioTransform. + throw audiocpp_backend::CapabilityError( + "audio-cpp: family '" + model->family() + + "' produced no audio for the TTS RPC"); + } + // Throws a plain runtime_error, i.e. INTERNAL, on a write failure: + // dst is LocalAI's own generated-content path and not anything the + // caller named, so a full disk there is a server fault and is worth + // retrying. Only an empty dst is INVALID_ARGUMENT, and that is + // already refused above with a message naming the RPC. + audiocpp_backend::write_audio_file(request->dst(), + *task_result.audio_output); + result->set_success(true); + // The path, because that is what core/backend/tts.go's caller reads + // back and what every other LocalAI TTS backend puts here. + result->set_message(request->dst()); + return GStatus::OK; + } catch (const std::exception &err) { + // Both channels, unlike the transcription-shaped handlers: Result + // has a success flag that the Go side checks in addition to the + // status, so leaving it default-false with no message would report a + // failure with no reason attached. + result->set_success(false); + result->set_message(err.what()); + return to_status(err); + } + } + + GStatus SoundGeneration(ServerContext *, + const backend::SoundGenerationRequest *request, + backend::Result *result) override { + try { + GStatus refusal = GStatus::OK; + const auto model = snapshot_for(request, refusal); + if (model == nullptr) { + return refusal; + } + + audiocpp_backend::RequestShape shape; + // No request signal chooses between generation tasks: this RPC has + // exactly one candidate, AudioGeneration. pinned_task is still + // copied, because without it the model's `task:` option is dead + // here, which is how a gen family pinned to something else would + // silently be routed to generation anyway. + shape.pinned_task = model->pinned_task(); + + model->check_can_serve(audiocpp_backend::Rpc::SoundGeneration, shape); + + if (request->dst().empty()) { + throw audiocpp_backend::ConfigError( + "audio-cpp: SoundGeneration needs a dst output path"); + } + + std::optional source; + if (request->has_src() && !request->src().empty()) { + // Native rate and channels. ace_step and stable_audio both + // resample per channel, so a downmix here would delete the + // stereo image of the very clip they are editing. + source = audiocpp_backend::read_audio_file( + request->src(), kVoiceReferenceSampleRate); + } + const auto task = audiocpp_backend::build_sound_generation_request( + *request, std::move(source)); + + audiocpp_backend::LaneEntry lane = model->acquire(0); + const auto session = model->session_for( + audiocpp_backend::Rpc::SoundGeneration, shape, lane); + const auto task_result = + audiocpp_backend::run_offline(session, task, lane); + + if (!task_result.audio_output.has_value()) { + throw audiocpp_backend::CapabilityError( + "audio-cpp: family '" + model->family() + + "' produced no audio for the SoundGeneration RPC"); + } + audiocpp_backend::write_audio_file(request->dst(), + *task_result.audio_output); + result->set_success(true); + result->set_message(request->dst()); + return GStatus::OK; + } catch (const std::exception &err) { + result->set_success(false); + result->set_message(err.what()); + return to_status(err); + } + } }; void RunServer(const std::string &addr) {