mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-31 02:18:50 -04:00
Code Issues Packages Projects Releases Wiki Activity

backend(audio-cpp): serve the TTS and SoundGeneration RPCs

Browse Source 
TTSRequest.voice is treated as a speaker reference clip when it names an
existing regular file, which makes routing prefer VoiceCloning, and as a named
preset otherwise, in which case it travels as VoiceReference::cached_voice_id.
Both the clip and SoundGenerationRequest.src are read at the file's own rate and
channel count: upstream's own CLI and server do exactly that, every consuming
family resamples internally and mostly with a better resampler than ours, and
ace_step and stable_audio resample their input per channel, so a downmix here
would delete the stereo image they are built to consume.

The request builders live in their own unit rather than in grpc-server.cpp's
anonymous namespace so they can be tested; grpc-server.cpp has a main() and
cannot be linked into a test binary. The option keys are the whole point of
these functions, so each one was grepped against the pinned upstream and the
accounting is written down beside it. instructions maps to "instruct", which is
what upstream's own server maps the OpenAI field to and what qwen3_tts and
omnivoice read, and to "caption" for irodori_tts; the style tag is spelled
"instruct" too, because "instructions" is looked up nowhere. duration maps to
"duration_seconds", read by all three generation families, with the proto's own
name kept only as a forward-tolerant alias. Keys that no family reads say so.

Both handlers answer a capability refusal before taking the lane and before any
file read, so a model that cannot synthesise does not queue behind somebody
else's run to be told no.

Assisted-by: Claude:claude-opus-5 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2026-07-26 21:25:59 +00:00
committed by localai-org-maint-bot
parent e873862893
commit d318b2ea94
5 changed files with 840 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
backend/cpp/audio-cpp/CMakeLists.txt
View File

@@ -131,6 +131,7 @@ add_executable(${TARGET}
transcript_assembly.cpp
result_map.cpp
stem_selection.cpp
generation_request.cpp
inference_lane.cpp
)
@@ -211,6 +212,25 @@ if(AUDIO_CPP_GRPC_BUILD_TESTS)
target_compile_options(result_map_ctest PRIVATE -Wall -Wextra -Wpedantic)
add_test(NAME result_map COMMAND result_map_ctest)
# Same shape as result_map_ctest: generation_request touches only the plain
# structs in engine/framework/runtime/session.h plus the generated protobuf
# messages, so the headers are all it needs and no engine_runtime is linked.
add_executable(generation_request_ctest
generation_request_ctest.cpp
generation_request.cpp)
target_include_directories(generation_request_ctest PRIVATE
"${AUDIO_CPP_DIR}/include"
"${CMAKE_CURRENT_SOURCE_DIR}"
# session.h reaches ggml.h through core/backend.h, and this target links
# no ggml, so it has to name the include directory itself.
"${AUDIO_CPP_DIR}/external/ggml/include")
target_link_libraries(generation_request_ctest PRIVATE
hw_grpc_proto
protobuf::libprotobuf
Threads::Threads)
target_compile_options(generation_request_ctest PRIVATE -Wall -Wextra -Wpedantic)
add_test(NAME generation_request COMMAND generation_request_ctest)
add_executable(audio_io_ctest
audio_io_ctest.cpp
audio_io.cpp)

227
backend/cpp/audio-cpp/generation_request.cpp Normal file
View File

@@ -0,0 +1,227 @@
#include "generation_request.h"
#include <filesystem>
#include <string>
#include <system_error>
#include <utility>
namespace audiocpp_backend {
namespace {
const char *bool_option(bool value) { return value ? "true" : "false"; }
} // namespace
bool voice_is_reference_file(const std::string &voice) {
if (voice.empty()) {
return false;
}
std::error_code ec;
return std::filesystem::is_regular_file(std::filesystem::path(voice), ec);
}
engine::runtime::TaskRequest
build_tts_request(const backend::TTSRequest &request,
std::optional<engine::runtime::AudioBuffer> reference_audio) {
engine::runtime::TaskRequest task;
// The Transcript, not an option, is where every TTS family reads its
// language: chatterbox normalises request.text_input->language into its
// voice-clone config, qwen3_tts reads it as out.language, ace_step turns it
// into vocal_language. Upstream's own HTTP server does the same and sets no
// language option at all (app/server/runtime.cpp build_speech_request).
engine::runtime::Transcript transcript;
transcript.text = request.text();
if (request.has_language()) {
transcript.language = request.language();
}
task.text_input = std::move(transcript);
engine::runtime::VoiceCondition condition;
bool condition_used = false;
if (reference_audio.has_value()) {
// A clip: VoiceReference::audio, at the file's own rate and channel
// count. See kVoiceReferenceSampleRate in grpc-server.cpp for why it is
// not folded first.
engine::runtime::VoiceReference reference;
reference.audio = std::move(*reference_audio);
condition.speaker = std::move(reference);
condition_used = true;
} else if (!request.voice().empty()) {
// A named preset. cached_voice_id is the channel that actually lands:
// supertonic (options.voice), pocket_tts (voice_config.preset_name),
// voxcpm2, vibevoice, fish_audio (its saved-reference lookup) and
// qwen3_tts CustomVoice all read request.voice->speaker->cached_voice_id,
// and upstream's own server puts a non-preset `voice` body field in
// exactly this slot (app/server/runtime.cpp build_speech_request).
engine::runtime::VoiceReference reference;
reference.cached_voice_id = request.voice();
condition.speaker = std::move(reference);
condition_used = true;
// Forward-tolerant alias only. A bare "voice" REQUEST OPTION is read by
// no family in the pinned upstream: grepping find_option for it returns
// nothing. It is sent so a family adopting the name later works with no
// change here, not because it does anything today.
task.options["voice"] = request.voice();
}
if (request.has_instructions() && !request.instructions().empty()) {
// "instruct" is the key upstream itself maps the OpenAI `instructions`
// body field onto (app/server/runtime.cpp: request.options["instruct"]
// = value->as_string()), and it is read: qwen3_tts VoiceDesign and
// CustomVoice both take find_option(options, {"instruct"}) first, and
// omnivoice reads it in resolve_instruct.
task.options["instruct"] = request.instructions();
// "caption" is irodori_tts's name for the same thing, read in its
// make_request and documented as the voice-design caption for the 600M
// VoiceDesign model (docs/tts.md). Without this, that family's voice
// design cannot be driven from this RPC at all.
task.options["caption"] = request.instructions();
// The proto field's own name, forwarded for the same forward-tolerant
// reason as "voice" above and with the same honest accounting: NO family
// in the pinned upstream reads a request option called "instructions".
task.options["instructions"] = request.instructions();
engine::runtime::StyleCondition style;
// "instruct", not "instructions". This tag IS read, and only under that
// spelling: omnivoice and qwen3_tts both fall back to
// request.voice->style->tags.find("instruct") when the option is absent.
// Spelling it "instructions" here would have made the whole
// StyleCondition dead weight.
style.tags["instruct"] = request.instructions();
if (request.has_language()) {
style.language = request.language();
}
condition.style = std::move(style);
condition_used = true;
}
if (condition_used) {
task.voice = std::move(condition);
}
if (request.has_language() && !request.language().empty()) {
// Forward-tolerant alias, exactly as in build_transcription_request. The
// families that read a "language" request option are the ASR ones
// (nemotron_asr, hviske_asr, vibevoice_asr, higgs_audio_stt), none of
// which this RPC can route to; pocket_tts reads one but from its
// ModelLoadRequest at load time, not from here. The Transcript above is
// what actually carries the language to a TTS family.
task.options["language"] = request.language();
}
// LAST, so an explicit params entry wins over anything derived above. That
// matters for "caption": a caller who sets params[caption] has named the
// exact string they want, and it must not be overwritten by `instructions`.
for (const auto &param : request.params()) {
task.options[param.first] = param.second;
}
return task;
}
engine::runtime::TaskRequest
build_sound_generation_request(const backend::SoundGenerationRequest &request,
std::optional<engine::runtime::AudioBuffer> source_audio) {
engine::runtime::TaskRequest task;
engine::runtime::Transcript transcript;
transcript.text = request.text();
if (request.has_language()) {
transcript.language = request.language();
}
task.text_input = std::move(transcript);
// src is the input clip for the editing routes. ace_step's repaint, cover
// and edit routes need it; stable_audio uses it as init_audio or
// inpaint_audio; heartmula refuses it outright.
if (source_audio.has_value()) {
task.audio_input = std::move(*source_audio);
}
// WHAT LANDS AND WHAT DOES NOT. Three families advertise AudioGeneration in
// the pinned upstream: ace_step, heartmula and stable_audio. Every key below
// was grepped against find_option/parse_*_option in src/ and include/ rather
// than assumed, because a key nobody reads is not a feature and shipping one
// while implying it works is the mistake this comment exists to prevent.
//
// Unknown REQUEST options cannot turn a valid request into an error:
// families look theirs up by name and ignore the rest, and the unknown-key
// refusals upstream does have are on SESSION options, which arrive at load
// time. So a forward-tolerant alias is free; it is just not a feature.
if (request.has_duration()) {
// duration_seconds is the key that works, and it works everywhere:
// ace_step (request_parser.cpp), heartmula (session.cpp, which also
// refuses a non-positive value) and stable_audio (request.cpp) all read
// it. This is the SoundGeneration analogue of Task 9's return_timestamps.
task.options["duration_seconds"] = std::to_string(request.duration());
// The proto field's own name. Read by exactly one family, omnivoice, and
// omnivoice advertises Tts rather than AudioGeneration, so this RPC can
// never route to it: DEAD here, kept only as a forward-tolerant alias.
task.options["duration"] = std::to_string(request.duration());
}
if (request.has_temperature()) {
// Read by heartmula. ace_step's sampling temperature is a different,
// narrower knob it calls lm_temperature (it drives the caption/thinking
// LM, not the audio diffusion), so this is deliberately NOT mapped onto
// it; a caller who wants it sets it through the request options that
// reach ace_step by name. stable_audio has no temperature at all.
task.options["temperature"] = std::to_string(request.temperature());
}
if (request.has_sample()) {
// do_sample is read widely upstream, but only by TTS and ASR families
// (chatterbox, index_tts2, miotts, moss, qwen3_tts, vibevoice,
// hviske_asr, voxtral_realtime). NO AudioGeneration family reads it, so
// it is dead on this route.
task.options["do_sample"] = bool_option(request.sample());
}
if (request.has_src_divisor()) {
// Read by nobody, anywhere in the pinned upstream. Forwarded because the
// proto documents it as part of this request and a family adopting it
// then works unchanged.
task.options["src_divisor"] = std::to_string(request.src_divisor());
}
if (request.has_think()) {
// "thinking" is the key ace_step actually reads (request_parser.cpp),
// which is why it is sent alongside the proto's own "think". "think" on
// its own is read by nobody.
task.options["thinking"] = bool_option(request.think());
task.options["think"] = bool_option(request.think());
}
if (request.has_caption()) {
// Read only by irodori_tts, which advertises Tts/VoiceCloning/VoiceDesign
// and not AudioGeneration, so it is unreachable from this RPC: DEAD here.
task.options["caption"] = request.caption();
}
if (request.has_lyrics()) {
// Read by ace_step and heartmula.
task.options["lyrics"] = request.lyrics();
}
if (request.has_bpm()) {
// Read by ace_step.
task.options["bpm"] = std::to_string(request.bpm());
}
if (request.has_keyscale()) {
// Read by ace_step.
task.options["keyscale"] = request.keyscale();
}
if (request.has_timesignature()) {
// Read by ace_step.
task.options["timesignature"] = request.timesignature();
}
if (request.has_instrumental()) {
// Read by nobody: "instrumental" appears in the pinned upstream only as
// a roformer STEM NAME, never as a request option. A forward-tolerant
// alias and nothing more.
task.options["instrumental"] = bool_option(request.instrumental());
}
if (request.has_language() && !request.language().empty()) {
// Alias again: the Transcript above is what ace_step reads as
// vocal_language. No AudioGeneration family reads a "language" option.
task.options["language"] = request.language();
}
return task;
}
} // namespace audiocpp_backend

65
backend/cpp/audio-cpp/generation_request.h Normal file
View File

@@ -0,0 +1,65 @@
#pragma once
// Builds the engine::runtime::TaskRequest for the two audio-PRODUCING offline
// RPCs, TTS and SoundGeneration, and answers the one filesystem question TTS
// routing depends on.
//
// It is a unit of its own rather than a pair of statics in grpc-server.cpp so
// that it can be tested: grpc-server.cpp has a main() and cannot be linked into
// a test binary, and everything here is a pure function of its arguments once
// the file read has been lifted out (which is why the reference clip arrives as
// an already-read buffer rather than a path). TTSStream reuses build_tts_request
// unchanged.
//
// Only the plain structs in engine/framework/runtime/session.h are touched, so
// this compiles against the header without linking engine_runtime, the same way
// result_map does.
#include "backend.pb.h"
#include "engine/framework/runtime/session.h"
#include <optional>
#include <string>
namespace audiocpp_backend {
// True when TTSRequest.voice names an existing regular file, in which case it
// is a speaker reference clip and routing prefers VoiceCloning; false when it is
// a named preset (or empty).
//
// The overload is LocalAI's, not this backend's: `voice` is the OpenAI speech
// field and different LocalAI backends have always read it both ways. Deciding
// it from the filesystem needs no new option and matches how somebody actually
// configures a cloning family, which is by pointing at a clip.
//
// A DIRECTORY is deliberately not a reference: is_regular_file, not exists. A
// directory named as a voice cannot be read as a WAV, and treating it as a
// reference would turn a preset typo into "cannot read /x as WAV" instead of
// letting it travel as the preset name it looks like.
//
// The error_code overload is used so an unreadable parent directory answers
// false rather than throwing. That is the right answer here: the name is then
// passed on as a preset, and if it really was meant to be a clip the family
// refuses a request it cannot serve, which is a better message than a
// filesystem exception thrown while classifying a string.
bool voice_is_reference_file(const std::string &voice);
// `reference_audio` is the already-read speaker clip, present exactly when
// voice_is_reference_file(request.voice()) was true. Passing it in rather than a
// path keeps this function pure and lets the caller do the read where the
// ordering rules (capability refusal first, then the lane) are enforced.
//
// It is taken BY VALUE and moved in: a reference clip is seconds of audio and
// the caller has no use for it afterwards.
engine::runtime::TaskRequest
build_tts_request(const backend::TTSRequest &request,
std::optional<engine::runtime::AudioBuffer> reference_audio);
// `source_audio` is SoundGenerationRequest.src already read, present exactly
// when the field was set and non-empty. Same reasoning as above.
engine::runtime::TaskRequest
build_sound_generation_request(const backend::SoundGenerationRequest &request,
std::optional<engine::runtime::AudioBuffer> source_audio);
} // namespace audiocpp_backend

322
backend/cpp/audio-cpp/generation_request_ctest.cpp Normal file
View File

@@ -0,0 +1,322 @@
// Tests for the TTS and SoundGeneration request builders, and for the
// filesystem rule that decides whether TTSRequest.voice is a speaker reference
// clip or a named preset.
//
// NAMED _ctest AND NOT _test ON PURPOSE: see the note at the top of
// result_map_ctest.cpp. This file needs the generated protobuf messages and the
// audio.cpp include path, neither of which backend/cpp/run-unit-tests.sh
// provides, so it is built and run by ctest:
//
// make -C backend/cpp/audio-cpp test-engine
//
// The assertions on OPTION KEYS are the point of this file, not decoration.
// Every one of them names a key that was grepped against the pinned upstream:
// "instruct" is read, "instructions" is not; "duration_seconds" is read,
// "duration" is not. A rename that looks harmless is exactly the change that
// silently stops a family honouring the request, so the spellings are pinned
// here rather than left to a comment.
#include "generation_request.h"
#include <cstdio>
#include <filesystem>
#include <fstream>
#include <string>
#include <unordered_map>
static int failures = 0;
static void check(bool ok, const std::string &name) {
if (!ok) {
failures++;
fprintf(stderr, "FAIL: %s\n", name.c_str());
} else {
fprintf(stderr, "ok: %s\n", name.c_str());
}
}
using namespace audiocpp_backend;
static bool has_key(const std::unordered_map<std::string, std::string> &options,
const std::string &key) {
return options.find(key) != options.end();
}
static std::string option_or(
const std::unordered_map<std::string, std::string> &options,
const std::string &key, const std::string &fallback) {
const auto it = options.find(key);
return it == options.end() ? fallback : it->second;
}
static engine::runtime::AudioBuffer clip(int sample_rate, int channels) {
engine::runtime::AudioBuffer buffer;
buffer.sample_rate = sample_rate;
buffer.channels = channels;
// Distinguishable content, so a builder that swapped one buffer for another
// (or default-constructed one) is visible rather than merely a size change.
buffer.samples = {0.25f, -0.5f, 0.75f, -1.0f};
return buffer;
}
static void test_voice_is_reference_file() {
const auto dir = std::filesystem::temp_directory_path() /
"audiocpp-generation-request-ctest";
std::filesystem::remove_all(dir);
std::filesystem::create_directories(dir);
const auto file = dir / "reference.wav";
{
std::ofstream out(file, std::ios::binary);
out << "not really a wav, but a regular file";
}
const auto subdir = dir / "a-directory";
std::filesystem::create_directories(subdir);
check(!voice_is_reference_file(""), "voice_is_reference_file: empty");
check(!voice_is_reference_file("alloy"),
"voice_is_reference_file: bare preset name");
check(!voice_is_reference_file((dir / "absent.wav").string()),
"voice_is_reference_file: missing path");
check(voice_is_reference_file(file.string()),
"voice_is_reference_file: existing regular file");
// A directory is NOT a reference. exists() would say yes here and the read
// would then fail with "cannot read <dir> as WAV", which sends the operator
// after a file problem instead of a preset typo.
check(!voice_is_reference_file(subdir.string()),
"voice_is_reference_file: directory is not a reference");
std::filesystem::remove_all(dir);
}
static void test_tts_plain() {
backend::TTSRequest request;
request.set_text("hello there");
const auto task = build_tts_request(request, std::nullopt);
check(task.text_input.has_value() && task.text_input->text == "hello there",
"tts: text reaches the transcript");
// No voice and no instructions means NO voice condition at all. A builder
// that always emitted one would make every family think a speaker was
// named, and chatterbox in particular refuses a prepare whose voice
// condition carries neither audio nor anything else it can use.
check(!task.voice.has_value(), "tts: no voice condition when nothing is set");
check(task.options.empty(), "tts: no options when nothing is set");
check(!task.audio_input.has_value(), "tts: no audio input");
}
static void test_tts_named_preset() {
backend::TTSRequest request;
request.set_text("hello");
request.set_voice("alloy");
const auto task = build_tts_request(request, std::nullopt);
check(task.voice.has_value() && task.voice->speaker.has_value(),
"tts preset: speaker condition present");
check(task.voice->speaker->cached_voice_id.has_value() &&
*task.voice->speaker->cached_voice_id == "alloy",
"tts preset: lands in cached_voice_id");
// The clip slot must stay empty, or a cloning family would try to prepare
// conditionals from a default-constructed buffer.
check(!task.voice->speaker->audio.has_value(),
"tts preset: no reference audio");
check(!task.voice->style.has_value(), "tts preset: no style condition");
check(option_or(task.options, "voice", "") == "alloy",
"tts preset: forwarded as the voice option too");
}
static void test_tts_reference_clip() {
backend::TTSRequest request;
request.set_text("hello");
request.set_voice("/tmp/reference.wav");
const auto task = build_tts_request(request, clip(44100, 2));
check(task.voice.has_value() && task.voice->speaker.has_value(),
"tts clip: speaker condition present");
check(task.voice->speaker->audio.has_value(),
"tts clip: reference audio present");
// Rate and channels survive untouched. This is the assertion that fails if
// anybody decides to fold the clip to 16 kHz mono on the way in.
check(task.voice->speaker->audio->sample_rate == 44100 &&
task.voice->speaker->audio->channels == 2 &&
task.voice->speaker->audio->samples.size() == 4,
"tts clip: rate, channels and samples pass through unchanged");
// A clip is NOT also a cached voice id, and the path must not travel as a
// preset name: a family reading cached_voice_id would then look up a voice
// called "/tmp/reference.wav".
check(!task.voice->speaker->cached_voice_id.has_value(),
"tts clip: no cached_voice_id");
check(!has_key(task.options, "voice"), "tts clip: no voice option");
}
static void test_tts_instructions() {
backend::TTSRequest request;
request.set_text("hello");
request.set_instructions("a calm older man, speaking slowly");
const auto task = build_tts_request(request, std::nullopt);
check(option_or(task.options, "instruct", "") ==
"a calm older man, speaking slowly",
"tts instructions: instruct option is the one qwen3_tts reads");
check(option_or(task.options, "caption", "") ==
"a calm older man, speaking slowly",
"tts instructions: caption option is the one irodori_tts reads");
check(option_or(task.options, "instructions", "") ==
"a calm older man, speaking slowly",
"tts instructions: proto field name forwarded as an alias");
check(task.voice.has_value() && task.voice->style.has_value(),
"tts instructions: style condition present");
// "instruct", not "instructions". omnivoice and qwen3_tts both look this tag
// up by that exact name and by no other.
check(option_or(task.voice->style->tags, "instruct", "") ==
"a calm older man, speaking slowly",
"tts instructions: style tag is spelled instruct");
check(!has_key(task.voice->style->tags, "instructions"),
"tts instructions: style tag is NOT spelled instructions");
// Instructions alone must not invent a speaker: has_voice_reference is what
// routing keys VoiceCloning off, and a speaker here would make a
// clone-capable family expect a clip it never received.
check(!task.voice->speaker.has_value(),
"tts instructions: no speaker without a voice");
}
static void test_tts_empty_instructions_are_not_instructions() {
backend::TTSRequest request;
request.set_text("hello");
request.set_instructions("");
const auto task = build_tts_request(request, std::nullopt);
// has_instructions() is true here, because the field was set. An empty
// string is still no instruction, and forwarding it would set an empty
// instruct option that qwen3_tts would prefer over its style tag fallback.
check(!has_key(task.options, "instruct"),
"tts: an empty instructions string sets no instruct option");
check(!task.voice.has_value(),
"tts: an empty instructions string sets no voice condition");
}
static void test_tts_language_and_params() {
backend::TTSRequest request;
request.set_text("ciao");
request.set_language("it");
request.set_instructions("warm");
(*request.mutable_params())["exaggeration"] = "0.7";
// An explicit param must win over the value derived from instructions.
(*request.mutable_params())["caption"] = "explicitly chosen caption";
const auto task = build_tts_request(request, std::nullopt);
check(task.text_input.has_value() && task.text_input->language == "it",
"tts language: reaches the transcript");
check(option_or(task.options, "language", "") == "it",
"tts language: forwarded as an option alias");
check(task.voice.has_value() && task.voice->style.has_value() &&
task.voice->style->language.has_value() &&
*task.voice->style->language == "it",
"tts language: reaches the style condition");
check(option_or(task.options, "exaggeration", "") == "0.7",
"tts params: passed through verbatim");
check(option_or(task.options, "caption", "") == "explicitly chosen caption",
"tts params: an explicit param overrides the derived caption");
}
static void test_sound_generation_minimal() {
backend::SoundGenerationRequest request;
request.set_text("a distant thunderstorm");
const auto task = build_sound_generation_request(request, std::nullopt);
check(task.text_input.has_value() &&
task.text_input->text == "a distant thunderstorm",
"sound: text reaches the transcript");
// Unset optionals must emit NOTHING. Emitting a zero for an unset duration
// would make heartmula refuse the request ("duration_seconds must be
// positive") on a request that never mentioned a duration.
check(task.options.empty(), "sound: unset optionals emit no options");
check(!task.audio_input.has_value(), "sound: no audio input without src");
check(!task.voice.has_value(), "sound: no voice condition");
}
static void test_sound_generation_full() {
backend::SoundGenerationRequest request;
request.set_text("a slow blues in E");
request.set_duration(30.0f);
request.set_temperature(0.8f);
request.set_sample(false);
request.set_src_divisor(4);
request.set_think(true);
request.set_caption("smoky bar recording");
request.set_lyrics("first line\nsecond line");
request.set_bpm(72);
request.set_keyscale("E minor");
request.set_language("en");
request.set_timesignature("4/4");
request.set_instrumental(true);
const auto task = build_sound_generation_request(request, clip(48000, 2));
// duration_seconds is the key every AudioGeneration family actually reads;
// "duration" rides along as an alias. Both spellings are pinned so a
// "cleanup" that keeps only the proto's own name is a test failure and not
// a silent loss of the duration.
check(option_or(task.options, "duration_seconds", "").rfind("30.", 0) == 0,
"sound: duration lands as duration_seconds");
check(option_or(task.options, "duration", "").rfind("30.", 0) == 0,
"sound: duration also forwarded under its own name");
check(option_or(task.options, "temperature", "").rfind("0.8", 0) == 0,
"sound: temperature forwarded");
// Set to FALSE, so this also proves the key is written whenever the field is
// present rather than only when the value is truthy.
check(option_or(task.options, "do_sample", "") == "false",
"sound: sample=false is forwarded as do_sample=false");
check(option_or(task.options, "src_divisor", "") == "4",
"sound: src_divisor forwarded");
check(option_or(task.options, "thinking", "") == "true",
"sound: think lands as thinking, the key ace_step reads");
check(option_or(task.options, "think", "") == "true",
"sound: think also forwarded under its own name");
check(option_or(task.options, "caption", "") == "smoky bar recording",
"sound: caption forwarded");
check(option_or(task.options, "lyrics", "") == "first line\nsecond line",
"sound: lyrics forwarded");
check(option_or(task.options, "bpm", "") == "72", "sound: bpm forwarded");
check(option_or(task.options, "keyscale", "") == "E minor",
"sound: keyscale forwarded");
check(option_or(task.options, "timesignature", "") == "4/4",
"sound: timesignature forwarded");
check(option_or(task.options, "instrumental", "") == "true",
"sound: instrumental forwarded");
check(option_or(task.options, "language", "") == "en",
"sound: language forwarded as an option alias");
check(task.text_input->language == "en",
"sound: language reaches the transcript, which is what ace_step reads");
check(task.audio_input.has_value() &&
task.audio_input->sample_rate == 48000 &&
task.audio_input->channels == 2 &&
task.audio_input->samples.size() == 4,
"sound: src passes through at its own rate and channel count");
}
int main() {
test_voice_is_reference_file();
test_tts_plain();
test_tts_named_preset();
test_tts_reference_clip();
test_tts_instructions();
test_tts_empty_instructions_are_not_instructions();
test_tts_language_and_params();
test_sound_generation_minimal();
test_sound_generation_full();
if (failures != 0) {
fprintf(stderr, "%d check(s) failed\n", failures);
return 1;
}
fprintf(stderr, "all checks passed\n");
return 0;
}

208
backend/cpp/audio-cpp/grpc-server.cpp
View File

@@ -6,8 +6,8 @@
// upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT.
//
// This commit adds LoadModel/Free/Status plus the AudioTranscription, VAD,
// Diarize and AudioTransform RPCs. The remaining audio RPCs land in later
// commits.
// Diarize, AudioTransform, TTS and SoundGeneration RPCs. The remaining audio
// RPCs land in later commits.
#include "backend.pb.h"
#include "backend.grpc.pb.h"
@@ -15,6 +15,7 @@
#include "audio_io.h"
#include "audio_units.h"
#include "capability_routing.h"
#include "generation_request.h"
#include "inference_lane.h"
#include "loaded_model.h"
#include "model_options.h"
@@ -36,6 +37,7 @@
#include <limits>
#include <memory>
#include <mutex>
#include <optional>
#include <set>
#include <string>
#include <thread>
@@ -178,6 +180,48 @@ constexpr int kSpeechSampleRate = 16000;
// ordinary music file at its own rate.
constexpr int kTransformSampleRate = 0;
// The rate TTS reads its speaker reference clip at, and the rate
// SoundGeneration reads its `src` editing clip at. Zero, i.e. the file's own
// rate and its own channel count, no resample and no downmix.
//
// Settled from upstream rather than reasoned about, and upstream settles it
// twice over:
//
// 1. Upstream does exactly this itself. Both its CLI and its HTTP server load
// a voice reference with minitts::cli::read_audio_buffer (app/cli/request.cpp),
// which is read_wav_f32 and then {wav.sample_rate, wav.channels,
// wav.samples} verbatim. app/server/runtime.cpp's build_speech_request
// puts that buffer straight into voice.speaker->audio, and its
// audio_input path (line 518) does the same for the editing clip. Every
// family that consumes a reference has therefore only ever been tested
// against native-rate, native-channel input.
// 2. Every consuming family converts it itself, and most of them with a
// BETTER resampler than read_audio_file's. Checked one at a time:
// chatterbox to_mono_audio, then 24 kHz, then 16 kHz derived from
// the 24 kHz path (conditionals.cpp), matching Python's
// prepare_conditionals ordering.
// index_tts2 mixdown_interleaved_to_mono_average, then a soxr or
// torchaudio sinc-hann resample to its mel rate and to
// 16 kHz (audio_features.cpp, waveform_22k).
// higgs_audio_tts mixdown, then sinc-hann to 24 kHz and 16 kHz.
// irodori_tts mixdown, then sinc-hann (codec.cpp).
// voxcpm2 mixdown, then soxr-or-linear (audiovae.cpp).
// omnivoice mixdown, then sinc-hann (audio_tokenizer.cpp).
// qwen3_tts convert_interleaved_audio_to_mono_linear_resampled.
// For the SoundGeneration side, ace_step resamples the LEFT and RIGHT
// channels SEPARATELY (pre_dit.cpp), and stable_audio resamples per
// channel too, so a mono downmix here would destroy input those two are
// built to consume as stereo.
//
// So folding to 16 kHz mono first would band-limit at 8 kHz through
// read_audio_file's LINEAR, unfiltered resampler, alias what is above it, and
// then hand that damaged signal to a good resampler for a second conversion.
// One conversion is the floor, and passing the file through unchanged is what
// keeps it at one. Same argument as kTransformSampleRate, same value, kept as a
// separate constant because the routes are different and a future per-model
// preferred-rate lookup would have to answer them separately.
constexpr int kVoiceReferenceSampleRate = 0;
// Parses ModelOptions.MainGPU into a device index.
//
// Not std::atoi: it returns 0 for anything unparseable, so "gpu1" or a device
@@ -953,6 +997,166 @@ public:
return to_status(err);
}
}
// The first RPC that GENERATES audio from text, together with
// SoundGeneration below. AudioTransform already wrote files, but it
// transformed audio it was given; these two have no required input audio at
// all, which is why both carry a Result with a success flag rather than a
// response message describing what was found.
//
// TTSRequest.voice is overloaded across LocalAI backends, some reading it as
// a named preset and some as a path to a clip. The rule here is decided from
// the filesystem: an existing regular file is a speaker reference and
// routing prefers VoiceCloning, anything else is a preset. Instructions with
// no clip prefer VoiceDesign, and a clip outranks instructions when both are
// set. None of that is re-derived here; it is capability_routing's
// task_candidates, and this handler's job is only to describe the request
// truthfully in the RequestShape.
GStatus TTS(ServerContext *, const backend::TTSRequest *request,
backend::Result *result) override {
try {
GStatus refusal = GStatus::OK;
const auto model = snapshot_for(request, refusal);
if (model == nullptr) {
// Answered with the STATUS ALONE, leaving Result at its default.
// core/backend/tts.go checks the transport error before it looks
// at res.Success and returns on it, so the Result is never read
// on this path; and the router matches a stale route on the
// NOT_FOUND code plus the sentinel in the status message, which
// is where check_model_identity already put it.
return refusal;
}
const bool voice_is_file =
audiocpp_backend::voice_is_reference_file(request->voice());
audiocpp_backend::RequestShape shape;
shape.has_voice_reference = voice_is_file;
shape.has_instructions =
request->has_instructions() && !request->instructions().empty();
shape.pinned_task = model->pinned_task();
// FIRST, before the lane and before any file read. A family that
// cannot synthesise at all is answering a question about itself, so
// it must not queue behind somebody else's run to be told no, and it
// must not blame the caller's reference clip for a decision that had
// nothing to do with it. Same ordering as Diarize and
// AudioTransform.
model->check_can_serve(audiocpp_backend::Rpc::Tts, shape);
if (request->dst().empty()) {
throw audiocpp_backend::ConfigError(
"audio-cpp: TTS needs a dst output path");
}
std::optional<engine::runtime::AudioBuffer> reference;
if (voice_is_file) {
// Native rate, native channels: see kVoiceReferenceSampleRate.
reference = audiocpp_backend::read_audio_file(
request->voice(), kVoiceReferenceSampleRate);
}
// Read before the lane, like every other file-fed handler here, so
// a slow or large clip is not decoded while holding it.
const auto task =
audiocpp_backend::build_tts_request(*request, std::move(reference));
// Named local: LaneEntry is immovable and has to span both
// session_for and run_offline.
audiocpp_backend::LaneEntry lane = model->acquire(0);
const auto session =
model->session_for(audiocpp_backend::Rpc::Tts, shape, lane);
const auto task_result =
audiocpp_backend::run_offline(session, task, lane);
if (!task_result.audio_output.has_value()) {
// A family that routed successfully and then produced no audio
// is a broken family rather than a wrong request, but from the
// caller's side the actionable fact is that this model does not
// do this, so CapabilityError. Same judgement as AudioTransform.
throw audiocpp_backend::CapabilityError(
"audio-cpp: family '" + model->family() +
"' produced no audio for the TTS RPC");
}
// Throws a plain runtime_error, i.e. INTERNAL, on a write failure:
// dst is LocalAI's own generated-content path and not anything the
// caller named, so a full disk there is a server fault and is worth
// retrying. Only an empty dst is INVALID_ARGUMENT, and that is
// already refused above with a message naming the RPC.
audiocpp_backend::write_audio_file(request->dst(),
*task_result.audio_output);
result->set_success(true);
// The path, because that is what core/backend/tts.go's caller reads
// back and what every other LocalAI TTS backend puts here.
result->set_message(request->dst());
return GStatus::OK;
} catch (const std::exception &err) {
// Both channels, unlike the transcription-shaped handlers: Result
// has a success flag that the Go side checks in addition to the
// status, so leaving it default-false with no message would report a
// failure with no reason attached.
result->set_success(false);
result->set_message(err.what());
return to_status(err);
}
}
GStatus SoundGeneration(ServerContext *,
const backend::SoundGenerationRequest *request,
backend::Result *result) override {
try {
GStatus refusal = GStatus::OK;
const auto model = snapshot_for(request, refusal);
if (model == nullptr) {
return refusal;
}
audiocpp_backend::RequestShape shape;
// No request signal chooses between generation tasks: this RPC has
// exactly one candidate, AudioGeneration. pinned_task is still
// copied, because without it the model's `task:` option is dead
// here, which is how a gen family pinned to something else would
// silently be routed to generation anyway.
shape.pinned_task = model->pinned_task();
model->check_can_serve(audiocpp_backend::Rpc::SoundGeneration, shape);
if (request->dst().empty()) {
throw audiocpp_backend::ConfigError(
"audio-cpp: SoundGeneration needs a dst output path");
}
std::optional<engine::runtime::AudioBuffer> source;
if (request->has_src() && !request->src().empty()) {
// Native rate and channels. ace_step and stable_audio both
// resample per channel, so a downmix here would delete the
// stereo image of the very clip they are editing.
source = audiocpp_backend::read_audio_file(
request->src(), kVoiceReferenceSampleRate);
}
const auto task = audiocpp_backend::build_sound_generation_request(
*request, std::move(source));
audiocpp_backend::LaneEntry lane = model->acquire(0);
const auto session = model->session_for(
audiocpp_backend::Rpc::SoundGeneration, shape, lane);
const auto task_result =
audiocpp_backend::run_offline(session, task, lane);
if (!task_result.audio_output.has_value()) {
throw audiocpp_backend::CapabilityError(
"audio-cpp: family '" + model->family() +
"' produced no audio for the SoundGeneration RPC");
}
audiocpp_backend::write_audio_file(request->dst(),
*task_result.audio_output);
result->set_success(true);
result->set_message(request->dst());
return GStatus::OK;
} catch (const std::exception &err) {
result->set_success(false);
result->set_message(err.what());
return to_status(err);
}
}
};
void RunServer(const std::string &addr) {