diff --git a/backend/cpp/audio-cpp/CMakeLists.txt b/backend/cpp/audio-cpp/CMakeLists.txt index 2fe8c25d3..53eb4eea6 100644 --- a/backend/cpp/audio-cpp/CMakeLists.txt +++ b/backend/cpp/audio-cpp/CMakeLists.txt @@ -87,6 +87,7 @@ add_subdirectory("${AUDIO_CPP_DIR}" "${CMAKE_CURRENT_BINARY_DIR}/audio-cpp" EXCL add_executable(${TARGET} grpc-server.cpp model_options.cpp + capability_routing.cpp ) target_include_directories(${TARGET} PRIVATE diff --git a/backend/cpp/audio-cpp/capability_routing.cpp b/backend/cpp/audio-cpp/capability_routing.cpp new file mode 100644 index 000000000..3e9414025 --- /dev/null +++ b/backend/cpp/audio-cpp/capability_routing.cpp @@ -0,0 +1,219 @@ +#include "capability_routing.h" + +#include + +namespace audiocpp_backend { +namespace { + +struct NamedTask { + Task task; + const char *name; +}; + +// Short names match audiocpp_cli's --task values (docs/usage.md). Two kinds +// have no CLI name upstream; they get stable names here so pinning still works. +const NamedTask kTaskNames[] = { + {Task::Vad, "vad"}, + {Task::Asr, "asr"}, + {Task::Diarization, "diar"}, + {Task::SourceSeparation, "sep"}, + {Task::AudioGeneration, "gen"}, + {Task::Tts, "tts"}, + {Task::VoiceCloning, "clon"}, + {Task::VoiceConversion, "vc"}, + {Task::SpeechToSpeech, "s2s"}, + {Task::Alignment, "align"}, + {Task::VoiceDesign, "vdes"}, + {Task::SpeakerRecognition, "spkrec"}, + {Task::Svc, "svc"}, +}; + +bool family_supports(const Capabilities &caps, Task task, Mode mode) { + for (const auto &capability : caps.tasks) { + if (capability.task != task) { + continue; + } + return std::find(capability.modes.begin(), capability.modes.end(), mode) != + capability.modes.end(); + } + return false; +} + +// Mode preference per RPC. Only AudioTranscriptionStream has a fallback: a +// server-streaming transcription can be satisfied by an offline run that emits +// one delta then the final result. Live transcription cannot, because it is +// bidirectional and must consume audio incrementally. +std::vector mode_candidates(Rpc rpc) { + switch (rpc) { + case Rpc::TtsStream: + case Rpc::AudioTranscriptionLive: + return {Mode::Streaming}; + case Rpc::AudioTranscriptionStream: + return {Mode::Streaming, Mode::Offline}; + default: + return {Mode::Offline}; + } +} + +std::vector task_candidates(Rpc rpc, const RequestShape &shape) { + switch (rpc) { + case Rpc::Tts: + case Rpc::TtsStream: + // A supplied speaker clip is the strongest signal: the caller named the + // voice they want. Free-form instructions come next. Both fall back to + // plain Tts so a family without the specialised task still answers. + if (shape.has_voice_reference) { + return {Task::VoiceCloning, Task::Tts, Task::VoiceDesign}; + } + if (shape.has_instructions) { + return {Task::VoiceDesign, Task::Tts, Task::VoiceCloning}; + } + return {Task::Tts, Task::VoiceCloning, Task::VoiceDesign}; + case Rpc::AudioTranscription: + case Rpc::AudioTranscriptionStream: + case Rpc::AudioTranscriptionLive: + // Asr first: `prompt` is also whisper-style decoding context, so its + // presence must not hijack a real ASR family into forced alignment. + if (shape.has_prompt_text) { + return {Task::Asr, Task::Alignment}; + } + return {Task::Asr}; + case Rpc::Vad: + return {Task::Vad}; + case Rpc::Diarize: + return {Task::Diarization}; + case Rpc::SoundGeneration: + return {Task::AudioGeneration}; + case Rpc::AudioTransform: + return {Task::SourceSeparation, Task::VoiceConversion, Task::Svc, + Task::SpeechToSpeech}; + } + return {}; +} + +std::string join_attempts(const std::vector &tasks, + const std::vector &modes) { + std::string out; + for (const Task task : tasks) { + for (const Mode mode : modes) { + if (!out.empty()) { + out += ", "; + } + out += task_name(task); + out += "/"; + out += mode_name(mode); + } + } + return out; +} + +} // namespace + +const char *task_name(Task task) { + for (const auto &entry : kTaskNames) { + if (entry.task == task) { + return entry.name; + } + } + return "unknown"; +} + +const char *mode_name(Mode mode) { + return mode == Mode::Streaming ? "streaming" : "offline"; +} + +const char *rpc_name(Rpc rpc) { + switch (rpc) { + case Rpc::Tts: + return "TTS"; + case Rpc::TtsStream: + return "TTSStream"; + case Rpc::AudioTranscription: + return "AudioTranscription"; + case Rpc::AudioTranscriptionStream: + return "AudioTranscriptionStream"; + case Rpc::AudioTranscriptionLive: + return "AudioTranscriptionLive"; + case Rpc::Vad: + return "VAD"; + case Rpc::Diarize: + return "Diarize"; + case Rpc::SoundGeneration: + return "SoundGeneration"; + case Rpc::AudioTransform: + return "AudioTransform"; + } + return "unknown"; +} + +bool parse_task_name(const std::string &value, Task &out) { + for (const auto &entry : kTaskNames) { + if (value == entry.name) { + out = entry.task; + return true; + } + } + return false; +} + +std::string describe_capabilities(const Capabilities &caps) { + std::string out; + for (const auto &capability : caps.tasks) { + for (const Mode mode : capability.modes) { + if (!out.empty()) { + out += ", "; + } + out += task_name(capability.task); + out += "/"; + out += mode_name(mode); + } + } + if (out.empty()) { + out = "nothing"; + } + return out; +} + +Route resolve_route(Rpc rpc, const RequestShape &shape, + const Capabilities &caps) { + Route route; + + std::vector tasks; + if (!shape.pinned_task.empty()) { + Task pinned = Task::Tts; + if (!parse_task_name(shape.pinned_task, pinned)) { + route.error = "audio-cpp: unknown task option '" + shape.pinned_task + + "'. Known tasks: gen, tts, clon, vc, svc, s2s, asr, " + "align, vad, diar, sep, vdes, spkrec"; + return route; + } + // A pinned task is honoured exactly. Silently rerouting would make the + // option meaningless and hide a misconfigured model. + tasks = {pinned}; + } else { + tasks = task_candidates(rpc, shape); + } + + const std::vector modes = mode_candidates(rpc); + + // Task-major: prefer the right task in a fallback mode over the wrong task + // in the preferred mode. + for (const Task task : tasks) { + for (const Mode mode : modes) { + if (family_supports(caps, task, mode)) { + route.ok = true; + route.task = task; + route.mode = mode; + return route; + } + } + } + + route.error = std::string("audio-cpp: family '") + caps.family + + "' cannot serve the " + rpc_name(rpc) + " RPC (tried " + + join_attempts(tasks, modes) + "); it supports: " + + describe_capabilities(caps); + return route; +} + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/capability_routing.h b/backend/cpp/audio-cpp/capability_routing.h new file mode 100644 index 000000000..e5ebb4dd6 --- /dev/null +++ b/backend/cpp/audio-cpp/capability_routing.h @@ -0,0 +1,88 @@ +#pragma once + +// Decides which audio.cpp (task, mode) pair serves a given LocalAI RPC, or +// produces the capability error when none can. Standard library only, so this +// unit is tested without an audio.cpp checkout; loaded_model.cpp converts +// to and from engine::runtime types at the boundary. + +#include +#include + +namespace audiocpp_backend { + +// Mirrors engine::runtime::VoiceTaskKind, same members and same order. +enum class Task { + Vad, + Asr, + Diarization, + SourceSeparation, + AudioGeneration, + Tts, + VoiceCloning, + VoiceConversion, + SpeechToSpeech, + Alignment, + VoiceDesign, + SpeakerRecognition, + Svc, +}; + +// Mirrors engine::runtime::RunMode. +enum class Mode { Offline, Streaming }; + +struct TaskCapability { + Task task = Task::Vad; + std::vector modes; +}; + +struct Capabilities { + std::string family; + std::vector tasks; +}; + +// The LocalAI RPCs this backend serves. The unimplemented ones are not listed: +// they never reach routing. +enum class Rpc { + Tts, + TtsStream, + AudioTranscription, + AudioTranscriptionStream, + AudioTranscriptionLive, + Vad, + Diarize, + SoundGeneration, + AudioTransform, +}; + +struct RequestShape { + // A speaker reference clip was supplied (TTSRequest.voice resolved to audio). + bool has_voice_reference = false; + // TTSRequest.instructions is set. + bool has_instructions = false; + // TranscriptRequest.prompt is set. + bool has_prompt_text = false; + // The model's `task:` option, empty when unset. Overrides routing. + std::string pinned_task; +}; + +struct Route { + bool ok = false; + Task task = Task::Tts; + Mode mode = Mode::Offline; + // Set when ok is false. Suitable verbatim as an UNIMPLEMENTED message. + std::string error; +}; + +Route resolve_route(Rpc rpc, const RequestShape &shape, const Capabilities &caps); + +// Canonical audio.cpp CLI short names: gen, tts, clon, vc, svc, s2s, asr, +// align, vad, diar, sep, vdes, spkrec. +const char *task_name(Task task); +const char *mode_name(Mode mode); +const char *rpc_name(Rpc rpc); +bool parse_task_name(const std::string &value, Task &out); + +// "asr/offline, asr/streaming", for error messages. +std::string describe_capabilities(const Capabilities &caps); + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/capability_routing_test.cpp b/backend/cpp/audio-cpp/capability_routing_test.cpp new file mode 100644 index 000000000..439c3b23e --- /dev/null +++ b/backend/cpp/audio-cpp/capability_routing_test.cpp @@ -0,0 +1,294 @@ +// Unit tests for capability_routing. Standard library only. The harness +// compiles this as a single translation unit, so the implementation is +// included directly rather than linked. + +#include "capability_routing.cpp" + +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const std::string &name) { + if (!ok) { + failures++; + fprintf(stderr, "FAIL: %s\n", name.c_str()); + } else { + fprintf(stderr, "ok: %s\n", name.c_str()); + } +} + +using namespace audiocpp_backend; + +// Mirrors what supertonic advertises: TTS offline and streaming. +static Capabilities supertonic() { + return Capabilities{"supertonic", + {{Task::Tts, {Mode::Offline, Mode::Streaming}}}}; +} + +// Mirrors chatterbox: TTS, cloning and voice conversion, offline only. +static Capabilities chatterbox() { + return Capabilities{"chatterbox", + {{Task::Tts, {Mode::Offline}}, + {Task::VoiceCloning, {Mode::Offline}}, + {Task::VoiceConversion, {Mode::Offline}}}}; +} + +// Mirrors nemotron_asr: ASR offline and streaming. +static Capabilities nemotron() { + return Capabilities{"nemotron_asr", + {{Task::Asr, {Mode::Offline, Mode::Streaming}}}}; +} + +// Mirrors qwen3_asr: ASR offline only. +static Capabilities qwen3_asr() { + return Capabilities{"qwen3_asr", {{Task::Asr, {Mode::Offline}}}}; +} + +// Mirrors qwen3_forced_aligner: alignment only. +static Capabilities aligner() { + return Capabilities{"qwen3_forced_aligner", + {{Task::Alignment, {Mode::Offline}}}}; +} + +// Mirrors htdemucs: separation only. +static Capabilities htdemucs() { + return Capabilities{"htdemucs", {{Task::SourceSeparation, {Mode::Offline}}}}; +} + +static void test_plain_tts() { + const auto r = resolve_route(Rpc::Tts, RequestShape{}, chatterbox()); + check(r.ok, "plain TTS routes"); + check(r.task == Task::Tts, "plain TTS picks Tts, not VoiceCloning"); + check(r.mode == Mode::Offline, "TTS runs offline"); +} + +static void test_tts_with_voice_reference_prefers_cloning() { + RequestShape shape; + shape.has_voice_reference = true; + const auto r = resolve_route(Rpc::Tts, shape, chatterbox()); + check(r.ok, "TTS with a voice reference routes"); + check(r.task == Task::VoiceCloning, "voice reference prefers VoiceCloning"); +} + +// supertonic has no VoiceCloning: a voice reference must fall back to Tts +// rather than failing the request. +static void test_tts_voice_reference_falls_back_to_tts() { + RequestShape shape; + shape.has_voice_reference = true; + const auto r = resolve_route(Rpc::Tts, shape, supertonic()); + check(r.ok, "voice reference on a clone-less family still routes"); + check(r.task == Task::Tts, "falls back to Tts"); +} + +static void test_tts_instructions_prefer_voice_design() { + RequestShape shape; + shape.has_instructions = true; + Capabilities caps{"qwen3_tts", + {{Task::Tts, {Mode::Offline}}, + {Task::VoiceDesign, {Mode::Offline}}}}; + const auto r = resolve_route(Rpc::Tts, shape, caps); + check(r.ok, "TTS with instructions routes"); + check(r.task == Task::VoiceDesign, "instructions prefer VoiceDesign"); +} + +// A voice reference is a stronger signal than free-form instructions: cloning +// a specific voice is what the user asked for. +static void test_voice_reference_beats_instructions() { + RequestShape shape; + shape.has_voice_reference = true; + shape.has_instructions = true; + Capabilities caps{"omnivoice", + {{Task::Tts, {Mode::Offline}}, + {Task::VoiceCloning, {Mode::Offline}}, + {Task::VoiceDesign, {Mode::Offline}}}}; + const auto r = resolve_route(Rpc::Tts, shape, caps); + check(r.ok, "both signals present routes"); + check(r.task == Task::VoiceCloning, "voice reference outranks instructions"); +} + +static void test_tts_stream_requires_streaming() { + const auto ok = resolve_route(Rpc::TtsStream, RequestShape{}, supertonic()); + check(ok.ok, "streaming TTS routes on supertonic"); + check(ok.mode == Mode::Streaming, "TTSStream runs in streaming mode"); + + const auto bad = resolve_route(Rpc::TtsStream, RequestShape{}, chatterbox()); + check(!bad.ok, "streaming TTS is refused on an offline-only family"); + check(bad.error.find("chatterbox") != std::string::npos, + "error names the family"); + check(bad.error.find("tts/offline") != std::string::npos, + "error lists what the family does support"); +} + +static void test_transcription_stream_falls_back_to_offline() { + const auto streaming = + resolve_route(Rpc::AudioTranscriptionStream, RequestShape{}, nemotron()); + check(streaming.ok && streaming.mode == Mode::Streaming, + "streaming ASR uses streaming mode when offered"); + + const auto offline = + resolve_route(Rpc::AudioTranscriptionStream, RequestShape{}, qwen3_asr()); + check(offline.ok, "streaming ASR falls back on an offline-only family"); + check(offline.mode == Mode::Offline, "fallback mode is offline"); + check(offline.task == Task::Asr, "fallback task is still Asr"); +} + +// Task preference dominates mode preference: it is better to run the right +// task in a fallback mode than the wrong task in the preferred mode. This is +// the only RPC where the two orderings can disagree, because it is the only +// one with more than one acceptable mode. +static void test_task_preference_beats_mode_preference() { + RequestShape shape; + shape.has_prompt_text = true; + Capabilities mixed{"mixed_asr_aligner", + {{Task::Asr, {Mode::Offline}}, + {Task::Alignment, {Mode::Streaming}}}}; + const auto r = resolve_route(Rpc::AudioTranscriptionStream, shape, mixed); + check(r.ok, "mixed family routes"); + check(r.task == Task::Asr, + "the preferred task wins even in its fallback mode"); + check(r.mode == Mode::Offline, + "the fallback mode is accepted to keep the preferred task"); +} + +// Live transcription is bidirectional and cannot be faked from an offline run. +static void test_live_transcription_has_no_offline_fallback() { + const auto r = + resolve_route(Rpc::AudioTranscriptionLive, RequestShape{}, qwen3_asr()); + check(!r.ok, "live transcription is refused on an offline-only family"); +} + +static void test_alignment_needs_prompt_text() { + const auto without = + resolve_route(Rpc::AudioTranscription, RequestShape{}, aligner()); + check(!without.ok, "aligner without a transcript is refused"); + + RequestShape shape; + shape.has_prompt_text = true; + const auto with = resolve_route(Rpc::AudioTranscription, shape, aligner()); + check(with.ok, "aligner with a transcript routes"); + check(with.task == Task::Alignment, "routes to Alignment"); +} + +// A real ASR family must not be hijacked to Alignment just because the caller +// passed a prompt: `prompt` is also whisper-style decoding context. +static void test_prompt_does_not_hijack_asr() { + RequestShape shape; + shape.has_prompt_text = true; + const auto r = resolve_route(Rpc::AudioTranscription, shape, nemotron()); + check(r.ok, "ASR with a prompt routes"); + check(r.task == Task::Asr, "Asr is preferred over Alignment"); +} + +static void test_audio_transform_prefers_separation() { + const auto sep = + resolve_route(Rpc::AudioTransform, RequestShape{}, htdemucs()); + check(sep.ok && sep.task == Task::SourceSeparation, "separation routes"); + + Capabilities miocodec{"miocodec", + {{Task::VoiceConversion, {Mode::Offline}}, + {Task::SpeechToSpeech, {Mode::Offline}}}}; + const auto vc = resolve_route(Rpc::AudioTransform, RequestShape{}, miocodec); + check(vc.ok && vc.task == Task::VoiceConversion, + "voice conversion is preferred over speech-to-speech"); +} + +static void test_pinned_task_overrides_routing() { + RequestShape shape; + shape.pinned_task = "s2s"; + Capabilities miocodec{"miocodec", + {{Task::VoiceConversion, {Mode::Offline}}, + {Task::SpeechToSpeech, {Mode::Offline}}}}; + const auto r = resolve_route(Rpc::AudioTransform, shape, miocodec); + check(r.ok && r.task == Task::SpeechToSpeech, "pinned task wins"); + + RequestShape bad; + bad.pinned_task = "not-a-task"; + const auto e = resolve_route(Rpc::AudioTransform, bad, miocodec); + check(!e.ok, "an unknown pinned task is an error"); + check(e.error.find("not-a-task") != std::string::npos, + "error names the bad task"); + + // A pinned task the family does not offer must fail, not silently reroute. + RequestShape unsupported; + unsupported.pinned_task = "sep"; + const auto u = resolve_route(Rpc::AudioTransform, unsupported, miocodec); + check(!u.ok, "a pinned but unsupported task is refused"); +} + +static void test_vad_and_diarize() { + Capabilities silero{"silero_vad", {{Task::Vad, {Mode::Offline, Mode::Streaming}}}}; + const auto v = resolve_route(Rpc::Vad, RequestShape{}, silero); + check(v.ok && v.task == Task::Vad && v.mode == Mode::Offline, "VAD routes offline"); + + const auto d = resolve_route(Rpc::Diarize, RequestShape{}, silero); + check(!d.ok, "diarization is refused on a VAD-only family"); + + Capabilities sortformer{"sortformer_diar", {{Task::Diarization, {Mode::Offline}}}}; + const auto ok = resolve_route(Rpc::Diarize, RequestShape{}, sortformer); + check(ok.ok && ok.task == Task::Diarization, "diarization routes"); +} + +static void test_sound_generation() { + Capabilities stable{"stable_audio", {{Task::AudioGeneration, {Mode::Offline}}}}; + const auto r = resolve_route(Rpc::SoundGeneration, RequestShape{}, stable); + check(r.ok && r.task == Task::AudioGeneration, "sound generation routes"); +} + +static void test_names_round_trip() { + const Task all[] = {Task::Vad, Task::Asr, Task::Diarization, + Task::SourceSeparation, Task::AudioGeneration, Task::Tts, + Task::VoiceCloning, Task::VoiceConversion, + Task::SpeechToSpeech, Task::Alignment, Task::VoiceDesign, + Task::SpeakerRecognition, Task::Svc}; + for (const Task t : all) { + Task parsed = Task::Vad; + const bool ok = parse_task_name(task_name(t), parsed); + check(ok && parsed == t, + std::string("task name round-trips: ") + task_name(t)); + } + check(std::string(mode_name(Mode::Offline)) == "offline", "offline name"); + check(std::string(mode_name(Mode::Streaming)) == "streaming", "streaming name"); +} + +static void test_describe_capabilities() { + const std::string described = describe_capabilities(nemotron()); + check(described.find("asr/offline") != std::string::npos, + "description lists asr/offline"); + check(described.find("asr/streaming") != std::string::npos, + "description lists asr/streaming"); +} + +static void test_empty_capabilities() { + const auto r = resolve_route(Rpc::Tts, RequestShape{}, Capabilities{"mystery", {}}); + check(!r.ok, "a family advertising nothing is refused"); + check(r.error.find("mystery") != std::string::npos, "error names the family"); +} + +int main() { + test_plain_tts(); + test_tts_with_voice_reference_prefers_cloning(); + test_tts_voice_reference_falls_back_to_tts(); + test_tts_instructions_prefer_voice_design(); + test_voice_reference_beats_instructions(); + test_tts_stream_requires_streaming(); + test_transcription_stream_falls_back_to_offline(); + test_task_preference_beats_mode_preference(); + test_live_transcription_has_no_offline_fallback(); + test_alignment_needs_prompt_text(); + test_prompt_does_not_hijack_asr(); + test_audio_transform_prefers_separation(); + test_pinned_task_overrides_routing(); + test_vad_and_diarize(); + test_sound_generation(); + test_names_round_trip(); + test_describe_capabilities(); + test_empty_capabilities(); + if (failures) { + fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + fprintf(stderr, "all capability_routing checks passed\n"); + return 0; +}