#include "capability_routing.h" #include namespace audiocpp_backend { namespace { struct NamedTask { Task task; const char *name; }; // Short names are exactly the strings audio.cpp prints and parses in // framework/runtime/session.cpp, so a name pinned here survives conversion at // the engine boundary and a name copied out of audio.cpp is accepted here. All // thirteen have an upstream name; only "spk" is absent from the --task table in // docs/usage.md. const NamedTask kTaskNames[] = { {Task::Vad, "vad"}, {Task::Asr, "asr"}, {Task::Diarization, "diar"}, {Task::SourceSeparation, "sep"}, {Task::AudioGeneration, "gen"}, {Task::Tts, "tts"}, {Task::VoiceCloning, "clon"}, {Task::VoiceConversion, "vc"}, {Task::SpeechToSpeech, "s2s"}, {Task::Alignment, "align"}, {Task::VoiceDesign, "vdes"}, {Task::SpeakerRecognition, "spk"}, {Task::Svc, "svc"}, }; // Accepted on input but never emitted. "spkrec" was this backend's own earlier // name for the kind; upstream only ever knew "spk". const NamedTask kTaskAliases[] = { {Task::SpeakerRecognition, "spkrec"}, }; // First match wins, which is safe because a Capabilities value holds at most one // entry per task: it mirrors upstream runtime::TaskCapability (model.h), which // pairs one kind with a modes vector, and no loader's supported_tasks list // repeats a kind. bool family_supports(const Capabilities &caps, Task task, Mode mode) { for (const auto &capability : caps.tasks) { if (capability.task != task) { continue; } return std::find(capability.modes.begin(), capability.modes.end(), mode) != capability.modes.end(); } return false; } // Mode preference per RPC. Only AudioTranscriptionStream has a fallback: a // server-streaming transcription can be satisfied by an offline run that emits // one delta then the final result. Live transcription cannot, because it is // bidirectional and must consume audio incrementally. std::vector mode_candidates(Rpc rpc) { switch (rpc) { case Rpc::TtsStream: case Rpc::AudioTranscriptionLive: return {Mode::Streaming}; case Rpc::AudioTranscriptionStream: return {Mode::Streaming, Mode::Offline}; default: return {Mode::Offline}; } } std::vector task_candidates(Rpc rpc, const RequestShape &shape) { switch (rpc) { case Rpc::Tts: case Rpc::TtsStream: // A supplied speaker clip is the strongest signal: the caller named the // voice they want. Free-form instructions come next. Both fall back to // plain Tts so a family without the specialised task still answers. if (shape.has_voice_reference) { return {Task::VoiceCloning, Task::Tts, Task::VoiceDesign}; } if (shape.has_instructions) { return {Task::VoiceDesign, Task::Tts, Task::VoiceCloning}; } return {Task::Tts, Task::VoiceCloning, Task::VoiceDesign}; case Rpc::AudioTranscription: case Rpc::AudioTranscriptionStream: case Rpc::AudioTranscriptionLive: // Asr first: `prompt` is also whisper-style decoding context, so its // presence must not hijack a real ASR family into forced alignment. if (shape.has_prompt_text) { return {Task::Asr, Task::Alignment}; } return {Task::Asr}; case Rpc::Vad: return {Task::Vad}; case Rpc::Diarize: return {Task::Diarization}; case Rpc::SoundGeneration: return {Task::AudioGeneration}; case Rpc::AudioTransform: // Svc is listed for completeness but is unreachable by auto-routing, by // design: the only families advertising it (seed_vc, vevo2) also // advertise VoiceConversion, which always wins, and no request signal // means "this input is singing". Singing voice conversion therefore // requires an explicit task:svc pin. return {Task::SourceSeparation, Task::VoiceConversion, Task::Svc, Task::SpeechToSpeech}; } return {}; } // The reasons behind unsupported_surfaces(), spelled once because AudioEncode // and AudioDecode share theirs. Each is phrased in terms of what upstream does // and does not have, so a reader can check it against the pinned checkout // rather than take it on trust. Every one of them was checked against // audio.cpp e800d435d130dc776baf6f3e6129bb62b1495c89, and one of the four // claims this backend was planned against did not survive that check: see // kTransformStreamReason. // // A latent upstream inconsistency worth knowing about but deliberately NOT put // on the wire, because it would mislead: model_spec/schema.cpp's task-string // whitelist does accept "codec" (and "dialogue"), while // model_spec/metadata.cpp's parse_task_kind has no branch for either and // throws "unknown model spec task". So a spec declaring "codec" validates and // then fails to load. That is a hole in upstream's own validation, not a codec // task this backend could reach. const char *const kCodecReason = "audio.cpp's VoiceTaskKind has no codec entry, so no family can be asked to " "turn PCM into codec frames or back; miocodec carries a Codec tag in " "upstream's README but its loader advertises only vc and s2s"; // NOT "streaming exists for tts and asr only", which is what this backend was // planned to say and is false: silero_vad advertises vad with RunMode::Streaming // (src/models/silero_vad/session.cpp). The claim that actually holds is the // narrower one below, about the four tasks AudioTransform routes to. // The trailing clause is not padding. The premise is an absence, and an absence // does not on its own make the RPC impossible: an offline sep family could be // buffered and emitted as a stream, which is what several LocalAI backends do. // Stopping at "nothing advertises streaming" would imply an impossibility the // evidence does not support. What is true, and what the caller needs, is that // this backend declines to dress an offline call up as a streaming one. const char *const kTransformStreamReason = "no audio.cpp family advertises streaming for any task AudioTransform routes " "to (sep, vc, svc, s2s); upstream advertises RunMode::Streaming for tts, asr " "and vad only, and no conversion or separation family even implements its " "IStreamingVoiceTaskSession interface, so a streaming transform here would be " "a buffered offline call in disguise, which this backend does not pretend to " "offer"; // "clip-to-clip processing against a target voice", NOT "voice conversion". The // latter is true of miocodec and FALSE of vevo2, whose s2s route is `editing` // and only `editing`: src/models/vevo2/session.cpp's default_route_for_task maps // SpeechToSpeech to Editing and route_matches_task accepts nothing else, and // docs/models/vevo2.md defines that route as "Edit source speech into new target // text while using the target voice", requiring --target-text. It rewrites what // was said. vevo2's actual voice conversion is its separate vc task, which is // why upstream's README tags the family "TTS, Music, VC, Edit". The conclusion // is unaffected: neither family converses. const char *const kAudioToAudioReason = "LocalAI's contract here is OpenAI-Realtime shaped, an audio conversation " "emitting audio, transcript and tool-call deltas from a system prompt and a " "tool list; audio.cpp's s2s is offline clip-to-clip processing against a " "target voice, declared only by miocodec (voice conversion) and vevo2 " "(speech editing), with no conversation, system prompt or tool loop"; const char *const kVoiceEmbedReason = "no audio.cpp family advertises the spk (SpeakerRecognition) task, so " "nothing in the engine can produce a speaker embedding; the task kind " "itself exists upstream, and TitaNet and ECAPA-TDNN exist as internal " "conditioning encoders, but neither is registered as a loadable family"; // The tasks an RPC is ever willing to route to, independent of request shape. // // DERIVED from task_candidates rather than restated, so a task added to an // RPC's candidate list cannot become inadmissible as a pin by omission. Setting // every shape flag yields each RPC's widest list: the per-flag branches only // reorder the same three tasks for Tts, and only ADD Alignment for // transcription, so the union is what comes back. std::vector admissible_tasks(Rpc rpc) { RequestShape widest; widest.has_voice_reference = true; widest.has_instructions = true; widest.has_prompt_text = true; return task_candidates(rpc, widest); } std::string join_task_names(const std::vector &tasks) { std::string out; for (const Task task : tasks) { if (!out.empty()) { out += ", "; } out += task_name(task); } if (out.empty()) { out = "nothing"; } return out; } std::string join_attempts(const std::vector &tasks, const std::vector &modes) { std::string out; for (const Task task : tasks) { for (const Mode mode : modes) { if (!out.empty()) { out += ", "; } out += task_name(task); out += "/"; out += mode_name(mode); } } return out; } } // namespace const char *task_name(Task task) { for (const auto &entry : kTaskNames) { if (entry.task == task) { return entry.name; } } return "unknown"; } const char *mode_name(Mode mode) { return mode == Mode::Streaming ? "streaming" : "offline"; } const char *rpc_name(Rpc rpc) { switch (rpc) { case Rpc::Tts: return "TTS"; case Rpc::TtsStream: return "TTSStream"; case Rpc::AudioTranscription: return "AudioTranscription"; case Rpc::AudioTranscriptionStream: return "AudioTranscriptionStream"; case Rpc::AudioTranscriptionLive: return "AudioTranscriptionLive"; case Rpc::Vad: return "VAD"; case Rpc::Diarize: return "Diarize"; case Rpc::SoundGeneration: return "SoundGeneration"; case Rpc::AudioTransform: return "AudioTransform"; } return "unknown"; } bool parse_task_name(const std::string &value, Task &out) { for (const auto &entry : kTaskNames) { if (value == entry.name) { out = entry.task; return true; } } for (const auto &entry : kTaskAliases) { if (value == entry.name) { out = entry.task; return true; } } return false; } std::string describe_capabilities(const Capabilities &caps) { std::string out; for (const auto &capability : caps.tasks) { for (const Mode mode : capability.modes) { if (!out.empty()) { out += ", "; } out += task_name(capability.task); out += "/"; out += mode_name(mode); } } if (out.empty()) { out = "nothing"; } return out; } const std::vector &unsupported_surfaces() { // Ordered as UnsupportedRpc declares them. unsupported_surface() names each // index in a switch rather than casting the enum, so the order is checked at // compile time rather than trusted. static const std::vector kSurfaces = { {"AudioEncode", kCodecReason}, {"AudioDecode", kCodecReason}, {"AudioTransformStream", kTransformStreamReason}, {"AudioToAudioStream", kAudioToAudioReason}, {"VoiceEmbed", kVoiceEmbedReason}, }; return kSurfaces; } // A switch with NO default label, deliberately. -Wswitch is on under -Wall, so a // sixth UnsupportedRpc added without a case here is a BUILD diagnostic, which is // the only place this class of mistake can be caught for free: a positional // static_cast(rpc) would compile fine and read past the end of the table // at run time, on the one code path whose entire job is to be diagnosable. The // table stays a table because the tests iterate it. // // The trailing return is unreachable through the enum and exists only for a // caller that hands over a value outside it, which is already undefined // behaviour by the time it arrives. const UnsupportedSurface &unsupported_surface(UnsupportedRpc rpc) { const std::vector &surfaces = unsupported_surfaces(); switch (rpc) { case UnsupportedRpc::AudioEncode: return surfaces[0]; case UnsupportedRpc::AudioDecode: return surfaces[1]; case UnsupportedRpc::AudioTransformStream: return surfaces[2]; case UnsupportedRpc::AudioToAudioStream: return surfaces[3]; case UnsupportedRpc::VoiceEmbed: return surfaces[4]; } return surfaces[0]; } std::string unsupported_surface_message(const Capabilities &caps, const char *rpc, const char *reason) { return std::string("audio-cpp: the ") + rpc + " RPC is not available through this backend because " + reason + ". Loaded family '" + caps.family + "' supports: " + describe_capabilities(caps); } std::string unsupported_surface_message(const char *rpc, const char *reason) { return std::string("audio-cpp: the ") + rpc + " RPC is not available through this backend because " + reason + ". No model is loaded, so there is no family to list; loading one " "would not change this answer"; } Route resolve_route(Rpc rpc, const RequestShape &shape, const Capabilities &caps) { Route route; std::vector tasks; if (!shape.pinned_task.empty()) { Task pinned = Task::Tts; if (!parse_task_name(shape.pinned_task, pinned)) { route.error = "audio-cpp: unknown task option '" + shape.pinned_task + "'. Known tasks: gen, tts, clon, vc, svc, s2s, asr, " "align, vad, diar, sep, vdes, spk"; return route; } // A pin is honoured exactly, but ONLY on an RPC that could have routed // to it anyway. It used to replace the candidate list wholesale for // every RPC, and because the model's `task:` option is copied into the // shape by all nine handlers, one pin bled across all nine surfaces and // produced wrong 200s rather than errors: nemotron with task:asr made // Vad return 200 with zero segments after a full ASR decode, so 14 // seconds of speech was reported as silence, and silero_vad with // task:vad made AudioTranscription return 200 with empty text and four // segments whose spans were VAD segments, which the srt/vtt/lrc writers // then rendered as a well formed subtitle file of four timed EMPTY // cues. Refusing is what the docs already promise: "if the family // cannot serve it, the request is refused rather than rerouted". // // Every legitimate pin survives, because a pin only ever names the task // its own RPC already routes to: svc is in AudioTransform's candidates, // tts/clon/vdes in TTS's, asr in transcription's, vad and diar in // theirs. const std::vector admissible = admissible_tasks(rpc); if (std::find(admissible.begin(), admissible.end(), pinned) == admissible.end()) { route.error = std::string("audio-cpp: this model pins task '") + task_name(pinned) + "', which the " + rpc_name(rpc) + " RPC never routes to (it routes to " + join_task_names(admissible) + "). Remove the task option to reach this RPC, or call " "the RPC the pinned task serves"; return route; } tasks = {pinned}; } else { tasks = task_candidates(rpc, shape); } const std::vector modes = mode_candidates(rpc); // Task-major: prefer the right task in a fallback mode over the wrong task // in the preferred mode. for (const Task task : tasks) { for (const Mode mode : modes) { if (family_supports(caps, task, mode)) { route.ok = true; route.task = task; route.mode = mode; return route; } } } route.error = std::string("audio-cpp: family '") + caps.family + "' cannot serve the " + rpc_name(rpc) + " RPC (tried " + join_attempts(tasks, modes) + "); it supports: " + describe_capabilities(caps); return route; } } // namespace audiocpp_backend