diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index d66d53628..a281ccce1 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=9a02e61326aaaf9d462b584ca5e0daba22c0abfc +AUDIO_CPP_VERSION?=cf124a67cc55d8f65a9a15eec69edbff0fb212c8 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/cpp/audio-cpp/capability_routing.cpp b/backend/cpp/audio-cpp/capability_routing.cpp index 9cd5b9b32..df597ef37 100644 --- a/backend/cpp/audio-cpp/capability_routing.cpp +++ b/backend/cpp/audio-cpp/capability_routing.cpp @@ -13,7 +13,7 @@ struct NamedTask { // Short names are exactly the strings audio.cpp prints and parses in // framework/runtime/session.cpp, so a name pinned here survives conversion at // the engine boundary and a name copied out of audio.cpp is accepted here. All -// thirteen have an upstream name; only "spk" is absent from the --task table in +// tasks have an upstream name; only "spk" is absent from the --task table in // docs/usage.md. const NamedTask kTaskNames[] = { {Task::Vad, "vad"}, @@ -30,6 +30,7 @@ const NamedTask kTaskNames[] = { {Task::SpeakerRecognition, "spk"}, {Task::Svc, "svc"}, {Task::Midi, "midi"}, + {Task::TurnDetection, "turn"}, }; // Accepted on input but never emitted. "spkrec" was this backend's own earlier diff --git a/backend/cpp/audio-cpp/capability_routing.h b/backend/cpp/audio-cpp/capability_routing.h index 6b0a9098b..1624b9d2e 100644 --- a/backend/cpp/audio-cpp/capability_routing.h +++ b/backend/cpp/audio-cpp/capability_routing.h @@ -26,6 +26,7 @@ enum class Task { SpeakerRecognition, Svc, Midi, + TurnDetection, }; // Mirrors engine::runtime::RunMode. @@ -78,7 +79,7 @@ struct Route { Route resolve_route(Rpc rpc, const RequestShape &shape, const Capabilities &caps); // Canonical audio.cpp short names: gen, tts, clon, vc, svc, s2s, asr, align, -// vad, diar, sep, vdes, spk. parse_task_name additionally accepts "spkrec" as +// vad, diar, sep, vdes, spk, midi, turn. parse_task_name additionally accepts "spkrec" as // a legacy alias; task_name only ever emits "spk". const char *task_name(Task task); const char *mode_name(Mode mode); diff --git a/backend/cpp/audio-cpp/capability_routing_test.cpp b/backend/cpp/audio-cpp/capability_routing_test.cpp index 9163d4ba3..d149b84f3 100644 --- a/backend/cpp/audio-cpp/capability_routing_test.cpp +++ b/backend/cpp/audio-cpp/capability_routing_test.cpp @@ -361,7 +361,8 @@ static void test_names_round_trip() { Task::SourceSeparation, Task::AudioGeneration, Task::Tts, Task::VoiceCloning, Task::VoiceConversion, Task::SpeechToSpeech, Task::Alignment, Task::VoiceDesign, - Task::SpeakerRecognition, Task::Svc, Task::Midi}; + Task::SpeakerRecognition, Task::Svc, Task::Midi, + Task::TurnDetection}; for (const Task t : all) { Task parsed = Task::Vad; const bool ok = parse_task_name(task_name(t), parsed); @@ -567,7 +568,28 @@ static void test_unsupported_surface_message_with_empty_capabilities() { "empty-capability message says the family supports nothing"); } +static void test_turn_detection_is_not_vad() { + Task turn = Task::Vad; + check(parse_task_name("turn", turn), "upstream turn task parses"); + check(std::string(task_name(turn)) == "turn", "turn task name round-trips"); + Capabilities caps{"smart_turn", {{turn, {Mode::Offline}}}}; + const Rpc all[] = {Rpc::Tts, Rpc::TtsStream, Rpc::AudioTranscription, + Rpc::AudioTranscriptionStream, Rpc::AudioTranscriptionLive, + Rpc::Vad, Rpc::Diarize, Rpc::SoundGeneration, Rpc::AudioTransform}; + for (const auto rpc : all) { + check(!resolve_route(rpc, RequestShape{}, caps).ok, + "turn detection cannot serve an existing RPC"); + RequestShape pinned; + pinned.pinned_task = "turn"; + check(!resolve_route(rpc, pinned, caps).ok, + "pinning turn detection cannot bypass RPC admission"); + } + check(describe_capabilities(caps).find("turn/offline") != std::string::npos, + "turn detection has an accurate capability diagnostic"); +} + int main() { + test_turn_detection_is_not_vad(); test_plain_tts(); test_tts_with_voice_reference_prefers_cloning(); test_tts_voice_reference_falls_back_to_tts(); diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp index 28bed59b3..15e2e758f 100644 --- a/backend/cpp/audio-cpp/loaded_model.cpp +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -67,10 +67,9 @@ static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeechToSpeech) == 8, "Voi static_assert(kEngine(engine::runtime::VoiceTaskKind::Alignment) == 9, "VoiceTaskKind drifted"); static_assert(kEngine(engine::runtime::VoiceTaskKind::VoiceDesign) == 10, "VoiceTaskKind drifted"); static_assert(kEngine(engine::runtime::VoiceTaskKind::SpeakerRecognition) == 11, "VoiceTaskKind drifted"); -// The last member. Pinning it pins the member count too, as long as the -// enumerators stay contiguous and unassigned, which upstream's declaration is. static_assert(kEngine(engine::runtime::VoiceTaskKind::Svc) == 12, "VoiceTaskKind drifted"); -static_assert(kEngine(engine::runtime::VoiceTaskKind::Midi) == 13, +static_assert(kEngine(engine::runtime::VoiceTaskKind::Midi) == 13, "VoiceTaskKind drifted"); +static_assert(kEngine(engine::runtime::VoiceTaskKind::TurnDetection) == 14, "engine::runtime::VoiceTaskKind gained, lost or reordered a member. " "audiocpp_backend::Task mirrors it positionally: update capability_routing.h, " "to_engine_task and from_engine_task together, then move this pin."); @@ -89,6 +88,7 @@ static_assert(kMirror(Task::VoiceDesign) == 10, "Task drifted from VoiceTaskKind static_assert(kMirror(Task::SpeakerRecognition) == 11, "Task drifted from VoiceTaskKind"); static_assert(kMirror(Task::Svc) == 12, "Task drifted from VoiceTaskKind"); static_assert(kMirror(Task::Midi) == 13, "Task drifted from VoiceTaskKind"); +static_assert(kMirror(Task::TurnDetection) == 14, "Task drifted from VoiceTaskKind"); static_assert(static_cast(engine::runtime::RunMode::Offline) == 0, "RunMode drifted"); static_assert(static_cast(engine::runtime::RunMode::Streaming) == 1, @@ -247,6 +247,7 @@ engine::runtime::VoiceTaskKind to_engine_task(Task task) { case Task::SpeakerRecognition: return K::SpeakerRecognition; case Task::Svc: return K::Svc; case Task::Midi: return K::Midi; + case Task::TurnDetection: return K::TurnDetection; } // Unreachable for any valid enumerator. No `default:` label, so -Wswitch // still reports a member this switch stops covering. @@ -270,6 +271,7 @@ Task from_engine_task(engine::runtime::VoiceTaskKind kind) { case K::SpeakerRecognition: return Task::SpeakerRecognition; case K::Svc: return Task::Svc; case K::Midi: return Task::Midi; + case K::TurnDetection: return Task::TurnDetection; } return Task::Vad; } diff --git a/docs/content/features/audio-cpp.md b/docs/content/features/audio-cpp.md index 59013ea5d..e77d68559 100644 --- a/docs/content/features/audio-cpp.md +++ b/docs/content/features/audio-cpp.md @@ -221,6 +221,10 @@ unaffected. ## Pinned tasks +The upstream `turn` task performs turn detection. LocalAI has no RPC for this +task, so requests to a model that only advertises `turn` are refused. It is not +routed through voice activity detection (`vad`). + Two routes are unreachable by auto-routing and need `task:` in the model config, because nothing in a request distinguishes them from a task the same family also advertises: