From 6ee95ab01bd1e84e5b52af4915f8e439732debd0 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sun, 26 Jul 2026 16:11:45 +0000 Subject: [PATCH] backend(audio-cpp): serve the AudioTranscription RPC Adds result_map, the engine-to-proto boundary, and wires the offline transcription RPC. The handler branches on the ROUTED task: for Asr the request's prompt is whisper-style decoding context and becomes a request option, for Alignment the same field IS the transcript to align and becomes the text input. Routing has already decided which. The result text is TaskResult.text_output verbatim and is never derived from the segments. audio.cpp carries transcript text in text_output and nowhere else, so deriving it returns an empty transcript for every producer that reports segments without word timing. transcript_assembly already enforces that; this commit's job is not to undo it at the proto boundary, and result_map_ctest pins it there. read_audio_file now takes the sample rate the caller needs. Both file-fed speech handlers ask for 16 kHz mono, for two reasons: silero_vad and sortformer_diar refuse anything else outright, which turned an ordinary 44.1 kHz upload into INTERNAL, and nemotron_asr emits word timestamps in its own 16 kHz feature domain whatever the input was, so only a 16 kHz buffer makes the emitted nanoseconds right. Zero keeps the file's native rate and channels, which is what source separation will need. LoadedModel::check_can_serve answers a capability refusal before the lane is taken and before the input file is read. Routing is a pure read of the immutable capabilities, so a model that cannot serve an RPC no longer waits out somebody else's run to say so. VAD and Diarize use it too. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto --- backend/cpp/audio-cpp/CMakeLists.txt | 50 +++++ backend/cpp/audio-cpp/audio_io.cpp | 51 ++++- backend/cpp/audio-cpp/audio_io.h | 30 ++- backend/cpp/audio-cpp/audio_io_ctest.cpp | 168 ++++++++++++++ backend/cpp/audio-cpp/grpc-server.cpp | 210 ++++++++++++++++-- backend/cpp/audio-cpp/loaded_model.cpp | 7 + backend/cpp/audio-cpp/loaded_model.h | 19 ++ backend/cpp/audio-cpp/result_map.cpp | 83 +++++++ backend/cpp/audio-cpp/result_map.h | 36 +++ backend/cpp/audio-cpp/result_map_ctest.cpp | 247 +++++++++++++++++++++ 10 files changed, 866 insertions(+), 35 deletions(-) create mode 100644 backend/cpp/audio-cpp/audio_io_ctest.cpp create mode 100644 backend/cpp/audio-cpp/result_map.cpp create mode 100644 backend/cpp/audio-cpp/result_map.h create mode 100644 backend/cpp/audio-cpp/result_map_ctest.cpp diff --git a/backend/cpp/audio-cpp/CMakeLists.txt b/backend/cpp/audio-cpp/CMakeLists.txt index 269e7b9e8..90b5e1c0d 100644 --- a/backend/cpp/audio-cpp/CMakeLists.txt +++ b/backend/cpp/audio-cpp/CMakeLists.txt @@ -93,6 +93,7 @@ add_executable(${TARGET} audio_io.cpp audio_units.cpp transcript_assembly.cpp + result_map.cpp inference_lane.cpp ) @@ -137,4 +138,53 @@ set_target_properties(${TARGET} PROPERTIES if(AUDIO_CPP_GRPC_BUILD_TESTS) enable_testing() + + # These are the units whose tests CANNOT run under + # backend/cpp/run-unit-tests.sh, because that script compiles each + # *_test.cpp standalone with no protobuf and no audio.cpp include path. + # They are named *_ctest.cpp so the script's glob does not pick them up and + # fail every backend's suite; everything that can be stdlib-only still is, + # and still lives in a *_test.cpp beside its unit. + # + # -Wall -Wextra -Wpedantic here and not on ${TARGET}: upstream's own + # add_compile_options is a property of its subdirectory and does not reach + # ours, so without naming them the tests would build as quietly as + # everything else. + add_executable(result_map_ctest + result_map_ctest.cpp + result_map.cpp + transcript_assembly.cpp + audio_units.cpp) + target_include_directories(result_map_ctest PRIVATE + "${AUDIO_CPP_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}" + # session.h reaches ggml.h through core/backend.h. Every other target + # here inherits that directory from the ggml target it links; this one + # links no ggml, so it has to name it. + "${AUDIO_CPP_DIR}/external/ggml/include") + # No engine_runtime: result_map touches only the plain structs in + # engine/framework/runtime/session.h, so the header is all it needs. + target_link_libraries(result_map_ctest PRIVATE + hw_grpc_proto + protobuf::libprotobuf + Threads::Threads) + target_compile_options(result_map_ctest PRIVATE -Wall -Wextra -Wpedantic) + add_test(NAME result_map COMMAND result_map_ctest) + + add_executable(audio_io_ctest + audio_io_ctest.cpp + audio_io.cpp) + target_include_directories(audio_io_ctest PRIVATE + "${AUDIO_CPP_DIR}/include" + "${CMAKE_CURRENT_SOURCE_DIR}") + target_link_libraries(audio_io_ctest PRIVATE + engine_runtime + ggml + Threads::Threads) + target_compile_options(audio_io_ctest PRIVATE -Wall -Wextra -Wpedantic) + set_target_properties(audio_io_ctest PROPERTIES + BUILD_RPATH "$ORIGIN" + INSTALL_RPATH "$ORIGIN" + BUILD_WITH_INSTALL_RPATH TRUE) + add_test(NAME audio_io COMMAND audio_io_ctest) endif() diff --git a/backend/cpp/audio-cpp/audio_io.cpp b/backend/cpp/audio-cpp/audio_io.cpp index 65dfba5fb..b6d1e6c63 100644 --- a/backend/cpp/audio-cpp/audio_io.cpp +++ b/backend/cpp/audio-cpp/audio_io.cpp @@ -2,15 +2,18 @@ #include "loaded_model.h" +#include "engine/framework/audio/conversion.h" #include "engine/framework/audio/wav_reader.h" #include "engine/framework/audio/wav_writer.h" #include +#include #include namespace audiocpp_backend { -engine::runtime::AudioBuffer read_audio_file(const std::string &path) { +engine::runtime::AudioBuffer read_audio_file(const std::string &path, + int target_sample_rate) { if (path.empty()) { throw ConfigError("audio-cpp: no input audio path was supplied"); } @@ -28,24 +31,52 @@ engine::runtime::AudioBuffer read_audio_file(const std::string &path) { if (!present) { throw ConfigError("audio-cpp: input audio does not exist: " + path); } - engine::runtime::AudioBuffer buffer; + engine::audio::WavData wav; try { - const engine::audio::WavData wav = - engine::audio::read_wav_f32(std::filesystem::path(path)); - buffer.sample_rate = wav.sample_rate; - // AudioBuffer's own default is 1, and a reader that reports 0 channels - // still gave us an interleaving of one. - buffer.channels = wav.channels > 0 ? wav.channels : 1; - buffer.samples = wav.samples; + wav = engine::audio::read_wav_f32(std::filesystem::path(path)); } catch (const std::exception &err) { throw ConfigError("audio-cpp: cannot read " + path + " as WAV: " + err.what()); } - if (buffer.sample_rate <= 0) { + if (wav.sample_rate <= 0) { throw ConfigError("audio-cpp: " + path + " declares a non-positive sample rate; every " "timestamp derived from it would be zero"); } + // AudioBuffer's own default is 1, and a reader that reports 0 channels + // still gave us an interleaving of one. Normalised before the conversion + // below rather than after, because mixdown_interleaved_to_mono_average + // throws on a non-positive channel count. + if (wav.channels <= 0) { + wav.channels = 1; + } + + engine::runtime::AudioBuffer buffer; + if (target_sample_rate <= 0) { + buffer.sample_rate = wav.sample_rate; + buffer.channels = wav.channels; + buffer.samples = std::move(wav.samples); + return buffer; + } + + buffer.sample_rate = target_sample_rate; + buffer.channels = 1; + try { + // A no-op copy when the rates already match, so the common 16 kHz + // upload pays only the mono mixdown it would have paid inside the + // family anyway. + buffer.samples = + engine::audio::convert_wav_to_mono_linear_resampled(wav, target_sample_rate); + } catch (const std::exception &err) { + // ConfigError, so this is INVALID_ARGUMENT rather than INTERNAL. What + // reaches here is a malformed input: a sample count that is not a whole + // number of frames is the realistic one, and it is the uploader's file + // that is truncated, not this backend that is broken. + throw ConfigError("audio-cpp: cannot resample " + path + " from " + + std::to_string(wav.sample_rate) + " Hz to " + + std::to_string(target_sample_rate) + + " Hz: " + err.what()); + } return buffer; } diff --git a/backend/cpp/audio-cpp/audio_io.h b/backend/cpp/audio-cpp/audio_io.h index 10f59d61c..6a882fdb0 100644 --- a/backend/cpp/audio-cpp/audio_io.h +++ b/backend/cpp/audio-cpp/audio_io.h @@ -13,16 +13,36 @@ namespace audiocpp_backend { -// Reads a WAV file at its native sample rate and channel count. Throws -// ConfigError when the file is missing, is not readable as WAV, or declares a -// non-positive sample rate: all three are user-fixable input problems rather -// than backend faults. +// Reads a WAV file. Throws ConfigError when the file is missing, is not +// readable as WAV, or declares a non-positive sample rate: all three are +// user-fixable input problems rather than backend faults. // // A declared sample rate of zero is refused rather than passed on, because // every downstream conversion in audio_units answers 0 for a non-positive rate. // Accepting it would turn a corrupt header into a response full of zero // timestamps, which reads as a real answer. -engine::runtime::AudioBuffer read_audio_file(const std::string &path); +// +// `target_sample_rate` is the rate the CALLER needs, in Hz: +// +// 0 (or negative) keep the file's own rate and channel count. +// positive downmix to mono and resample to that rate. Resampling is +// skipped when the file already declares it, so passing the +// rate a route needs costs nothing on the common input. +// +// It is a parameter, and not a constant inside this function, because the +// routes that read audio do not agree on an answer. Speech routes want 16 kHz +// mono; source separation does not, and folding a 44.1 kHz stereo input to +// 16 kHz mono for demucs or roformer would destroy the very thing they separate +// (both refuse a rate other than their own outright). Making the caller name +// the rate keeps that decision where the route is known. +// +// Downmixing along with the resample is not an extra liberty: every family a +// positive rate is used for (silero_vad, sortformer_diar and every ASR family) +// begins by calling the same mixdown_interleaved_to_mono_average on whatever it +// is given. Doing it once here produces the identical samples and halves the +// buffer that is then moved through the request. +engine::runtime::AudioBuffer read_audio_file(const std::string &path, + int target_sample_rate); // Writes 16-bit PCM WAV, creating parent directories. Throws ConfigError when // the destination cannot be written. diff --git a/backend/cpp/audio-cpp/audio_io_ctest.cpp b/backend/cpp/audio-cpp/audio_io_ctest.cpp new file mode 100644 index 000000000..f967a65ae --- /dev/null +++ b/backend/cpp/audio-cpp/audio_io_ctest.cpp @@ -0,0 +1,168 @@ +// Tests for audio_io's reading contract, and in particular for the resampling +// that keeps a 44.1 or 48 kHz upload from reaching a family that only accepts +// 16 kHz. +// +// NAMED _ctest AND NOT _test ON PURPOSE: see the note at the top of +// result_map_ctest.cpp. This file links the audio.cpp engine, so it is built +// and run by ctest, not by backend/cpp/run-unit-tests.sh. +// +// make -C backend/cpp/audio-cpp test-engine + +#include "audio_io.h" + +#include "loaded_model.h" + +#include +#include +#include +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const std::string &name) { + if (!ok) { + failures++; + fprintf(stderr, "FAIL: %s\n", name.c_str()); + } else { + fprintf(stderr, "ok: %s\n", name.c_str()); + } +} + +using namespace audiocpp_backend; + +// A one-second tone, interleaved across `channels`. Real audio rather than +// silence so a resample that dropped its input would be visible as a flat +// buffer, not just as a different length. +static engine::runtime::AudioBuffer tone(int sample_rate, int channels, + float seconds) { + engine::runtime::AudioBuffer buffer; + buffer.sample_rate = sample_rate; + buffer.channels = channels; + const auto frames = + static_cast(static_cast(sample_rate) * seconds); + buffer.samples.reserve(frames * static_cast(channels)); + for (size_t frame = 0; frame < frames; ++frame) { + const float value = 0.5f * std::sin(2.0f * 3.14159265f * 220.0f * + static_cast(frame) / + static_cast(sample_rate)); + for (int channel = 0; channel < channels; ++channel) { + buffer.samples.push_back(value); + } + } + return buffer; +} + +static float peak(const std::vector &samples) { + float highest = 0.0f; + for (const float sample : samples) { + highest = std::max(highest, std::abs(sample)); + } + return highest; +} + +static std::filesystem::path scratch_dir() { + const auto dir = std::filesystem::temp_directory_path() / "audiocpp-io-ctest"; + std::filesystem::create_directories(dir); + return dir; +} + +// The I2 fixture. Before the resample this returned a 44.1 kHz buffer, which +// silero_vad and sortformer_diar both reject with a plain runtime_error, which +// the server maps to INTERNAL. A 44.1 kHz WAV is an ordinary upload. +static void test_441k_stereo_is_read_as_16k_mono() { + const auto path = scratch_dir() / "input-44100-stereo.wav"; + write_audio_file(path.string(), tone(44100, 2, 1.0f)); + + const auto audio = read_audio_file(path.string(), 16000); + check(audio.sample_rate == 16000, "44.1 kHz input is resampled to 16 kHz"); + check(audio.channels == 1, "stereo input is downmixed to mono"); + // Linear resampling lands within a sample or two of the exact ratio. + const auto frames = static_cast(audio.samples.size()); + check(frames > 15990 && frames < 16010, + "one second in stays one second out"); + check(peak(audio.samples) > 0.2f, + "the resampled buffer still carries the signal"); +} + +static void test_48k_is_read_as_16k() { + const auto path = scratch_dir() / "input-48000-mono.wav"; + write_audio_file(path.string(), tone(48000, 1, 0.5f)); + + const auto audio = read_audio_file(path.string(), 16000); + check(audio.sample_rate == 16000, "48 kHz input is resampled to 16 kHz"); + const auto frames = static_cast(audio.samples.size()); + check(frames > 7990 && frames < 8010, "half a second in, half a second out"); +} + +// The common case: the upload is already 16 kHz mono, and nothing is resampled. +static void test_16k_mono_passes_through_unchanged() { + const auto path = scratch_dir() / "input-16000-mono.wav"; + const auto source = tone(16000, 1, 1.0f); + write_audio_file(path.string(), source); + + const auto audio = read_audio_file(path.string(), 16000); + check(audio.sample_rate == 16000, "16 kHz stays 16 kHz"); + check(audio.channels == 1, "mono stays mono"); + check(audio.samples.size() == source.samples.size(), + "a matching rate resamples nothing"); +} + +// Rate 0 means "give me the file as it is", which is what a source separation +// route needs: demucs and roformer refuse anything but their own 44.1 kHz and +// work on stereo, so the reader must not force them to 16 kHz mono. +static void test_zero_target_keeps_the_native_format() { + const auto path = scratch_dir() / "input-native.wav"; + write_audio_file(path.string(), tone(44100, 2, 0.25f)); + + const auto audio = read_audio_file(path.string(), 0); + check(audio.sample_rate == 44100, "a zero target keeps the file's rate"); + check(audio.channels == 2, "a zero target keeps the file's channels"); +} + +static void test_missing_file_is_a_config_error() { + bool threw_config_error = false; + try { + read_audio_file((scratch_dir() / "does-not-exist.wav").string(), 16000); + } catch (const ConfigError &) { + threw_config_error = true; + } catch (const std::exception &) { + // Any other type maps to INTERNAL, which is what this asserts against. + } + check(threw_config_error, "a missing input file is INVALID_ARGUMENT, not INTERNAL"); +} + +static void test_unreadable_file_is_a_config_error() { + const auto path = scratch_dir() / "not-a-wav.wav"; + { + FILE *file = fopen(path.string().c_str(), "wb"); + if (file != nullptr) { + fputs("this is not a RIFF header", file); + fclose(file); + } + } + bool threw_config_error = false; + try { + read_audio_file(path.string(), 16000); + } catch (const ConfigError &) { + threw_config_error = true; + } catch (const std::exception &) { + } + check(threw_config_error, "a non-WAV input is INVALID_ARGUMENT, not INTERNAL"); +} + +int main() { + test_441k_stereo_is_read_as_16k_mono(); + test_48k_is_read_as_16k(); + test_16k_mono_passes_through_unchanged(); + test_zero_target_keeps_the_native_format(); + test_missing_file_is_a_config_error(); + test_unreadable_file_is_a_config_error(); + if (failures) { + fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + fprintf(stderr, "all audio_io checks passed\n"); + return 0; +} diff --git a/backend/cpp/audio-cpp/grpc-server.cpp b/backend/cpp/audio-cpp/grpc-server.cpp index 73e6ca1bd..ef2f7f1dc 100644 --- a/backend/cpp/audio-cpp/grpc-server.cpp +++ b/backend/cpp/audio-cpp/grpc-server.cpp @@ -5,8 +5,8 @@ // src/ or tests/ is used: those are application internals, they are where // upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT. // -// This commit adds LoadModel/Free/Status plus the VAD and Diarize RPCs. The -// remaining audio RPCs land in later commits. +// This commit adds LoadModel/Free/Status plus the AudioTranscription, VAD and +// Diarize RPCs. The remaining audio RPCs land in later commits. #include "backend.pb.h" #include "backend.grpc.pb.h" @@ -17,6 +17,7 @@ #include "inference_lane.h" #include "loaded_model.h" #include "model_options.h" +#include "result_map.h" #include #include @@ -91,6 +92,34 @@ std::shared_ptr snapshot_unchecked() { // constant to delete. constexpr int kVadSampleRate = 16000; +// The rate every file-fed speech route reads its input at, and the rate the +// spans that come back are therefore interpreted in. Passed to read_audio_file, +// which resamples only when the file differs. +// +// Two independent reasons, and the second is the one that is easy to miss: +// +// 1. Some families refuse anything else outright. silero_vad throws +// "Silero VAD 16k model only supports sample_rate=16000" and +// sortformer_diar throws "Sortformer diar currently requires 16 kHz input +// audio". That is what made a 44.1 kHz upload return INTERNAL with an +// engine-internal message instead of an answer. +// 2. The families that do NOT refuse still do not all express their result +// spans in the input's domain. nemotron_asr builds every word timestamp as +// token_frame * hop_length * subsampling_factor, which is its own 16 kHz +// feature domain whatever the input was; the handler then converts those +// spans with the buffer's rate. Feed it 44.1 kHz and every emitted +// timestamp is 2.76x too small, with a 200 and no diagnostic. Resampling +// the input to 16 kHz makes the buffer domain and the span domain the same +// one, which is the only reason the nanoseconds are right. +// +// The cost, stated plainly: vibevoice_asr resamples internally to 24 kHz, so a +// 48 kHz upload now travels 48 -> 16 -> 24 rather than 48 -> 24, losing the +// 8-12 kHz band it could have kept. That is accepted because a silently wrong +// timestamp is worse than a band-limited one, and because every other LocalAI +// ASR path already feeds 16 kHz. If the framework ever publishes a per-model +// preferred input rate, this constant is what should become that lookup. +constexpr int kSpeechSampleRate = 16000; + // Parses ModelOptions.MainGPU into a device index. // // Not std::atoi: it returns 0 for anything unparseable, so "gpu1" or a device @@ -193,6 +222,71 @@ snapshot_for(const Request *request, GStatus &out) { return model; } +// Builds the TaskRequest for a transcription-shaped RPC. `task` is the ROUTED +// audio.cpp task, not a guess from the request: it decides how +// TranscriptRequest.prompt is used, and routing has already decided it. +// +// The audio is taken by value and moved in. A long recording runs to tens of +// megabytes and the caller has no use for it afterwards; the previous shape, +// a const reference, copied it. +engine::runtime::TaskRequest +build_transcription_request(const backend::TranscriptRequest &request, + audiocpp_backend::Task task, + engine::runtime::AudioBuffer audio) { + engine::runtime::TaskRequest task_request; + task_request.audio_input = std::move(audio); + + if (task == audiocpp_backend::Task::Alignment) { + // For forced alignment the prompt IS the transcript to align, so it + // becomes the text input rather than a decoding hint. Set even when + // empty: an aligner given no text should say so itself rather than be + // handed an audio-only request it cannot describe. + engine::runtime::Transcript transcript; + transcript.text = request.prompt(); + transcript.language = request.language(); + task_request.text_input = transcript; + } else if (!request.prompt().empty()) { + // For ASR the prompt is decoding context, the whisper meaning. + task_request.options["prompt"] = request.prompt(); + } + + if (!request.language().empty()) { + task_request.options["language"] = request.language(); + } + if (request.translate()) { + task_request.options["translate"] = "true"; + } + if (request.temperature() > 0.0f) { + task_request.options["temperature"] = std::to_string(request.temperature()); + } + for (const auto &granularity : request.timestamp_granularities()) { + // Upstream families read this as a request for word-level timing. + if (granularity == "word") { + task_request.options["word_timestamps"] = "true"; + } + } + // Every option above is advisory. audio.cpp families look their request + // options up by name (runtime::find_option) and ignore the rest; the + // unknown-key refusals upstream does have are on SESSION options, which + // arrive at load time, not here. So an option a family does not read costs + // nothing, and none of these can turn a valid request into an error. + // + // TranscriptRequest.threads is deliberately NOT forwarded: thread count is + // a SessionOptions field fixed when the session was created, so a + // per-request value has nowhere to go and pretending otherwise would be a + // knob that silently does nothing. + return task_request; +} + +// Duration of a possibly multi-channel buffer, in seconds. Frames, not floats: +// a stereo buffer holds two floats per position and would otherwise report +// twice its real length. +float audio_duration_seconds(const engine::runtime::AudioBuffer &audio) { + const std::int64_t frames = audiocpp_backend::interleaved_frame_count( + audio.samples.size(), audio.channels); + return audiocpp_backend::samples_to_seconds(frames, audio.sample_rate); +} + // Maps a thrown exception onto the gRPC status the client should see. GStatus to_status(const std::exception &err) { if (dynamic_cast(&err) != nullptr) { @@ -316,6 +410,63 @@ public: return GStatus::OK; } + GStatus AudioTranscription(ServerContext *, + const backend::TranscriptRequest *request, + backend::TranscriptResult *response) override { + try { + GStatus refusal = GStatus::OK; + const auto model = snapshot_for(request, refusal); + if (model == nullptr) { + return refusal; + } + + audiocpp_backend::RequestShape shape; + // has_prompt_text is what lets a family that can only align be + // reached through this RPC at all, so it is not decoration. + shape.has_prompt_text = !request->prompt().empty(); + shape.pinned_task = model->pinned_task(); + + // Capability refusal before the lane and before the file read, for + // the reasons spelled out in Diarize. + model->check_can_serve(audiocpp_backend::Rpc::AudioTranscription, shape); + + // TranscriptRequest.dst is the INPUT audio path, despite the field + // name. The HTTP layer materialises the upload to a temp file and + // passes the path; nothing is written back. + auto audio = audiocpp_backend::read_audio_file(request->dst(), + kSpeechSampleRate); + // Both read before the buffer is moved into the request below. The + // rate is the BUFFER's, which after read_audio_file is + // kSpeechSampleRate and not necessarily the file's, and it is the + // domain the result spans come back in. + const int sample_rate = audio.sample_rate; + const float duration = audio_duration_seconds(audio); + + audiocpp_backend::LaneEntry lane = model->acquire(0); + const auto session = model->session_for( + audiocpp_backend::Rpc::AudioTranscription, shape, lane); + + // session.task, not the request: for Asr the prompt is decoding + // context, for Alignment it is the transcript to align, and routing + // has already decided which of those this is. + const auto task_request = + build_transcription_request(*request, session.task, std::move(audio)); + const auto result = + audiocpp_backend::run_offline(session, task_request, lane); + + // text comes from result.text_output verbatim, never from the + // segments. See THE RULE in result_map.h. + audiocpp_backend::fill_transcript_result(result, sample_rate, duration, + response); + // eou stays false. It marks a decode that ended on the model's + // end-of-utterance token, which is a cache-aware STREAMING concept; + // an offline run over a whole file has no turn to yield. + return GStatus::OK; + } catch (const std::exception &err) { + return to_status(err); + } + } + // Verifying this by hand: silero_vad is a SPEECH detector, so a synthetic // stimulus does not exercise it. A 220 Hz sine, broadband noise and a // harmonic buzz all return zero segments, correctly. Use real speech, which @@ -344,6 +495,12 @@ public: // otherwise derived from the RPC alone. shape.pinned_task = model->pinned_task(); + // Before the lane. A model that cannot do VAD at all answers + // immediately instead of queueing behind somebody else's run only + // to be refused; routing is pure and needs no lane. session_for + // routes again below and reaches the same answer. + model->check_can_serve(audiocpp_backend::Rpc::Vad, shape); + engine::runtime::TaskRequest task; task.audio_input = audiocpp_backend::buffer_from_mono( std::vector(request->audio().begin(), request->audio().end()), @@ -386,21 +543,30 @@ public: audiocpp_backend::RequestShape shape; shape.pinned_task = model->pinned_task(); - // Lane then route then read, in that order, and the read is last on - // purpose. A family that cannot diarize at all should say so, not - // complain about the input file first: on a VAD-only model, reading - // the audio would surface "cannot read /tmp/x.wav" and send the - // operator hunting a file problem instead of a model choice. The - // read costs a lane wait in exchange, which is the same wait every - // capability refusal already pays because session_for needs the lane. - audiocpp_backend::LaneEntry lane = model->acquire(0); - const auto session = - model->session_for(audiocpp_backend::Rpc::Diarize, shape, lane); + // Route, then read, then lane, in that order, and every step of it + // is deliberate. + // + // The capability refusal comes first because a family that cannot + // diarize at all should say so, not complain about the input file: + // on a VAD-only model, reading the audio first would surface + // "cannot read /tmp/x.wav" and send the operator hunting a file + // problem instead of a model choice. + // + // It also comes before the lane, which is the part that used to be + // impossible. Routing needs no lane, so the refusal no longer waits + // out somebody else's thirty second run to be told no, and neither + // does the file read. + model->check_can_serve(audiocpp_backend::Rpc::Diarize, shape); // DiarizeRequest.dst is the INPUT path, despite the field name: the // HTTP layer materialises the upload to a temp file and passes the // path here. Nothing is written back. - auto audio = audiocpp_backend::read_audio_file(request->dst()); + // + // Read at 16 kHz mono: sortformer refuses any other rate, and the + // HTTP layer copies the upload byte for byte, so whatever the user + // posted is what arrives. See kSpeechSampleRate. + auto audio = audiocpp_backend::read_audio_file(request->dst(), + kSpeechSampleRate); const int sample_rate = audio.sample_rate; // Read off the buffer before it is moved into the request below. // Frames, not floats: a stereo input holds two floats per position @@ -408,6 +574,10 @@ public: const std::int64_t frames = audiocpp_backend::interleaved_frame_count( audio.samples.size(), audio.channels); + audiocpp_backend::LaneEntry lane = model->acquire(0); + const auto session = + model->session_for(audiocpp_backend::Rpc::Diarize, shape, lane); + engine::runtime::TaskRequest task; // Moved, not copied: a long recording runs to tens of megabytes and // this is its only owner. @@ -481,13 +651,13 @@ public: // Seconds, like VADSegment. Only TranscriptSegment/Word are ns. // // Converted against the INPUT rate, which is correct and has - // been checked against upstream rather than assumed: sortformer - // refuses any input whose rate differs from its feature config - // (frontend.cpp throws), and every TimeSpan it emits is built - // as llround(seconds * 16000.0) (postprocess.cpp). So the span - // domain and the input domain are the same 16 kHz, and this is - // not a place where a resampling family could silently scale - // every timestamp by a constant. + // been checked against upstream rather than assumed: every + // TimeSpan sortformer emits is built as + // llround(seconds * 16000.0) (postprocess.cpp). The read above + // guarantees the buffer is 16 kHz, so the span domain and the + // input domain are the same one, and this is not a place where + // a resampling family could silently scale every timestamp by a + // constant. out->set_start(audiocpp_backend::samples_to_seconds( turn.span.start_sample, sample_rate)); out->set_end(audiocpp_backend::samples_to_seconds( diff --git a/backend/cpp/audio-cpp/loaded_model.cpp b/backend/cpp/audio-cpp/loaded_model.cpp index 9be67a17d..bdcb376f9 100644 --- a/backend/cpp/audio-cpp/loaded_model.cpp +++ b/backend/cpp/audio-cpp/loaded_model.cpp @@ -337,6 +337,13 @@ LoadedModel::LoadedModel(const std::string &resolved_path, capabilities_ = to_capabilities(family, engine_caps); } +void LoadedModel::check_can_serve(Rpc rpc, const RequestShape &shape) const { + const Route route = resolve_route(rpc, shape, capabilities_); + if (!route.ok) { + throw CapabilityError(route.error); + } +} + LoadedModel::Session LoadedModel::session_for(Rpc rpc, const RequestShape &shape, LaneEntry &lane) { // Proof of holding only. Nothing here reads it, and nothing should: its diff --git a/backend/cpp/audio-cpp/loaded_model.h b/backend/cpp/audio-cpp/loaded_model.h index d0711a38a..f4f0a137a 100644 --- a/backend/cpp/audio-cpp/loaded_model.h +++ b/backend/cpp/audio-cpp/loaded_model.h @@ -98,6 +98,25 @@ public: // from the load to the request that honours it. const std::string &pinned_task() const noexcept { return pinned_task_; } + // Throws the same CapabilityError session_for would throw when this family + // cannot serve the RPC, and does nothing otherwise. + // + // It exists so a refusal does not have to buy a place in the queue first. + // resolve_route is a pure function of capabilities_, which is fixed at + // construction and never written again, so unlike the session cache it + // needs no lane and no lock: a model that cannot transcribe can say so + // while another request is halfway through a thirty second run. Without + // this the refusal waits for that run to finish only to be told no. + // + // It does NOT replace the routing inside session_for, and must not be made + // to: session_for still needs the route to key the session cache. The two + // calls agree because both read the same immutable capabilities. What this + // one adds is only the ordering, so call it before acquire(). + // + // Const and lane-free on purpose. If a future edit makes routing depend on + // mutable state, this must grow the lane parameter its siblings carry. + void check_can_serve(Rpc rpc, const RequestShape &shape) const; + // Routes the RPC and returns the cached session, creating it on first use. // Throws CapabilityError when this family cannot serve the RPC, and a plain // runtime_error when it can but the session could not be built, which is an diff --git a/backend/cpp/audio-cpp/result_map.cpp b/backend/cpp/audio-cpp/result_map.cpp new file mode 100644 index 000000000..3b4fb58a0 --- /dev/null +++ b/backend/cpp/audio-cpp/result_map.cpp @@ -0,0 +1,83 @@ +#include "result_map.h" + +#include "transcript_assembly.h" + +#include +#include + +namespace audiocpp_backend { + +void fill_transcript_result(const engine::runtime::TaskResult &result, + int sample_rate, float duration_seconds, + backend::TranscriptResult *out) { + if (out == nullptr) { + return; + } + + std::vector speech_segments; + speech_segments.reserve(result.speech_segments.size()); + for (const auto &segment : result.speech_segments) { + speech_segments.push_back( + Span{segment.span.start_sample, segment.span.end_sample}); + } + + std::vector speaker_turns; + speaker_turns.reserve(result.speaker_turns.size()); + for (const auto &turn : result.speaker_turns) { + speaker_turns.push_back( + SpeakerSpan{Span{turn.span.start_sample, turn.span.end_sample}, + turn.speaker_id}); + } + + std::vector words; + words.reserve(result.word_timestamps.size()); + for (const auto &word : result.word_timestamps) { + words.push_back( + WordSpan{Span{word.span.start_sample, word.span.end_sample}, + word.word}); + } + + // The ONLY read of transcript text in this function, and the only one there + // may ever be. See THE RULE in the header. + const std::string text = + result.text_output.has_value() ? result.text_output->text : std::string(); + + const AssembledTranscript assembled = assemble_transcript( + text, speech_segments, speaker_turns, words, sample_rate); + + out->set_text(assembled.text); + // language has no source inside transcript_assembly, which is span-shaped + // only, so it is read straight off the engine result here. Left untouched + // when the family reported no text output at all: an empty string would be + // indistinguishable from a family that genuinely detected no language, and + // the field is documented as optional. + if (result.text_output.has_value()) { + out->set_language(result.text_output->language); + } + out->set_duration(duration_seconds); + + // Cleared rather than appended to. A caller that fills the same message + // twice (a stream's final_result being rebuilt, say) would otherwise emit + // every segment twice, and the second call's ids would restart at 0 and + // collide with the first call's. + out->clear_segments(); + for (const auto &segment : assembled.segments) { + auto *out_segment = out->add_segments(); + out_segment->set_id(segment.id); + // NANOSECONDS. TranscriptSegment and TranscriptWord are the only + // messages in backend.proto that use them; VADSegment and DiarizeSegment + // are float seconds. assemble_transcript has already converted. + out_segment->set_start(segment.start_ns); + out_segment->set_end(segment.end_ns); + out_segment->set_text(segment.text); + out_segment->set_speaker(segment.speaker); + for (const auto &word : segment.words) { + auto *out_word = out_segment->add_words(); + out_word->set_start(word.start_ns); + out_word->set_end(word.end_ns); + out_word->set_text(word.text); + } + } +} + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/result_map.h b/backend/cpp/audio-cpp/result_map.h new file mode 100644 index 000000000..c3d1a4ba3 --- /dev/null +++ b/backend/cpp/audio-cpp/result_map.h @@ -0,0 +1,36 @@ +#pragma once + +// Converts engine::runtime results into LocalAI proto messages. All of the +// non-trivial shaping lives in transcript_assembly, which is stdlib-only and +// unit tested; this unit is the thin engine-typed boundary around it. + +#include "backend.pb.h" + +#include "engine/framework/runtime/session.h" + +namespace audiocpp_backend { + +// Fills text, language, duration, segments and per-segment words. +// +// THE RULE: the top-level text is TaskResult.text_output verbatim. It is never +// derived from segments or words. audio.cpp carries transcript text in +// text_output and nowhere else: speech_segments, speaker_turns and +// word_timestamps carry spans and labels and no text at all. Deriving the +// transcript from them therefore returns an EMPTY text for every producer that +// reports segments without word timing, which real VibeVoice diarized ASR does. +// An earlier attempt at this backend shipped exactly that bug. assemble_transcript +// enforces the rule and is heavily tested; this unit's job is not to re-derive +// it but to not undo it at the proto boundary. +// +// `sample_rate` is the rate the result's spans are expressed in, which is the +// rate of the AudioBuffer that was handed to the session, NOT the rate of the +// file the caller uploaded. Those differ whenever read_audio_file resampled, +// which is why the handler passes the buffer's rate rather than the file's. +// +// Segments are replaced, not appended to, so a message filled twice does not +// accumulate. +void fill_transcript_result(const engine::runtime::TaskResult &result, + int sample_rate, float duration_seconds, + backend::TranscriptResult *out); + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/result_map_ctest.cpp b/backend/cpp/audio-cpp/result_map_ctest.cpp new file mode 100644 index 000000000..fc12016da --- /dev/null +++ b/backend/cpp/audio-cpp/result_map_ctest.cpp @@ -0,0 +1,247 @@ +// Tests for result_map, the engine-to-proto boundary. +// +// NAMED _ctest AND NOT _test ON PURPOSE. backend/cpp/run-unit-tests.sh globs +// every *_test.cpp under backend/cpp/ and compiles it as a single standalone +// translation unit with no include path beyond its own directory. This file +// needs backend.pb.h and the audio.cpp framework headers, so it is built and +// run by ctest instead: +// +// make -C backend/cpp/audio-cpp test-engine +// +// Renaming it to *_test.cpp would break the standalone suite for every backend. +// +// What is worth testing here is exactly one thing, and it is not the field +// copying: THE RULE. TaskResult carries transcript text in text_output and +// nowhere else, so the proto's text must be that string verbatim. An earlier +// attempt at this backend derived it from the segments, which returns an empty +// transcript for every producer that reports segments without word timing. +// transcript_assembly already enforces the rule and is tested on its own; these +// checks are here so that a future edit cannot undo it at the boundary. + +#include "result_map.h" + +#include +#include + +static int failures = 0; + +static void check(bool ok, const std::string &name) { + if (!ok) { + failures++; + fprintf(stderr, "FAIL: %s\n", name.c_str()); + } else { + fprintf(stderr, "ok: %s\n", name.c_str()); + } +} + +using namespace audiocpp_backend; +namespace rt = engine::runtime; + +static const int kRate = 16000; + +static rt::SpeechSegment speech(std::int64_t start, std::int64_t end) { + rt::SpeechSegment segment; + segment.span.start_sample = start; + segment.span.end_sample = end; + return segment; +} + +static rt::SpeakerTurn turn(std::int64_t start, std::int64_t end, + const std::string &speaker) { + rt::SpeakerTurn out; + out.span.start_sample = start; + out.span.end_sample = end; + out.speaker_id = speaker; + return out; +} + +static rt::WordTimestamp word(std::int64_t start, std::int64_t end, + const std::string &text) { + rt::WordTimestamp out; + out.span.start_sample = start; + out.span.end_sample = end; + out.word = text; + return out; +} + +// THE REGRESSION. A diarized ASR result: real text, real speaker turns, and no +// word timing at all. This is the vibevoice_asr shape, and it is the one that +// came back empty before. +static void test_text_survives_segments_without_words() { + rt::TaskResult result; + rt::Transcript transcript; + transcript.text = "hello there general kenobi"; + transcript.language = "en"; + result.text_output = transcript; + result.speaker_turns.push_back(turn(0, 16000, "speaker_0")); + result.speaker_turns.push_back(turn(16000, 32000, "speaker_1")); + + backend::TranscriptResult out; + fill_transcript_result(result, kRate, 2.0f, &out); + + check(out.text() == "hello there general kenobi", + "diarized result keeps text_output verbatim"); + check(out.language() == "en", "language comes from text_output"); + check(out.segments_size() == 2, "both speaker turns become segments"); + if (out.segments_size() == 2) { + check(out.segments(0).speaker() == "speaker_0", + "first segment keeps its own speaker label"); + check(out.segments(1).speaker() == "speaker_1", + "second segment keeps its own speaker label"); + check(out.segments(1).start() == 1000000000LL, + "segment start is nanoseconds, not samples"); + check(out.segments(1).end() == 2000000000LL, + "segment end is nanoseconds, not samples"); + } +} + +// The same rule seen from the other side: text present, spans present, and the +// per-segment text empty because there is nothing truthful to split. A boundary +// that derived the top-level text from these segments would produce "". +static void test_speech_segments_do_not_supply_the_text() { + rt::TaskResult result; + rt::Transcript transcript; + transcript.text = "one two three"; + result.text_output = transcript; + result.speech_segments.push_back(speech(0, 8000)); + result.speech_segments.push_back(speech(8000, 16000)); + + backend::TranscriptResult out; + fill_transcript_result(result, kRate, 1.0f, &out); + + check(out.text() == "one two three", + "speech segments without words do not empty the transcript"); + check(out.segments_size() == 2, "both speech segments are emitted"); + if (out.segments_size() == 2) { + check(out.segments(0).text().empty() && out.segments(1).text().empty(), + "per-segment text stays empty when there is no word timing"); + } +} + +static void test_words_reach_the_proto_in_nanoseconds() { + rt::TaskResult result; + rt::Transcript transcript; + transcript.text = "hi there"; + result.text_output = transcript; + result.word_timestamps.push_back(word(0, 8000, "hi")); + result.word_timestamps.push_back(word(8000, 16000, "there")); + + backend::TranscriptResult out; + fill_transcript_result(result, kRate, 1.0f, &out); + + check(out.text() == "hi there", "word-timed result keeps text_output"); + check(out.segments_size() == 1, "words with no spans yield one covering segment"); + if (out.segments_size() == 1) { + const auto &segment = out.segments(0); + check(segment.words_size() == 2, "both words are emitted"); + if (segment.words_size() == 2) { + check(segment.words(0).text() == "hi", "first word text"); + check(segment.words(0).start() == 0, "first word start"); + check(segment.words(0).end() == 500000000LL, + "first word end is 0.5 s in nanoseconds"); + check(segment.words(1).start() == 500000000LL, "second word start"); + check(segment.words(1).end() == 1000000000LL, "second word end"); + } + } +} + +// The buffer's rate, not the file's, is what the spans mean. Passing 8000 for +// the same spans has to halve every timestamp, which is what makes resampling +// the input at read time load-bearing rather than cosmetic. +static void test_sample_rate_scales_the_timestamps() { + rt::TaskResult result; + rt::Transcript transcript; + transcript.text = "x"; + result.text_output = transcript; + result.speech_segments.push_back(speech(0, 8000)); + + backend::TranscriptResult out; + fill_transcript_result(result, 8000, 1.0f, &out); + + check(out.segments_size() == 1, "one segment at 8 kHz"); + if (out.segments_size() == 1) { + check(out.segments(0).end() == 1000000000LL, + "8000 samples at 8 kHz is one second"); + } +} + +static void test_duration_is_carried_through() { + rt::TaskResult result; + rt::Transcript transcript; + transcript.text = "x"; + result.text_output = transcript; + + backend::TranscriptResult out; + fill_transcript_result(result, kRate, 14.07f, &out); + + check(out.duration() > 14.06f && out.duration() < 14.08f, + "duration is set from the argument"); +} + +// No text output at all. A VAD-shaped result reaching this boundary must not +// invent a transcript, and must not overwrite a language the caller had already +// decided on. +static void test_missing_text_output_leaves_language_alone() { + rt::TaskResult result; + result.speech_segments.push_back(speech(0, 16000)); + + backend::TranscriptResult out; + out.set_language("it"); + fill_transcript_result(result, kRate, 1.0f, &out); + + check(out.text().empty(), "no text_output means no text"); + check(out.language() == "it", + "a result with no text_output does not clear the language"); + check(out.segments_size() == 1, "spans are still emitted"); +} + +// Filling the same message twice must replace, not accumulate: the second +// call's ids restart at 0 and would collide with the first call's. +static void test_refilling_replaces_the_segments() { + rt::TaskResult first; + rt::Transcript transcript; + transcript.text = "first"; + first.text_output = transcript; + first.speech_segments.push_back(speech(0, 16000)); + first.speech_segments.push_back(speech(16000, 32000)); + + backend::TranscriptResult out; + fill_transcript_result(first, kRate, 2.0f, &out); + + rt::TaskResult second; + rt::Transcript replacement; + replacement.text = "second"; + second.text_output = replacement; + second.speech_segments.push_back(speech(0, 16000)); + fill_transcript_result(second, kRate, 1.0f, &out); + + check(out.text() == "second", "the second fill replaces the text"); + check(out.segments_size() == 1, + "the second fill replaces the segments instead of appending"); +} + +static void test_empty_result_is_empty() { + rt::TaskResult result; + backend::TranscriptResult out; + fill_transcript_result(result, kRate, 0.0f, &out); + + check(out.text().empty(), "empty result has no text"); + check(out.segments_size() == 0, "empty result has no segments"); +} + +int main() { + test_text_survives_segments_without_words(); + test_speech_segments_do_not_supply_the_text(); + test_words_reach_the_proto_in_nanoseconds(); + test_sample_rate_scales_the_timestamps(); + test_duration_is_carried_through(); + test_missing_text_output_leaves_language_alone(); + test_refilling_replaces_the_segments(); + test_empty_result_is_empty(); + if (failures) { + fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + fprintf(stderr, "all result_map checks passed\n"); + return 0; +}