mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-31 10:28:43 -04:00
Code Issues Packages Projects Releases Wiki Activity

backend(audio-cpp): serve the AudioTranscription RPC

Browse Source 
Adds result_map, the engine-to-proto boundary, and wires the offline
transcription RPC.

The handler branches on the ROUTED task: for Asr the request's prompt is
whisper-style decoding context and becomes a request option, for Alignment
the same field IS the transcript to align and becomes the text input.
Routing has already decided which.

The result text is TaskResult.text_output verbatim and is never derived
from the segments. audio.cpp carries transcript text in text_output and
nowhere else, so deriving it returns an empty transcript for every
producer that reports segments without word timing. transcript_assembly
already enforces that; this commit's job is not to undo it at the proto
boundary, and result_map_ctest pins it there.

read_audio_file now takes the sample rate the caller needs. Both file-fed
speech handlers ask for 16 kHz mono, for two reasons: silero_vad and
sortformer_diar refuse anything else outright, which turned an ordinary
44.1 kHz upload into INTERNAL, and nemotron_asr emits word timestamps in
its own 16 kHz feature domain whatever the input was, so only a 16 kHz
buffer makes the emitted nanoseconds right. Zero keeps the file's native
rate and channels, which is what source separation will need.

LoadedModel::check_can_serve answers a capability refusal before the lane
is taken and before the input file is read. Routing is a pure read of the
immutable capabilities, so a model that cannot serve an RPC no longer
waits out somebody else's run to say so. VAD and Diarize use it too.

Assisted-by: Claude:claude-opus-5 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2026-07-26 16:11:45 +00:00
parent 51dec81e9a
commit 6ee95ab01b
10 changed files with 866 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

50
backend/cpp/audio-cpp/CMakeLists.txt
View File

@@ -93,6 +93,7 @@ add_executable(${TARGET}
audio_io.cpp
audio_units.cpp
transcript_assembly.cpp
result_map.cpp
inference_lane.cpp
)
@@ -137,4 +138,53 @@ set_target_properties(${TARGET} PROPERTIES
if(AUDIO_CPP_GRPC_BUILD_TESTS)
enable_testing()
# These are the units whose tests CANNOT run under
# backend/cpp/run-unit-tests.sh, because that script compiles each
# *_test.cpp standalone with no protobuf and no audio.cpp include path.
# They are named *_ctest.cpp so the script's glob does not pick them up and
# fail every backend's suite; everything that can be stdlib-only still is,
# and still lives in a *_test.cpp beside its unit.
#
# -Wall -Wextra -Wpedantic here and not on ${TARGET}: upstream's own
# add_compile_options is a property of its subdirectory and does not reach
# ours, so without naming them the tests would build as quietly as
# everything else.
add_executable(result_map_ctest
result_map_ctest.cpp
result_map.cpp
transcript_assembly.cpp
audio_units.cpp)
target_include_directories(result_map_ctest PRIVATE
"${AUDIO_CPP_DIR}/include"
"${CMAKE_CURRENT_SOURCE_DIR}"
# session.h reaches ggml.h through core/backend.h. Every other target
# here inherits that directory from the ggml target it links; this one
# links no ggml, so it has to name it.
"${AUDIO_CPP_DIR}/external/ggml/include")
# No engine_runtime: result_map touches only the plain structs in
# engine/framework/runtime/session.h, so the header is all it needs.
target_link_libraries(result_map_ctest PRIVATE
hw_grpc_proto
protobuf::libprotobuf
Threads::Threads)
target_compile_options(result_map_ctest PRIVATE -Wall -Wextra -Wpedantic)
add_test(NAME result_map COMMAND result_map_ctest)
add_executable(audio_io_ctest
audio_io_ctest.cpp
audio_io.cpp)
target_include_directories(audio_io_ctest PRIVATE
"${AUDIO_CPP_DIR}/include"
"${CMAKE_CURRENT_SOURCE_DIR}")
target_link_libraries(audio_io_ctest PRIVATE
engine_runtime
ggml
Threads::Threads)
target_compile_options(audio_io_ctest PRIVATE -Wall -Wextra -Wpedantic)
set_target_properties(audio_io_ctest PROPERTIES
BUILD_RPATH "$ORIGIN"
INSTALL_RPATH "$ORIGIN"
BUILD_WITH_INSTALL_RPATH TRUE)
add_test(NAME audio_io COMMAND audio_io_ctest)
endif()

51
backend/cpp/audio-cpp/audio_io.cpp
View File

@@ -2,15 +2,18 @@
#include "loaded_model.h"
#include "engine/framework/audio/conversion.h"
#include "engine/framework/audio/wav_reader.h"
#include "engine/framework/audio/wav_writer.h"
#include <filesystem>
#include <string>
#include <utility>
namespace audiocpp_backend {
engine::runtime::AudioBuffer read_audio_file(const std::string &path) {
engine::runtime::AudioBuffer read_audio_file(const std::string &path,
int target_sample_rate) {
if (path.empty()) {
throw ConfigError("audio-cpp: no input audio path was supplied");
}
@@ -28,24 +31,52 @@ engine::runtime::AudioBuffer read_audio_file(const std::string &path) {
if (!present) {
throw ConfigError("audio-cpp: input audio does not exist: " + path);
}
engine::runtime::AudioBuffer buffer;
engine::audio::WavData wav;
try {
const engine::audio::WavData wav =
engine::audio::read_wav_f32(std::filesystem::path(path));
buffer.sample_rate = wav.sample_rate;
// AudioBuffer's own default is 1, and a reader that reports 0 channels
// still gave us an interleaving of one.
buffer.channels = wav.channels > 0 ? wav.channels : 1;
buffer.samples = wav.samples;
wav = engine::audio::read_wav_f32(std::filesystem::path(path));
} catch (const std::exception &err) {
throw ConfigError("audio-cpp: cannot read " + path +
" as WAV: " + err.what());
}
if (buffer.sample_rate <= 0) {
if (wav.sample_rate <= 0) {
throw ConfigError("audio-cpp: " + path +
" declares a non-positive sample rate; every "
"timestamp derived from it would be zero");
}
// AudioBuffer's own default is 1, and a reader that reports 0 channels
// still gave us an interleaving of one. Normalised before the conversion
// below rather than after, because mixdown_interleaved_to_mono_average
// throws on a non-positive channel count.
if (wav.channels <= 0) {
wav.channels = 1;
}
engine::runtime::AudioBuffer buffer;
if (target_sample_rate <= 0) {
buffer.sample_rate = wav.sample_rate;
buffer.channels = wav.channels;
buffer.samples = std::move(wav.samples);
return buffer;
}
buffer.sample_rate = target_sample_rate;
buffer.channels = 1;
try {
// A no-op copy when the rates already match, so the common 16 kHz
// upload pays only the mono mixdown it would have paid inside the
// family anyway.
buffer.samples =
engine::audio::convert_wav_to_mono_linear_resampled(wav, target_sample_rate);
} catch (const std::exception &err) {
// ConfigError, so this is INVALID_ARGUMENT rather than INTERNAL. What
// reaches here is a malformed input: a sample count that is not a whole
// number of frames is the realistic one, and it is the uploader's file
// that is truncated, not this backend that is broken.
throw ConfigError("audio-cpp: cannot resample " + path + " from " +
std::to_string(wav.sample_rate) + " Hz to " +
std::to_string(target_sample_rate) +
" Hz: " + err.what());
}
return buffer;
}

30
backend/cpp/audio-cpp/audio_io.h
View File

@@ -13,16 +13,36 @@
namespace audiocpp_backend {
// Reads a WAV file at its native sample rate and channel count. Throws
// ConfigError when the file is missing, is not readable as WAV, or declares a
// non-positive sample rate: all three are user-fixable input problems rather
// than backend faults.
// Reads a WAV file. Throws ConfigError when the file is missing, is not
// readable as WAV, or declares a non-positive sample rate: all three are
// user-fixable input problems rather than backend faults.
//
// A declared sample rate of zero is refused rather than passed on, because
// every downstream conversion in audio_units answers 0 for a non-positive rate.
// Accepting it would turn a corrupt header into a response full of zero
// timestamps, which reads as a real answer.
engine::runtime::AudioBuffer read_audio_file(const std::string &path);
//
// `target_sample_rate` is the rate the CALLER needs, in Hz:
//
// 0 (or negative) keep the file's own rate and channel count.
// positive downmix to mono and resample to that rate. Resampling is
// skipped when the file already declares it, so passing the
// rate a route needs costs nothing on the common input.
//
// It is a parameter, and not a constant inside this function, because the
// routes that read audio do not agree on an answer. Speech routes want 16 kHz
// mono; source separation does not, and folding a 44.1 kHz stereo input to
// 16 kHz mono for demucs or roformer would destroy the very thing they separate
// (both refuse a rate other than their own outright). Making the caller name
// the rate keeps that decision where the route is known.
//
// Downmixing along with the resample is not an extra liberty: every family a
// positive rate is used for (silero_vad, sortformer_diar and every ASR family)
// begins by calling the same mixdown_interleaved_to_mono_average on whatever it
// is given. Doing it once here produces the identical samples and halves the
// buffer that is then moved through the request.
engine::runtime::AudioBuffer read_audio_file(const std::string &path,
int target_sample_rate);
// Writes 16-bit PCM WAV, creating parent directories. Throws ConfigError when
// the destination cannot be written.

168
backend/cpp/audio-cpp/audio_io_ctest.cpp Normal file
View File

@@ -0,0 +1,168 @@
// Tests for audio_io's reading contract, and in particular for the resampling
// that keeps a 44.1 or 48 kHz upload from reaching a family that only accepts
// 16 kHz.
//
// NAMED _ctest AND NOT _test ON PURPOSE: see the note at the top of
// result_map_ctest.cpp. This file links the audio.cpp engine, so it is built
// and run by ctest, not by backend/cpp/run-unit-tests.sh.
//
// make -C backend/cpp/audio-cpp test-engine
#include "audio_io.h"
#include "loaded_model.h"
#include <algorithm>
#include <cmath>
#include <cstdio>
#include <filesystem>
#include <string>
#include <vector>
static int failures = 0;
static void check(bool ok, const std::string &name) {
if (!ok) {
failures++;
fprintf(stderr, "FAIL: %s\n", name.c_str());
} else {
fprintf(stderr, "ok: %s\n", name.c_str());
}
}
using namespace audiocpp_backend;
// A one-second tone, interleaved across `channels`. Real audio rather than
// silence so a resample that dropped its input would be visible as a flat
// buffer, not just as a different length.
static engine::runtime::AudioBuffer tone(int sample_rate, int channels,
float seconds) {
engine::runtime::AudioBuffer buffer;
buffer.sample_rate = sample_rate;
buffer.channels = channels;
const auto frames =
static_cast<size_t>(static_cast<double>(sample_rate) * seconds);
buffer.samples.reserve(frames * static_cast<size_t>(channels));
for (size_t frame = 0; frame < frames; ++frame) {
const float value = 0.5f * std::sin(2.0f * 3.14159265f * 220.0f *
static_cast<float>(frame) /
static_cast<float>(sample_rate));
for (int channel = 0; channel < channels; ++channel) {
buffer.samples.push_back(value);
}
}
return buffer;
}
static float peak(const std::vector<float> &samples) {
float highest = 0.0f;
for (const float sample : samples) {
highest = std::max(highest, std::abs(sample));
}
return highest;
}
static std::filesystem::path scratch_dir() {
const auto dir = std::filesystem::temp_directory_path() / "audiocpp-io-ctest";
std::filesystem::create_directories(dir);
return dir;
}
// The I2 fixture. Before the resample this returned a 44.1 kHz buffer, which
// silero_vad and sortformer_diar both reject with a plain runtime_error, which
// the server maps to INTERNAL. A 44.1 kHz WAV is an ordinary upload.
static void test_441k_stereo_is_read_as_16k_mono() {
const auto path = scratch_dir() / "input-44100-stereo.wav";
write_audio_file(path.string(), tone(44100, 2, 1.0f));
const auto audio = read_audio_file(path.string(), 16000);
check(audio.sample_rate == 16000, "44.1 kHz input is resampled to 16 kHz");
check(audio.channels == 1, "stereo input is downmixed to mono");
// Linear resampling lands within a sample or two of the exact ratio.
const auto frames = static_cast<long long>(audio.samples.size());
check(frames > 15990 && frames < 16010,
"one second in stays one second out");
check(peak(audio.samples) > 0.2f,
"the resampled buffer still carries the signal");
}
static void test_48k_is_read_as_16k() {
const auto path = scratch_dir() / "input-48000-mono.wav";
write_audio_file(path.string(), tone(48000, 1, 0.5f));
const auto audio = read_audio_file(path.string(), 16000);
check(audio.sample_rate == 16000, "48 kHz input is resampled to 16 kHz");
const auto frames = static_cast<long long>(audio.samples.size());
check(frames > 7990 && frames < 8010, "half a second in, half a second out");
}
// The common case: the upload is already 16 kHz mono, and nothing is resampled.
static void test_16k_mono_passes_through_unchanged() {
const auto path = scratch_dir() / "input-16000-mono.wav";
const auto source = tone(16000, 1, 1.0f);
write_audio_file(path.string(), source);
const auto audio = read_audio_file(path.string(), 16000);
check(audio.sample_rate == 16000, "16 kHz stays 16 kHz");
check(audio.channels == 1, "mono stays mono");
check(audio.samples.size() == source.samples.size(),
"a matching rate resamples nothing");
}
// Rate 0 means "give me the file as it is", which is what a source separation
// route needs: demucs and roformer refuse anything but their own 44.1 kHz and
// work on stereo, so the reader must not force them to 16 kHz mono.
static void test_zero_target_keeps_the_native_format() {
const auto path = scratch_dir() / "input-native.wav";
write_audio_file(path.string(), tone(44100, 2, 0.25f));
const auto audio = read_audio_file(path.string(), 0);
check(audio.sample_rate == 44100, "a zero target keeps the file's rate");
check(audio.channels == 2, "a zero target keeps the file's channels");
}
static void test_missing_file_is_a_config_error() {
bool threw_config_error = false;
try {
read_audio_file((scratch_dir() / "does-not-exist.wav").string(), 16000);
} catch (const ConfigError &) {
threw_config_error = true;
} catch (const std::exception &) {
// Any other type maps to INTERNAL, which is what this asserts against.
}
check(threw_config_error, "a missing input file is INVALID_ARGUMENT, not INTERNAL");
}
static void test_unreadable_file_is_a_config_error() {
const auto path = scratch_dir() / "not-a-wav.wav";
{
FILE *file = fopen(path.string().c_str(), "wb");
if (file != nullptr) {
fputs("this is not a RIFF header", file);
fclose(file);
}
}
bool threw_config_error = false;
try {
read_audio_file(path.string(), 16000);
} catch (const ConfigError &) {
threw_config_error = true;
} catch (const std::exception &) {
}
check(threw_config_error, "a non-WAV input is INVALID_ARGUMENT, not INTERNAL");
}
int main() {
test_441k_stereo_is_read_as_16k_mono();
test_48k_is_read_as_16k();
test_16k_mono_passes_through_unchanged();
test_zero_target_keeps_the_native_format();
test_missing_file_is_a_config_error();
test_unreadable_file_is_a_config_error();
if (failures) {
fprintf(stderr, "%d check(s) failed\n", failures);
return 1;
}
fprintf(stderr, "all audio_io checks passed\n");
return 0;
}

210
backend/cpp/audio-cpp/grpc-server.cpp
View File

@@ -5,8 +5,8 @@
// src/ or tests/ is used: those are application internals, they are where
// upstream expects churn, and upstream is Apache-2.0 while LocalAI is MIT.
//
// This commit adds LoadModel/Free/Status plus the VAD and Diarize RPCs. The
// remaining audio RPCs land in later commits.
// This commit adds LoadModel/Free/Status plus the AudioTranscription, VAD and
// Diarize RPCs. The remaining audio RPCs land in later commits.
#include "backend.pb.h"
#include "backend.grpc.pb.h"
@@ -17,6 +17,7 @@
#include "inference_lane.h"
#include "loaded_model.h"
#include "model_options.h"
#include "result_map.h"
#include <grpcpp/grpcpp.h>
#include <grpcpp/server.h>
@@ -91,6 +92,34 @@ std::shared_ptr<audiocpp_backend::LoadedModel> snapshot_unchecked() {
// constant to delete.
constexpr int kVadSampleRate = 16000;
// The rate every file-fed speech route reads its input at, and the rate the
// spans that come back are therefore interpreted in. Passed to read_audio_file,
// which resamples only when the file differs.
//
// Two independent reasons, and the second is the one that is easy to miss:
//
// 1. Some families refuse anything else outright. silero_vad throws
// "Silero VAD 16k model only supports sample_rate=16000" and
// sortformer_diar throws "Sortformer diar currently requires 16 kHz input
// audio". That is what made a 44.1 kHz upload return INTERNAL with an
// engine-internal message instead of an answer.
// 2. The families that do NOT refuse still do not all express their result
// spans in the input's domain. nemotron_asr builds every word timestamp as
// token_frame * hop_length * subsampling_factor, which is its own 16 kHz
// feature domain whatever the input was; the handler then converts those
// spans with the buffer's rate. Feed it 44.1 kHz and every emitted
// timestamp is 2.76x too small, with a 200 and no diagnostic. Resampling
// the input to 16 kHz makes the buffer domain and the span domain the same
// one, which is the only reason the nanoseconds are right.
//
// The cost, stated plainly: vibevoice_asr resamples internally to 24 kHz, so a
// 48 kHz upload now travels 48 -> 16 -> 24 rather than 48 -> 24, losing the
// 8-12 kHz band it could have kept. That is accepted because a silently wrong
// timestamp is worse than a band-limited one, and because every other LocalAI
// ASR path already feeds 16 kHz. If the framework ever publishes a per-model
// preferred input rate, this constant is what should become that lookup.
constexpr int kSpeechSampleRate = 16000;
// Parses ModelOptions.MainGPU into a device index.
//
// Not std::atoi: it returns 0 for anything unparseable, so "gpu1" or a device
@@ -193,6 +222,71 @@ snapshot_for(const Request *request, GStatus &out) {
return model;
}
// Builds the TaskRequest for a transcription-shaped RPC. `task` is the ROUTED
// audio.cpp task, not a guess from the request: it decides how
// TranscriptRequest.prompt is used, and routing has already decided it.
//
// The audio is taken by value and moved in. A long recording runs to tens of
// megabytes and the caller has no use for it afterwards; the previous shape,
// a const reference, copied it.
engine::runtime::TaskRequest
build_transcription_request(const backend::TranscriptRequest &request,
audiocpp_backend::Task task,
engine::runtime::AudioBuffer audio) {
engine::runtime::TaskRequest task_request;
task_request.audio_input = std::move(audio);
if (task == audiocpp_backend::Task::Alignment) {
// For forced alignment the prompt IS the transcript to align, so it
// becomes the text input rather than a decoding hint. Set even when
// empty: an aligner given no text should say so itself rather than be
// handed an audio-only request it cannot describe.
engine::runtime::Transcript transcript;
transcript.text = request.prompt();
transcript.language = request.language();
task_request.text_input = transcript;
} else if (!request.prompt().empty()) {
// For ASR the prompt is decoding context, the whisper meaning.
task_request.options["prompt"] = request.prompt();
}
if (!request.language().empty()) {
task_request.options["language"] = request.language();
}
if (request.translate()) {
task_request.options["translate"] = "true";
}
if (request.temperature() > 0.0f) {
task_request.options["temperature"] = std::to_string(request.temperature());
}
for (const auto &granularity : request.timestamp_granularities()) {
// Upstream families read this as a request for word-level timing.
if (granularity == "word") {
task_request.options["word_timestamps"] = "true";
}
}
// Every option above is advisory. audio.cpp families look their request
// options up by name (runtime::find_option) and ignore the rest; the
// unknown-key refusals upstream does have are on SESSION options, which
// arrive at load time, not here. So an option a family does not read costs
// nothing, and none of these can turn a valid request into an error.
//
// TranscriptRequest.threads is deliberately NOT forwarded: thread count is
// a SessionOptions field fixed when the session was created, so a
// per-request value has nowhere to go and pretending otherwise would be a
// knob that silently does nothing.
return task_request;
}
// Duration of a possibly multi-channel buffer, in seconds. Frames, not floats:
// a stereo buffer holds two floats per position and would otherwise report
// twice its real length.
float audio_duration_seconds(const engine::runtime::AudioBuffer &audio) {
const std::int64_t frames = audiocpp_backend::interleaved_frame_count(
audio.samples.size(), audio.channels);
return audiocpp_backend::samples_to_seconds(frames, audio.sample_rate);
}
// Maps a thrown exception onto the gRPC status the client should see.
GStatus to_status(const std::exception &err) {
if (dynamic_cast<const audiocpp_backend::ConfigError *>(&err) != nullptr) {
@@ -316,6 +410,63 @@ public:
return GStatus::OK;
}
GStatus AudioTranscription(ServerContext *,
const backend::TranscriptRequest *request,
backend::TranscriptResult *response) override {
try {
GStatus refusal = GStatus::OK;
const auto model = snapshot_for(request, refusal);
if (model == nullptr) {
return refusal;
}
audiocpp_backend::RequestShape shape;
// has_prompt_text is what lets a family that can only align be
// reached through this RPC at all, so it is not decoration.
shape.has_prompt_text = !request->prompt().empty();
shape.pinned_task = model->pinned_task();
// Capability refusal before the lane and before the file read, for
// the reasons spelled out in Diarize.
model->check_can_serve(audiocpp_backend::Rpc::AudioTranscription, shape);
// TranscriptRequest.dst is the INPUT audio path, despite the field
// name. The HTTP layer materialises the upload to a temp file and
// passes the path; nothing is written back.
auto audio = audiocpp_backend::read_audio_file(request->dst(),
kSpeechSampleRate);
// Both read before the buffer is moved into the request below. The
// rate is the BUFFER's, which after read_audio_file is
// kSpeechSampleRate and not necessarily the file's, and it is the
// domain the result spans come back in.
const int sample_rate = audio.sample_rate;
const float duration = audio_duration_seconds(audio);
audiocpp_backend::LaneEntry lane = model->acquire(0);
const auto session = model->session_for(
audiocpp_backend::Rpc::AudioTranscription, shape, lane);
// session.task, not the request: for Asr the prompt is decoding
// context, for Alignment it is the transcript to align, and routing
// has already decided which of those this is.
const auto task_request =
build_transcription_request(*request, session.task, std::move(audio));
const auto result =
audiocpp_backend::run_offline(session, task_request, lane);
// text comes from result.text_output verbatim, never from the
// segments. See THE RULE in result_map.h.
audiocpp_backend::fill_transcript_result(result, sample_rate, duration,
response);
// eou stays false. It marks a decode that ended on the model's
// end-of-utterance token, which is a cache-aware STREAMING concept;
// an offline run over a whole file has no turn to yield.
return GStatus::OK;
} catch (const std::exception &err) {
return to_status(err);
}
}
// Verifying this by hand: silero_vad is a SPEECH detector, so a synthetic
// stimulus does not exercise it. A 220 Hz sine, broadband noise and a
// harmonic buzz all return zero segments, correctly. Use real speech, which
@@ -344,6 +495,12 @@ public:
// otherwise derived from the RPC alone.
shape.pinned_task = model->pinned_task();
// Before the lane. A model that cannot do VAD at all answers
// immediately instead of queueing behind somebody else's run only
// to be refused; routing is pure and needs no lane. session_for
// routes again below and reaches the same answer.
model->check_can_serve(audiocpp_backend::Rpc::Vad, shape);
engine::runtime::TaskRequest task;
task.audio_input = audiocpp_backend::buffer_from_mono(
std::vector<float>(request->audio().begin(), request->audio().end()),
@@ -386,21 +543,30 @@ public:
audiocpp_backend::RequestShape shape;
shape.pinned_task = model->pinned_task();
// Lane then route then read, in that order, and the read is last on
// purpose. A family that cannot diarize at all should say so, not
// complain about the input file first: on a VAD-only model, reading
// the audio would surface "cannot read /tmp/x.wav" and send the
// operator hunting a file problem instead of a model choice. The
// read costs a lane wait in exchange, which is the same wait every
// capability refusal already pays because session_for needs the lane.
audiocpp_backend::LaneEntry lane = model->acquire(0);
const auto session =
model->session_for(audiocpp_backend::Rpc::Diarize, shape, lane);
// Route, then read, then lane, in that order, and every step of it
// is deliberate.
//
// The capability refusal comes first because a family that cannot
// diarize at all should say so, not complain about the input file:
// on a VAD-only model, reading the audio first would surface
// "cannot read /tmp/x.wav" and send the operator hunting a file
// problem instead of a model choice.
//
// It also comes before the lane, which is the part that used to be
// impossible. Routing needs no lane, so the refusal no longer waits
// out somebody else's thirty second run to be told no, and neither
// does the file read.
model->check_can_serve(audiocpp_backend::Rpc::Diarize, shape);
// DiarizeRequest.dst is the INPUT path, despite the field name: the
// HTTP layer materialises the upload to a temp file and passes the
// path here. Nothing is written back.
auto audio = audiocpp_backend::read_audio_file(request->dst());
//
// Read at 16 kHz mono: sortformer refuses any other rate, and the
// HTTP layer copies the upload byte for byte, so whatever the user
// posted is what arrives. See kSpeechSampleRate.
auto audio = audiocpp_backend::read_audio_file(request->dst(),
kSpeechSampleRate);
const int sample_rate = audio.sample_rate;
// Read off the buffer before it is moved into the request below.
// Frames, not floats: a stereo input holds two floats per position
@@ -408,6 +574,10 @@ public:
const std::int64_t frames = audiocpp_backend::interleaved_frame_count(
audio.samples.size(), audio.channels);
audiocpp_backend::LaneEntry lane = model->acquire(0);
const auto session =
model->session_for(audiocpp_backend::Rpc::Diarize, shape, lane);
engine::runtime::TaskRequest task;
// Moved, not copied: a long recording runs to tens of megabytes and
// this is its only owner.
@@ -481,13 +651,13 @@ public:
// Seconds, like VADSegment. Only TranscriptSegment/Word are ns.
//
// Converted against the INPUT rate, which is correct and has
// been checked against upstream rather than assumed: sortformer
// refuses any input whose rate differs from its feature config
// (frontend.cpp throws), and every TimeSpan it emits is built
// as llround(seconds * 16000.0) (postprocess.cpp). So the span
// domain and the input domain are the same 16 kHz, and this is
// not a place where a resampling family could silently scale
// every timestamp by a constant.
// been checked against upstream rather than assumed: every
// TimeSpan sortformer emits is built as
// llround(seconds * 16000.0) (postprocess.cpp). The read above
// guarantees the buffer is 16 kHz, so the span domain and the
// input domain are the same one, and this is not a place where
// a resampling family could silently scale every timestamp by a
// constant.
out->set_start(audiocpp_backend::samples_to_seconds(
turn.span.start_sample, sample_rate));
out->set_end(audiocpp_backend::samples_to_seconds(

7
backend/cpp/audio-cpp/loaded_model.cpp
View File

@@ -337,6 +337,13 @@ LoadedModel::LoadedModel(const std::string &resolved_path,
capabilities_ = to_capabilities(family, engine_caps);
}
void LoadedModel::check_can_serve(Rpc rpc, const RequestShape &shape) const {
const Route route = resolve_route(rpc, shape, capabilities_);
if (!route.ok) {
throw CapabilityError(route.error);
}
}
LoadedModel::Session LoadedModel::session_for(Rpc rpc, const RequestShape &shape,
LaneEntry &lane) {
// Proof of holding only. Nothing here reads it, and nothing should: its

19
backend/cpp/audio-cpp/loaded_model.h
View File

@@ -98,6 +98,25 @@ public:
// from the load to the request that honours it.
const std::string &pinned_task() const noexcept { return pinned_task_; }
// Throws the same CapabilityError session_for would throw when this family
// cannot serve the RPC, and does nothing otherwise.
//
// It exists so a refusal does not have to buy a place in the queue first.
// resolve_route is a pure function of capabilities_, which is fixed at
// construction and never written again, so unlike the session cache it
// needs no lane and no lock: a model that cannot transcribe can say so
// while another request is halfway through a thirty second run. Without
// this the refusal waits for that run to finish only to be told no.
//
// It does NOT replace the routing inside session_for, and must not be made
// to: session_for still needs the route to key the session cache. The two
// calls agree because both read the same immutable capabilities. What this
// one adds is only the ordering, so call it before acquire().
//
// Const and lane-free on purpose. If a future edit makes routing depend on
// mutable state, this must grow the lane parameter its siblings carry.
void check_can_serve(Rpc rpc, const RequestShape &shape) const;
// Routes the RPC and returns the cached session, creating it on first use.
// Throws CapabilityError when this family cannot serve the RPC, and a plain
// runtime_error when it can but the session could not be built, which is an

83
backend/cpp/audio-cpp/result_map.cpp Normal file
View File

@@ -0,0 +1,83 @@
#include "result_map.h"
#include "transcript_assembly.h"
#include <string>
#include <vector>
namespace audiocpp_backend {
void fill_transcript_result(const engine::runtime::TaskResult &result,
int sample_rate, float duration_seconds,
backend::TranscriptResult *out) {
if (out == nullptr) {
return;
}
std::vector<Span> speech_segments;
speech_segments.reserve(result.speech_segments.size());
for (const auto &segment : result.speech_segments) {
speech_segments.push_back(
Span{segment.span.start_sample, segment.span.end_sample});
}
std::vector<SpeakerSpan> speaker_turns;
speaker_turns.reserve(result.speaker_turns.size());
for (const auto &turn : result.speaker_turns) {
speaker_turns.push_back(
SpeakerSpan{Span{turn.span.start_sample, turn.span.end_sample},
turn.speaker_id});
}
std::vector<WordSpan> words;
words.reserve(result.word_timestamps.size());
for (const auto &word : result.word_timestamps) {
words.push_back(
WordSpan{Span{word.span.start_sample, word.span.end_sample},
word.word});
}
// The ONLY read of transcript text in this function, and the only one there
// may ever be. See THE RULE in the header.
const std::string text =
result.text_output.has_value() ? result.text_output->text : std::string();
const AssembledTranscript assembled = assemble_transcript(
text, speech_segments, speaker_turns, words, sample_rate);
out->set_text(assembled.text);
// language has no source inside transcript_assembly, which is span-shaped
// only, so it is read straight off the engine result here. Left untouched
// when the family reported no text output at all: an empty string would be
// indistinguishable from a family that genuinely detected no language, and
// the field is documented as optional.
if (result.text_output.has_value()) {
out->set_language(result.text_output->language);
}
out->set_duration(duration_seconds);
// Cleared rather than appended to. A caller that fills the same message
// twice (a stream's final_result being rebuilt, say) would otherwise emit
// every segment twice, and the second call's ids would restart at 0 and
// collide with the first call's.
out->clear_segments();
for (const auto &segment : assembled.segments) {
auto *out_segment = out->add_segments();
out_segment->set_id(segment.id);
// NANOSECONDS. TranscriptSegment and TranscriptWord are the only
// messages in backend.proto that use them; VADSegment and DiarizeSegment
// are float seconds. assemble_transcript has already converted.
out_segment->set_start(segment.start_ns);
out_segment->set_end(segment.end_ns);
out_segment->set_text(segment.text);
out_segment->set_speaker(segment.speaker);
for (const auto &word : segment.words) {
auto *out_word = out_segment->add_words();
out_word->set_start(word.start_ns);
out_word->set_end(word.end_ns);
out_word->set_text(word.text);
}
}
}
} // namespace audiocpp_backend

36
backend/cpp/audio-cpp/result_map.h Normal file
View File

@@ -0,0 +1,36 @@
#pragma once
// Converts engine::runtime results into LocalAI proto messages. All of the
// non-trivial shaping lives in transcript_assembly, which is stdlib-only and
// unit tested; this unit is the thin engine-typed boundary around it.
#include "backend.pb.h"
#include "engine/framework/runtime/session.h"
namespace audiocpp_backend {
// Fills text, language, duration, segments and per-segment words.
//
// THE RULE: the top-level text is TaskResult.text_output verbatim. It is never
// derived from segments or words. audio.cpp carries transcript text in
// text_output and nowhere else: speech_segments, speaker_turns and
// word_timestamps carry spans and labels and no text at all. Deriving the
// transcript from them therefore returns an EMPTY text for every producer that
// reports segments without word timing, which real VibeVoice diarized ASR does.
// An earlier attempt at this backend shipped exactly that bug. assemble_transcript
// enforces the rule and is heavily tested; this unit's job is not to re-derive
// it but to not undo it at the proto boundary.
//
// `sample_rate` is the rate the result's spans are expressed in, which is the
// rate of the AudioBuffer that was handed to the session, NOT the rate of the
// file the caller uploaded. Those differ whenever read_audio_file resampled,
// which is why the handler passes the buffer's rate rather than the file's.
//
// Segments are replaced, not appended to, so a message filled twice does not
// accumulate.
void fill_transcript_result(const engine::runtime::TaskResult &result,
int sample_rate, float duration_seconds,
backend::TranscriptResult *out);
} // namespace audiocpp_backend

247
backend/cpp/audio-cpp/result_map_ctest.cpp Normal file
View File

@@ -0,0 +1,247 @@
// Tests for result_map, the engine-to-proto boundary.
//
// NAMED _ctest AND NOT _test ON PURPOSE. backend/cpp/run-unit-tests.sh globs
// every *_test.cpp under backend/cpp/ and compiles it as a single standalone
// translation unit with no include path beyond its own directory. This file
// needs backend.pb.h and the audio.cpp framework headers, so it is built and
// run by ctest instead:
//
// make -C backend/cpp/audio-cpp test-engine
//
// Renaming it to *_test.cpp would break the standalone suite for every backend.
//
// What is worth testing here is exactly one thing, and it is not the field
// copying: THE RULE. TaskResult carries transcript text in text_output and
// nowhere else, so the proto's text must be that string verbatim. An earlier
// attempt at this backend derived it from the segments, which returns an empty
// transcript for every producer that reports segments without word timing.
// transcript_assembly already enforces the rule and is tested on its own; these
// checks are here so that a future edit cannot undo it at the boundary.
#include "result_map.h"
#include <cstdio>
#include <string>
static int failures = 0;
static void check(bool ok, const std::string &name) {
if (!ok) {
failures++;
fprintf(stderr, "FAIL: %s\n", name.c_str());
} else {
fprintf(stderr, "ok: %s\n", name.c_str());
}
}
using namespace audiocpp_backend;
namespace rt = engine::runtime;
static const int kRate = 16000;
static rt::SpeechSegment speech(std::int64_t start, std::int64_t end) {
rt::SpeechSegment segment;
segment.span.start_sample = start;
segment.span.end_sample = end;
return segment;
}
static rt::SpeakerTurn turn(std::int64_t start, std::int64_t end,
const std::string &speaker) {
rt::SpeakerTurn out;
out.span.start_sample = start;
out.span.end_sample = end;
out.speaker_id = speaker;
return out;
}
static rt::WordTimestamp word(std::int64_t start, std::int64_t end,
const std::string &text) {
rt::WordTimestamp out;
out.span.start_sample = start;
out.span.end_sample = end;
out.word = text;
return out;
}
// THE REGRESSION. A diarized ASR result: real text, real speaker turns, and no
// word timing at all. This is the vibevoice_asr shape, and it is the one that
// came back empty before.
static void test_text_survives_segments_without_words() {
rt::TaskResult result;
rt::Transcript transcript;
transcript.text = "hello there general kenobi";
transcript.language = "en";
result.text_output = transcript;
result.speaker_turns.push_back(turn(0, 16000, "speaker_0"));
result.speaker_turns.push_back(turn(16000, 32000, "speaker_1"));
backend::TranscriptResult out;
fill_transcript_result(result, kRate, 2.0f, &out);
check(out.text() == "hello there general kenobi",
"diarized result keeps text_output verbatim");
check(out.language() == "en", "language comes from text_output");
check(out.segments_size() == 2, "both speaker turns become segments");
if (out.segments_size() == 2) {
check(out.segments(0).speaker() == "speaker_0",
"first segment keeps its own speaker label");
check(out.segments(1).speaker() == "speaker_1",
"second segment keeps its own speaker label");
check(out.segments(1).start() == 1000000000LL,
"segment start is nanoseconds, not samples");
check(out.segments(1).end() == 2000000000LL,
"segment end is nanoseconds, not samples");
}
}
// The same rule seen from the other side: text present, spans present, and the
// per-segment text empty because there is nothing truthful to split. A boundary
// that derived the top-level text from these segments would produce "".
static void test_speech_segments_do_not_supply_the_text() {
rt::TaskResult result;
rt::Transcript transcript;
transcript.text = "one two three";
result.text_output = transcript;
result.speech_segments.push_back(speech(0, 8000));
result.speech_segments.push_back(speech(8000, 16000));
backend::TranscriptResult out;
fill_transcript_result(result, kRate, 1.0f, &out);
check(out.text() == "one two three",
"speech segments without words do not empty the transcript");
check(out.segments_size() == 2, "both speech segments are emitted");
if (out.segments_size() == 2) {
check(out.segments(0).text().empty() && out.segments(1).text().empty(),
"per-segment text stays empty when there is no word timing");
}
}
static void test_words_reach_the_proto_in_nanoseconds() {
rt::TaskResult result;
rt::Transcript transcript;
transcript.text = "hi there";
result.text_output = transcript;
result.word_timestamps.push_back(word(0, 8000, "hi"));
result.word_timestamps.push_back(word(8000, 16000, "there"));
backend::TranscriptResult out;
fill_transcript_result(result, kRate, 1.0f, &out);
check(out.text() == "hi there", "word-timed result keeps text_output");
check(out.segments_size() == 1, "words with no spans yield one covering segment");
if (out.segments_size() == 1) {
const auto &segment = out.segments(0);
check(segment.words_size() == 2, "both words are emitted");
if (segment.words_size() == 2) {
check(segment.words(0).text() == "hi", "first word text");
check(segment.words(0).start() == 0, "first word start");
check(segment.words(0).end() == 500000000LL,
"first word end is 0.5 s in nanoseconds");
check(segment.words(1).start() == 500000000LL, "second word start");
check(segment.words(1).end() == 1000000000LL, "second word end");
}
}
}
// The buffer's rate, not the file's, is what the spans mean. Passing 8000 for
// the same spans has to halve every timestamp, which is what makes resampling
// the input at read time load-bearing rather than cosmetic.
static void test_sample_rate_scales_the_timestamps() {
rt::TaskResult result;
rt::Transcript transcript;
transcript.text = "x";
result.text_output = transcript;
result.speech_segments.push_back(speech(0, 8000));
backend::TranscriptResult out;
fill_transcript_result(result, 8000, 1.0f, &out);
check(out.segments_size() == 1, "one segment at 8 kHz");
if (out.segments_size() == 1) {
check(out.segments(0).end() == 1000000000LL,
"8000 samples at 8 kHz is one second");
}
}
static void test_duration_is_carried_through() {
rt::TaskResult result;
rt::Transcript transcript;
transcript.text = "x";
result.text_output = transcript;
backend::TranscriptResult out;
fill_transcript_result(result, kRate, 14.07f, &out);
check(out.duration() > 14.06f && out.duration() < 14.08f,
"duration is set from the argument");
}
// No text output at all. A VAD-shaped result reaching this boundary must not
// invent a transcript, and must not overwrite a language the caller had already
// decided on.
static void test_missing_text_output_leaves_language_alone() {
rt::TaskResult result;
result.speech_segments.push_back(speech(0, 16000));
backend::TranscriptResult out;
out.set_language("it");
fill_transcript_result(result, kRate, 1.0f, &out);
check(out.text().empty(), "no text_output means no text");
check(out.language() == "it",
"a result with no text_output does not clear the language");
check(out.segments_size() == 1, "spans are still emitted");
}
// Filling the same message twice must replace, not accumulate: the second
// call's ids restart at 0 and would collide with the first call's.
static void test_refilling_replaces_the_segments() {
rt::TaskResult first;
rt::Transcript transcript;
transcript.text = "first";
first.text_output = transcript;
first.speech_segments.push_back(speech(0, 16000));
first.speech_segments.push_back(speech(16000, 32000));
backend::TranscriptResult out;
fill_transcript_result(first, kRate, 2.0f, &out);
rt::TaskResult second;
rt::Transcript replacement;
replacement.text = "second";
second.text_output = replacement;
second.speech_segments.push_back(speech(0, 16000));
fill_transcript_result(second, kRate, 1.0f, &out);
check(out.text() == "second", "the second fill replaces the text");
check(out.segments_size() == 1,
"the second fill replaces the segments instead of appending");
}
static void test_empty_result_is_empty() {
rt::TaskResult result;
backend::TranscriptResult out;
fill_transcript_result(result, kRate, 0.0f, &out);
check(out.text().empty(), "empty result has no text");
check(out.segments_size() == 0, "empty result has no segments");
}
int main() {
test_text_survives_segments_without_words();
test_speech_segments_do_not_supply_the_text();
test_words_reach_the_proto_in_nanoseconds();
test_sample_rate_scales_the_timestamps();
test_duration_is_carried_through();
test_missing_text_output_leaves_language_alone();
test_refilling_replaces_the_segments();
test_empty_result_is_empty();
if (failures) {
fprintf(stderr, "%d check(s) failed\n", failures);
return 1;
}
fprintf(stderr, "all result_map checks passed\n");
return 0;
}