diff --git a/backend/cpp/audio-cpp/CMakeLists.txt b/backend/cpp/audio-cpp/CMakeLists.txt index 729dc70e2..d2b6a7f57 100644 --- a/backend/cpp/audio-cpp/CMakeLists.txt +++ b/backend/cpp/audio-cpp/CMakeLists.txt @@ -89,6 +89,7 @@ add_executable(${TARGET} model_options.cpp capability_routing.cpp audio_units.cpp + transcript_assembly.cpp ) target_include_directories(${TARGET} PRIVATE diff --git a/backend/cpp/audio-cpp/transcript_assembly.cpp b/backend/cpp/audio-cpp/transcript_assembly.cpp new file mode 100644 index 000000000..0bf0987e6 --- /dev/null +++ b/backend/cpp/audio-cpp/transcript_assembly.cpp @@ -0,0 +1,157 @@ +#include "transcript_assembly.h" + +#include "audio_units.h" + +#include +#include +#include + +namespace audiocpp_backend { +namespace { + +std::int64_t midpoint(const Span &span) { + return span.start_sample + (span.end_sample - span.start_sample) / 2; +} + +bool contains(const Span &span, std::int64_t sample) { + return sample >= span.start_sample && sample < span.end_sample; +} + +std::int64_t overlap(const Span &a, const Span &b) { + const std::int64_t begin = std::max(a.start_sample, b.start_sample); + const std::int64_t end = std::min(a.end_sample, b.end_sample); + return end > begin ? end - begin : 0; +} + +std::string join_words(const std::vector &words) { + std::string out; + for (const auto &word : words) { + if (word.text.empty()) { + continue; + } + if (!out.empty()) { + out += " "; + } + out += word.text; + } + return out; +} + +std::string speaker_for(const Span &segment, + const std::vector &turns) { + std::string best; + std::int64_t best_overlap = 0; + for (const auto &turn : turns) { + const std::int64_t shared = overlap(segment, turn.span); + if (shared > best_overlap) { + best_overlap = shared; + best = turn.speaker; + } + } + return best; +} + +// Chooses the segment spans, per the documented precedence. +std::vector choose_segment_spans(const std::string &text_output, + const std::vector &speech_segments, + const std::vector &speaker_turns, + const std::vector &words) { + if (!speech_segments.empty()) { + return speech_segments; + } + if (!speaker_turns.empty()) { + std::vector spans; + spans.reserve(speaker_turns.size()); + for (const auto &turn : speaker_turns) { + spans.push_back(turn.span); + } + return spans; + } + if (!words.empty()) { + Span covering = words.front().span; + for (const auto &word : words) { + covering.start_sample = + std::min(covering.start_sample, word.span.start_sample); + covering.end_sample = std::max(covering.end_sample, word.span.end_sample); + } + return {covering}; + } + if (!text_output.empty()) { + // A zero span rather than a fabricated duration: the model reported no + // timing, and inventing one would be a lie the caller cannot detect. + return {Span{0, 0}}; + } + return {}; +} + +// Returns the index of the segment a word belongs to, or the nearest segment +// when the word falls outside all of them. +size_t segment_index_for_word(const std::vector &spans, const Span &word) { + const std::int64_t centre = midpoint(word); + for (size_t i = 0; i < spans.size(); ++i) { + if (contains(spans[i], centre)) { + return i; + } + } + size_t nearest = 0; + std::int64_t best_distance = std::numeric_limits::max(); + for (size_t i = 0; i < spans.size(); ++i) { + const std::int64_t distance = std::llabs(midpoint(spans[i]) - centre); + if (distance < best_distance) { + best_distance = distance; + nearest = i; + } + } + return nearest; +} + +} // namespace + +AssembledTranscript assemble_transcript(const std::string &text_output, + const std::vector &speech_segments, + const std::vector &speaker_turns, + const std::vector &words, + int sample_rate) { + AssembledTranscript assembled; + // THE RULE. Never derived from spans. + assembled.text = text_output; + + const std::vector spans = + choose_segment_spans(text_output, speech_segments, speaker_turns, words); + if (spans.empty()) { + return assembled; + } + + assembled.segments.resize(spans.size()); + for (size_t i = 0; i < spans.size(); ++i) { + OutSegment &segment = assembled.segments[i]; + segment.id = static_cast(i); + segment.start_ns = samples_to_nanoseconds(spans[i].start_sample, sample_rate); + segment.end_ns = samples_to_nanoseconds(spans[i].end_sample, sample_rate); + segment.speaker = speaker_for(spans[i], speaker_turns); + } + + for (const auto &word : words) { + const size_t index = segment_index_for_word(spans, word.span); + OutWord out; + out.start_ns = samples_to_nanoseconds(word.span.start_sample, sample_rate); + out.end_ns = samples_to_nanoseconds(word.span.end_sample, sample_rate); + out.text = word.word; + assembled.segments[index].words.push_back(out); + } + + for (auto &segment : assembled.segments) { + segment.text = join_words(segment.words); + } + + // A single segment with no word timing carries the whole transcript. With + // several segments there is no defensible way to split the text, so their + // per-segment text stays empty and only the top-level text is authoritative. + if (assembled.segments.size() == 1 && assembled.segments[0].words.empty()) { + assembled.segments[0].text = text_output; + } + + return assembled; +} + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/transcript_assembly.h b/backend/cpp/audio-cpp/transcript_assembly.h new file mode 100644 index 000000000..f5866bdc2 --- /dev/null +++ b/backend/cpp/audio-cpp/transcript_assembly.h @@ -0,0 +1,74 @@ +#pragma once + +// Builds LocalAI's TranscriptResult shape from audio.cpp's TaskResult spans. +// Standard library only; result_map.cpp converts the engine types into these +// PODs at the boundary. +// +// THE RULE: the top-level transcript text is text_output verbatim, always. +// audio.cpp carries transcript text in exactly one place, TaskResult.text_output. +// speech_segments, speaker_turns and word_timestamps carry spans and labels but +// no text. Deriving the top-level text by concatenating per-segment text +// therefore yields an empty transcript for every producer that reports segments +// without word timestamps, which includes VibeVoice diarized ASR. + +#include +#include +#include + +namespace audiocpp_backend { + +// Sample-index span, mirroring engine::runtime::TimeSpan. +struct Span { + std::int64_t start_sample = 0; + std::int64_t end_sample = 0; +}; + +struct WordSpan { + Span span; + std::string word; +}; + +struct SpeakerSpan { + Span span; + std::string speaker; +}; + +struct OutWord { + std::int64_t start_ns = 0; + std::int64_t end_ns = 0; + std::string text; +}; + +struct OutSegment { + int id = 0; + std::int64_t start_ns = 0; + std::int64_t end_ns = 0; + std::string text; + std::string speaker; + std::vector words; +}; + +struct AssembledTranscript { + std::string text; + std::vector segments; +}; + +// Segment source, first non-empty wins: +// 1. speech_segments +// 2. speaker_turns +// 3. one segment spanning all words, when words are present +// 4. one zero-span segment carrying the full text, when text is present +// 5. no segments +// +// Words attach to the segment whose range contains their midpoint; a word +// outside every segment attaches to the nearest one by midpoint distance so it +// is never silently dropped. A segment's text is its words joined by a single +// space, except that a lone segment with no words carries the full text. +// A segment's speaker is the speaker turn with the greatest overlap. +AssembledTranscript assemble_transcript(const std::string &text_output, + const std::vector &speech_segments, + const std::vector &speaker_turns, + const std::vector &words, + int sample_rate); + +} // namespace audiocpp_backend diff --git a/backend/cpp/audio-cpp/transcript_assembly_test.cpp b/backend/cpp/audio-cpp/transcript_assembly_test.cpp new file mode 100644 index 000000000..889418b4f --- /dev/null +++ b/backend/cpp/audio-cpp/transcript_assembly_test.cpp @@ -0,0 +1,390 @@ +// Unit tests for transcript_assembly. Standard library only. The harness +// compiles this as a single translation unit, so both implementations are +// included directly rather than linked. +// +// Every fixture below mirrors a producer shape actually observed from +// audio.cpp families. Do not replace them with invented shapes: an invented +// shape is what let the earlier attempt ship an empty transcript. + +#include "audio_units.cpp" +#include "transcript_assembly.cpp" + +#include +#include +#include +#include + +static int failures = 0; + +static void check(bool ok, const std::string &name) { + if (!ok) { + failures++; + fprintf(stderr, "FAIL: %s\n", name.c_str()); + } else { + fprintf(stderr, "ok: %s\n", name.c_str()); + } +} + +using namespace audiocpp_backend; + +// Indexed access that reports a named failure instead of running off the end. +// std::vector::operator[] past the end is undefined behaviour, so a regression +// that drops a segment would crash the process here and take every later check +// with it. Returning a default element keeps the rest of the suite reporting. +static const OutSegment &segment_at(const AssembledTranscript &out, size_t index, + const std::string &name) { + static const OutSegment missing; + if (index >= out.segments.size()) { + failures++; + fprintf(stderr, "FAIL: %s (segment %zu is missing)\n", name.c_str(), index); + return missing; + } + return out.segments[index]; +} + +static const OutWord &word_at(const OutSegment &segment, size_t index, + const std::string &name) { + static const OutWord missing; + if (index >= segment.words.size()) { + failures++; + fprintf(stderr, "FAIL: %s (word %zu is missing)\n", name.c_str(), index); + return missing; + } + return segment.words[index]; +} + +static const int kRate = 16000; + +// Shape A: word timestamps only. Emitted by nemotron_asr, qwen3_asr and +// qwen3_forced_aligner, all of which set text_output plus word_timestamps and +// leave speech_segments empty. +static void test_words_only() { + const std::vector words = { + {{0, 8000}, "hello"}, + {{8000, 16000}, "world"}, + }; + const auto out = assemble_transcript("hello world", {}, {}, words, kRate); + + check(out.text == "hello world", "text is text_output verbatim"); + check(out.segments.size() == 1, "words with no segments yield one segment"); + const OutSegment &first = segment_at(out, 0, "words only segment"); + check(first.start_ns == 0, "segment starts at the first word"); + check(first.end_ns == 1000000000LL, "segment ends at the last word"); + check(first.words.size() == 2, "both words attached"); + check(word_at(first, 0, "first word").text == "hello", "first word text"); + check(word_at(first, 1, "second word").start_ns == 500000000LL, + "second word start in ns"); + check(first.text == "hello world", "segment text joins its words"); + check(first.id == 0, "ids are zero based"); +} + +// Shape A': the same producer, but text_output is punctuated and cased while +// the word timestamps are not. qwen3_asr rebuilds text_output from its word +// list only when timestamps are requested, so the two genuinely differ; this +// pins the sole segment's text to its words rather than to the top-level text. +static void test_words_only_with_punctuated_text_output() { + const std::vector words = { + {{0, 8000}, "hello"}, + {{8000, 16000}, "world"}, + }; + const auto out = assemble_transcript("Hello, world!", {}, {}, words, kRate); + + check(out.text == "Hello, world!", "punctuated text_output is untouched"); + check(out.segments.size() == 1, "one segment"); + check(segment_at(out, 0, "punctuated segment").text == "hello world", + "a segment with words takes its text from the words, not text_output"); +} + +// Shape A'': a merged word list whose last word is not the one that ends +// latest. audio.cpp concatenates per-chunk word lists in chunk order +// (append_chunk_word_timestamps in framework/audio/chunking.cpp). It drops a +// word whose global start falls before the chunk's keep span, but it never +// clips a word's end to that boundary, so the last word kept from one chunk can +// outlast the first word kept from the next. The covering span must therefore +// be the extent of every word, not the span from the first to the last. +static void test_covering_span_spans_every_word() { + const std::vector words = { + {{0, 4000}, "a"}, + // Kept from the earlier chunk, ending past the chunk boundary. + {{4000, 10000}, "b"}, + // First word of the next chunk, shorter, so it ends earlier. + {{8000, 9000}, "c"}, + }; + const auto out = assemble_transcript("a b c", {}, {}, words, kRate); + + check(out.segments.size() == 1, "one covering segment"); + check(segment_at(out, 0, "covering segment").end_ns == 625000000LL, + "the covering span reaches the latest word end, not the last word's"); +} + +// Shape B: speech segments, no words. Emitted by ASR families that report +// utterance boundaries without word-level timing. +static void test_segments_without_words() { + const std::vector segments = {{0, 16000}, {16000, 32000}}; + const auto out = assemble_transcript("one two three", segments, {}, {}, kRate); + + // The regression: this must NOT be empty. + check(out.text == "one two three", "multi-segment text is not empty"); + check(out.segments.size() == 2, "both segments survive"); + check(segment_at(out, 0, "first segment").end_ns == 1000000000LL, + "first segment ends at 1s"); + check(segment_at(out, 1, "second segment").start_ns == 1000000000LL, + "second segment starts at 1s"); + check(segment_at(out, 0, "first segment").text.empty(), + "per-segment text stays empty when there are no words to split by"); + check(segment_at(out, 1, "second segment").id == 1, "ids increment"); +} + +// Shape C: speech segments plus speaker turns, no words. This is the real +// VibeVoice diarized ASR shape that broke the earlier attempt. +static void test_segments_with_speaker_turns_no_words() { + const std::vector segments = {{0, 16000}, {16000, 32000}}; + const std::vector turns = { + {{0, 16000}, "SPEAKER_00"}, + {{16000, 32000}, "SPEAKER_01"}, + }; + const auto out = assemble_transcript("hi there", segments, turns, {}, kRate); + + check(out.text == "hi there", "diarized multi-segment text is not empty"); + check(out.segments.size() == 2, "two segments"); + check(segment_at(out, 0, "first diarized segment").speaker == "SPEAKER_00", + "first speaker assigned"); + check(segment_at(out, 1, "second diarized segment").speaker == "SPEAKER_01", + "second speaker assigned"); +} + +// Shape C': the same producer, but the utterance boundaries and the speaker +// boundaries disagree. VibeVoice chunk merging shifts both lists independently, +// so they need not line up. speech_segments is the segment source and speaker +// turns only label it, which the matching-span fixture above cannot show. +static void test_speech_segments_outrank_speaker_turns() { + const std::vector segments = {{0, 32000}}; + const std::vector turns = { + {{0, 16000}, "SPEAKER_00"}, + {{16000, 32000}, "SPEAKER_01"}, + }; + const auto out = assemble_transcript("hi there", segments, turns, {}, kRate); + + check(out.segments.size() == 1, + "speech segments decide the segmentation, not speaker turns"); + check(segment_at(out, 0, "single utterance").end_ns == 2000000000LL, + "the utterance keeps its own span"); +} + +// Speaker turns outrank word timestamps as a segment source: a diarized result +// is segmented by who spoke, and words only fill the turns in. +static void test_speaker_turns_outrank_words() { + const std::vector turns = { + {{0, 16000}, "SPEAKER_00"}, + {{16000, 32000}, "SPEAKER_01"}, + }; + const std::vector words = { + {{0, 8000}, "hi"}, + {{16000, 24000}, "there"}, + }; + const auto out = assemble_transcript("hi there", {}, turns, words, kRate); + + check(out.segments.size() == 2, "the two turns segment the result"); + check(segment_at(out, 0, "turn 0").text == "hi", "first turn takes its word"); + check(segment_at(out, 1, "turn 1").text == "there", "second turn takes its word"); +} + +// Shape D: text only. Emitted by ASR families that report no timing at all, +// such as hviske_asr and citrinet_asr. +static void test_text_only() { + const auto out = assemble_transcript("just text", {}, {}, {}, kRate); + + check(out.text == "just text", "text survives"); + check(out.segments.size() == 1, "a single synthetic segment is emitted"); + const OutSegment &only = segment_at(out, 0, "synthetic segment"); + check(only.start_ns == 0 && only.end_ns == 0, + "synthetic segment has zero span, not a fabricated duration"); + check(only.text == "just text", "the sole segment carries the full text"); +} + +// Shape E: speaker turns only, no speech segments and no text. This is +// sortformer_diar, reached through the Diarize RPC. +static void test_speaker_turns_only() { + const std::vector turns = { + {{0, 24000}, "0"}, + {{24000, 48000}, "1"}, + }; + const auto out = assemble_transcript("", {}, turns, {}, kRate); + + check(out.text.empty(), "no text is reported when the model produced none"); + check(out.segments.size() == 2, "turns become segments"); + check(segment_at(out, 0, "turn 0").speaker == "0", + "speaker label preserved verbatim"); + check(segment_at(out, 1, "turn 1").start_ns == 1500000000LL, + "second turn starts at 1.5s"); +} + +// Shape F: nothing at all. A model that ran but produced no output must not +// crash or fabricate a segment. +static void test_empty() { + const auto out = assemble_transcript("", {}, {}, {}, kRate); + check(out.text.empty(), "empty stays empty"); + check(out.segments.empty(), "no segments are invented"); +} + +// Shape H: speech segments with no text and no words at all. This is the VAD +// path, silero_vad and marblenet_vad, which fill speech_segments and never +// touch text_output. It reaches the lone-segment rule with nothing to carry. +static void test_vad_segments_without_text() { + const std::vector segments = {{0, 16000}, {24000, 32000}}; + const auto out = assemble_transcript("", segments, {}, {}, kRate); + + check(out.text.empty(), "VAD reports no text"); + check(out.segments.size() == 2, "both speech regions survive"); + check(segment_at(out, 1, "second speech region").start_ns == 1500000000LL, + "second region starts at 1.5s"); + check(segment_at(out, 0, "first speech region").text.empty(), + "a VAD segment carries no text"); + + const std::vector one = {{0, 16000}}; + const auto single = assemble_transcript("", one, {}, {}, kRate); + check(single.segments.size() == 1, "a single speech region survives"); + check(segment_at(single, 0, "lone speech region").text.empty(), + "a lone VAD segment does not fabricate text"); +} + +// Shape G: segments and words together. Words are assigned by midpoint so a +// word straddling a boundary lands in exactly one segment. +static void test_words_distributed_into_segments() { + const std::vector segments = {{0, 16000}, {16000, 32000}}; + const std::vector words = { + {{0, 4000}, "alpha"}, + {{4000, 8000}, "beta"}, + // Straddles the boundary; midpoint 16000 falls in the second segment. + {{12000, 20000}, "gamma"}, + {{20000, 28000}, "delta"}, + }; + const auto out = assemble_transcript("alpha beta gamma delta", segments, {}, + words, kRate); + + check(out.text == "alpha beta gamma delta", "top level text unchanged"); + check(out.segments.size() == 2, "two segments"); + check(segment_at(out, 0, "first segment").words.size() == 2, + "first segment takes two words"); + check(segment_at(out, 1, "second segment").words.size() == 2, + "second segment takes two words"); + check(segment_at(out, 0, "first segment").text == "alpha beta", + "first segment text"); + check(segment_at(out, 1, "second segment").text == "gamma delta", + "boundary-straddling word lands by midpoint"); +} + +// The midpoint rule is not the same as either endpoint rule. "early" starts in +// the first segment but ends in the second, and "late" the other way round; +// each must land where its midpoint says, which no start-only or end-only rule +// reproduces. +static void test_words_assigned_by_midpoint_not_endpoint() { + const std::vector segments = {{0, 16000}, {16000, 32000}}; + const std::vector words = { + // Midpoint 12000 -> first segment, although it ends in the second. + {{4000, 20000}, "early"}, + // Midpoint 20000 -> second segment, although it starts in the first. + {{12000, 28000}, "late"}, + }; + const auto out = assemble_transcript("early late", segments, {}, words, kRate); + + check(segment_at(out, 0, "first segment").text == "early", + "a word ending past the boundary stays where its midpoint is"); + check(segment_at(out, 1, "second segment").text == "late", + "a word starting before the boundary follows its midpoint"); +} + +// A word outside every segment must still be reachable rather than dropped +// silently, so it attaches to the nearest segment by midpoint distance. +static void test_word_outside_all_segments() { + const std::vector segments = {{0, 16000}}; + const std::vector words = { + {{0, 8000}, "inside"}, + {{40000, 48000}, "outside"}, + }; + const auto out = assemble_transcript("inside outside", segments, {}, words, + kRate); + check(out.segments.size() == 1, "one segment"); + check(segment_at(out, 0, "sole segment").words.size() == 2, + "the stray word is not dropped"); +} + +// The fallback picks the nearest segment, which is not the same as picking the +// first. With one segment the two are indistinguishable, so this uses three and +// puts the stray word past the last one. +static void test_stray_word_goes_to_the_nearest_segment() { + const std::vector segments = {{0, 8000}, {8000, 16000}, {16000, 24000}}; + const std::vector words = { + // Midpoint 44000, nearest the third segment. + {{40000, 48000}, "trailing"}, + }; + const auto out = assemble_transcript("trailing", segments, {}, words, kRate); + + check(segment_at(out, 0, "first segment").words.empty(), + "the stray word does not fall back to the first segment"); + check(segment_at(out, 2, "third segment").text == "trailing", + "the stray word attaches to the nearest segment"); +} + +// Speaker assignment uses greatest overlap, not first match, so a turn that +// barely touches a segment does not win over one that covers it. +static void test_speaker_assigned_by_greatest_overlap() { + const std::vector segments = {{8000, 24000}}; + const std::vector turns = { + {{0, 9000}, "brief"}, // overlaps 1000 samples + {{9000, 24000}, "main"} // overlaps 15000 samples + }; + const auto out = assemble_transcript("x", segments, turns, {}, kRate); + check(out.segments.size() == 1, "one segment"); + check(segment_at(out, 0, "sole segment").speaker == "main", + "greatest overlap wins"); +} + +// A segment no turn touches gets no speaker rather than the label of whichever +// turn happened to be listed first. +static void test_segment_without_any_overlapping_turn_has_no_speaker() { + const std::vector segments = {{0, 8000}, {40000, 48000}}; + const std::vector turns = {{{0, 8000}, "SPEAKER_00"}}; + const auto out = assemble_transcript("x", segments, turns, {}, kRate); + + check(segment_at(out, 0, "overlapped segment").speaker == "SPEAKER_00", + "the overlapped segment is labelled"); + check(segment_at(out, 1, "unlabelled segment").speaker.empty(), + "a segment no turn overlaps is left unlabelled"); +} + +static void test_zero_sample_rate_is_safe() { + const std::vector segments = {{0, 16000}}; + const auto out = assemble_transcript("x", segments, {}, {}, 0); + check(out.segments.size() == 1, "a zero sample rate still yields the segment"); + const OutSegment &only = segment_at(out, 0, "sole segment"); + check(only.start_ns == 0 && only.end_ns == 0, + "unknown sample rate yields zero timings rather than garbage"); +} + +int main() { + test_words_only(); + test_words_only_with_punctuated_text_output(); + test_covering_span_spans_every_word(); + test_segments_without_words(); + test_segments_with_speaker_turns_no_words(); + test_speech_segments_outrank_speaker_turns(); + test_speaker_turns_outrank_words(); + test_text_only(); + test_speaker_turns_only(); + test_empty(); + test_vad_segments_without_text(); + test_words_distributed_into_segments(); + test_words_assigned_by_midpoint_not_endpoint(); + test_word_outside_all_segments(); + test_stray_word_goes_to_the_nearest_segment(); + test_speaker_assigned_by_greatest_overlap(); + test_segment_without_any_overlapping_turn_has_no_speaker(); + test_zero_sample_rate_is_safe(); + if (failures) { + fprintf(stderr, "%d check(s) failed\n", failures); + return 1; + } + fprintf(stderr, "all transcript_assembly checks passed\n"); + return 0; +}