mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-31 18:38:23 -04:00
backend(audio-cpp): assemble transcripts from runtime spans
The top-level transcript text is TaskResult.text_output verbatim. audio.cpp carries text nowhere else: speech_segments, speaker_turns and word_timestamps hold spans and labels only, so deriving the text from them empties the transcript for any producer that omits word timing, VibeVoice diarized ASR included. Fixtures cover every observed producer shape. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
@@ -89,6 +89,7 @@ add_executable(${TARGET}
|
||||
model_options.cpp
|
||||
capability_routing.cpp
|
||||
audio_units.cpp
|
||||
transcript_assembly.cpp
|
||||
)
|
||||
|
||||
target_include_directories(${TARGET} PRIVATE
|
||||
|
||||
157
backend/cpp/audio-cpp/transcript_assembly.cpp
Normal file
157
backend/cpp/audio-cpp/transcript_assembly.cpp
Normal file
@@ -0,0 +1,157 @@
|
||||
#include "transcript_assembly.h"
|
||||
|
||||
#include "audio_units.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdlib>
|
||||
#include <limits>
|
||||
|
||||
namespace audiocpp_backend {
|
||||
namespace {
|
||||
|
||||
std::int64_t midpoint(const Span &span) {
|
||||
return span.start_sample + (span.end_sample - span.start_sample) / 2;
|
||||
}
|
||||
|
||||
bool contains(const Span &span, std::int64_t sample) {
|
||||
return sample >= span.start_sample && sample < span.end_sample;
|
||||
}
|
||||
|
||||
std::int64_t overlap(const Span &a, const Span &b) {
|
||||
const std::int64_t begin = std::max(a.start_sample, b.start_sample);
|
||||
const std::int64_t end = std::min(a.end_sample, b.end_sample);
|
||||
return end > begin ? end - begin : 0;
|
||||
}
|
||||
|
||||
std::string join_words(const std::vector<OutWord> &words) {
|
||||
std::string out;
|
||||
for (const auto &word : words) {
|
||||
if (word.text.empty()) {
|
||||
continue;
|
||||
}
|
||||
if (!out.empty()) {
|
||||
out += " ";
|
||||
}
|
||||
out += word.text;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
std::string speaker_for(const Span &segment,
|
||||
const std::vector<SpeakerSpan> &turns) {
|
||||
std::string best;
|
||||
std::int64_t best_overlap = 0;
|
||||
for (const auto &turn : turns) {
|
||||
const std::int64_t shared = overlap(segment, turn.span);
|
||||
if (shared > best_overlap) {
|
||||
best_overlap = shared;
|
||||
best = turn.speaker;
|
||||
}
|
||||
}
|
||||
return best;
|
||||
}
|
||||
|
||||
// Chooses the segment spans, per the documented precedence.
|
||||
std::vector<Span> choose_segment_spans(const std::string &text_output,
|
||||
const std::vector<Span> &speech_segments,
|
||||
const std::vector<SpeakerSpan> &speaker_turns,
|
||||
const std::vector<WordSpan> &words) {
|
||||
if (!speech_segments.empty()) {
|
||||
return speech_segments;
|
||||
}
|
||||
if (!speaker_turns.empty()) {
|
||||
std::vector<Span> spans;
|
||||
spans.reserve(speaker_turns.size());
|
||||
for (const auto &turn : speaker_turns) {
|
||||
spans.push_back(turn.span);
|
||||
}
|
||||
return spans;
|
||||
}
|
||||
if (!words.empty()) {
|
||||
Span covering = words.front().span;
|
||||
for (const auto &word : words) {
|
||||
covering.start_sample =
|
||||
std::min(covering.start_sample, word.span.start_sample);
|
||||
covering.end_sample = std::max(covering.end_sample, word.span.end_sample);
|
||||
}
|
||||
return {covering};
|
||||
}
|
||||
if (!text_output.empty()) {
|
||||
// A zero span rather than a fabricated duration: the model reported no
|
||||
// timing, and inventing one would be a lie the caller cannot detect.
|
||||
return {Span{0, 0}};
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
// Returns the index of the segment a word belongs to, or the nearest segment
|
||||
// when the word falls outside all of them.
|
||||
size_t segment_index_for_word(const std::vector<Span> &spans, const Span &word) {
|
||||
const std::int64_t centre = midpoint(word);
|
||||
for (size_t i = 0; i < spans.size(); ++i) {
|
||||
if (contains(spans[i], centre)) {
|
||||
return i;
|
||||
}
|
||||
}
|
||||
size_t nearest = 0;
|
||||
std::int64_t best_distance = std::numeric_limits<std::int64_t>::max();
|
||||
for (size_t i = 0; i < spans.size(); ++i) {
|
||||
const std::int64_t distance = std::llabs(midpoint(spans[i]) - centre);
|
||||
if (distance < best_distance) {
|
||||
best_distance = distance;
|
||||
nearest = i;
|
||||
}
|
||||
}
|
||||
return nearest;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
AssembledTranscript assemble_transcript(const std::string &text_output,
|
||||
const std::vector<Span> &speech_segments,
|
||||
const std::vector<SpeakerSpan> &speaker_turns,
|
||||
const std::vector<WordSpan> &words,
|
||||
int sample_rate) {
|
||||
AssembledTranscript assembled;
|
||||
// THE RULE. Never derived from spans.
|
||||
assembled.text = text_output;
|
||||
|
||||
const std::vector<Span> spans =
|
||||
choose_segment_spans(text_output, speech_segments, speaker_turns, words);
|
||||
if (spans.empty()) {
|
||||
return assembled;
|
||||
}
|
||||
|
||||
assembled.segments.resize(spans.size());
|
||||
for (size_t i = 0; i < spans.size(); ++i) {
|
||||
OutSegment &segment = assembled.segments[i];
|
||||
segment.id = static_cast<int>(i);
|
||||
segment.start_ns = samples_to_nanoseconds(spans[i].start_sample, sample_rate);
|
||||
segment.end_ns = samples_to_nanoseconds(spans[i].end_sample, sample_rate);
|
||||
segment.speaker = speaker_for(spans[i], speaker_turns);
|
||||
}
|
||||
|
||||
for (const auto &word : words) {
|
||||
const size_t index = segment_index_for_word(spans, word.span);
|
||||
OutWord out;
|
||||
out.start_ns = samples_to_nanoseconds(word.span.start_sample, sample_rate);
|
||||
out.end_ns = samples_to_nanoseconds(word.span.end_sample, sample_rate);
|
||||
out.text = word.word;
|
||||
assembled.segments[index].words.push_back(out);
|
||||
}
|
||||
|
||||
for (auto &segment : assembled.segments) {
|
||||
segment.text = join_words(segment.words);
|
||||
}
|
||||
|
||||
// A single segment with no word timing carries the whole transcript. With
|
||||
// several segments there is no defensible way to split the text, so their
|
||||
// per-segment text stays empty and only the top-level text is authoritative.
|
||||
if (assembled.segments.size() == 1 && assembled.segments[0].words.empty()) {
|
||||
assembled.segments[0].text = text_output;
|
||||
}
|
||||
|
||||
return assembled;
|
||||
}
|
||||
|
||||
} // namespace audiocpp_backend
|
||||
74
backend/cpp/audio-cpp/transcript_assembly.h
Normal file
74
backend/cpp/audio-cpp/transcript_assembly.h
Normal file
@@ -0,0 +1,74 @@
|
||||
#pragma once
|
||||
|
||||
// Builds LocalAI's TranscriptResult shape from audio.cpp's TaskResult spans.
|
||||
// Standard library only; result_map.cpp converts the engine types into these
|
||||
// PODs at the boundary.
|
||||
//
|
||||
// THE RULE: the top-level transcript text is text_output verbatim, always.
|
||||
// audio.cpp carries transcript text in exactly one place, TaskResult.text_output.
|
||||
// speech_segments, speaker_turns and word_timestamps carry spans and labels but
|
||||
// no text. Deriving the top-level text by concatenating per-segment text
|
||||
// therefore yields an empty transcript for every producer that reports segments
|
||||
// without word timestamps, which includes VibeVoice diarized ASR.
|
||||
|
||||
#include <cstdint>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
namespace audiocpp_backend {
|
||||
|
||||
// Sample-index span, mirroring engine::runtime::TimeSpan.
|
||||
struct Span {
|
||||
std::int64_t start_sample = 0;
|
||||
std::int64_t end_sample = 0;
|
||||
};
|
||||
|
||||
struct WordSpan {
|
||||
Span span;
|
||||
std::string word;
|
||||
};
|
||||
|
||||
struct SpeakerSpan {
|
||||
Span span;
|
||||
std::string speaker;
|
||||
};
|
||||
|
||||
struct OutWord {
|
||||
std::int64_t start_ns = 0;
|
||||
std::int64_t end_ns = 0;
|
||||
std::string text;
|
||||
};
|
||||
|
||||
struct OutSegment {
|
||||
int id = 0;
|
||||
std::int64_t start_ns = 0;
|
||||
std::int64_t end_ns = 0;
|
||||
std::string text;
|
||||
std::string speaker;
|
||||
std::vector<OutWord> words;
|
||||
};
|
||||
|
||||
struct AssembledTranscript {
|
||||
std::string text;
|
||||
std::vector<OutSegment> segments;
|
||||
};
|
||||
|
||||
// Segment source, first non-empty wins:
|
||||
// 1. speech_segments
|
||||
// 2. speaker_turns
|
||||
// 3. one segment spanning all words, when words are present
|
||||
// 4. one zero-span segment carrying the full text, when text is present
|
||||
// 5. no segments
|
||||
//
|
||||
// Words attach to the segment whose range contains their midpoint; a word
|
||||
// outside every segment attaches to the nearest one by midpoint distance so it
|
||||
// is never silently dropped. A segment's text is its words joined by a single
|
||||
// space, except that a lone segment with no words carries the full text.
|
||||
// A segment's speaker is the speaker turn with the greatest overlap.
|
||||
AssembledTranscript assemble_transcript(const std::string &text_output,
|
||||
const std::vector<Span> &speech_segments,
|
||||
const std::vector<SpeakerSpan> &speaker_turns,
|
||||
const std::vector<WordSpan> &words,
|
||||
int sample_rate);
|
||||
|
||||
} // namespace audiocpp_backend
|
||||
390
backend/cpp/audio-cpp/transcript_assembly_test.cpp
Normal file
390
backend/cpp/audio-cpp/transcript_assembly_test.cpp
Normal file
@@ -0,0 +1,390 @@
|
||||
// Unit tests for transcript_assembly. Standard library only. The harness
|
||||
// compiles this as a single translation unit, so both implementations are
|
||||
// included directly rather than linked.
|
||||
//
|
||||
// Every fixture below mirrors a producer shape actually observed from
|
||||
// audio.cpp families. Do not replace them with invented shapes: an invented
|
||||
// shape is what let the earlier attempt ship an empty transcript.
|
||||
|
||||
#include "audio_units.cpp"
|
||||
#include "transcript_assembly.cpp"
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
static int failures = 0;
|
||||
|
||||
static void check(bool ok, const std::string &name) {
|
||||
if (!ok) {
|
||||
failures++;
|
||||
fprintf(stderr, "FAIL: %s\n", name.c_str());
|
||||
} else {
|
||||
fprintf(stderr, "ok: %s\n", name.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
using namespace audiocpp_backend;
|
||||
|
||||
// Indexed access that reports a named failure instead of running off the end.
|
||||
// std::vector::operator[] past the end is undefined behaviour, so a regression
|
||||
// that drops a segment would crash the process here and take every later check
|
||||
// with it. Returning a default element keeps the rest of the suite reporting.
|
||||
static const OutSegment &segment_at(const AssembledTranscript &out, size_t index,
|
||||
const std::string &name) {
|
||||
static const OutSegment missing;
|
||||
if (index >= out.segments.size()) {
|
||||
failures++;
|
||||
fprintf(stderr, "FAIL: %s (segment %zu is missing)\n", name.c_str(), index);
|
||||
return missing;
|
||||
}
|
||||
return out.segments[index];
|
||||
}
|
||||
|
||||
static const OutWord &word_at(const OutSegment &segment, size_t index,
|
||||
const std::string &name) {
|
||||
static const OutWord missing;
|
||||
if (index >= segment.words.size()) {
|
||||
failures++;
|
||||
fprintf(stderr, "FAIL: %s (word %zu is missing)\n", name.c_str(), index);
|
||||
return missing;
|
||||
}
|
||||
return segment.words[index];
|
||||
}
|
||||
|
||||
static const int kRate = 16000;
|
||||
|
||||
// Shape A: word timestamps only. Emitted by nemotron_asr, qwen3_asr and
|
||||
// qwen3_forced_aligner, all of which set text_output plus word_timestamps and
|
||||
// leave speech_segments empty.
|
||||
static void test_words_only() {
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 8000}, "hello"},
|
||||
{{8000, 16000}, "world"},
|
||||
};
|
||||
const auto out = assemble_transcript("hello world", {}, {}, words, kRate);
|
||||
|
||||
check(out.text == "hello world", "text is text_output verbatim");
|
||||
check(out.segments.size() == 1, "words with no segments yield one segment");
|
||||
const OutSegment &first = segment_at(out, 0, "words only segment");
|
||||
check(first.start_ns == 0, "segment starts at the first word");
|
||||
check(first.end_ns == 1000000000LL, "segment ends at the last word");
|
||||
check(first.words.size() == 2, "both words attached");
|
||||
check(word_at(first, 0, "first word").text == "hello", "first word text");
|
||||
check(word_at(first, 1, "second word").start_ns == 500000000LL,
|
||||
"second word start in ns");
|
||||
check(first.text == "hello world", "segment text joins its words");
|
||||
check(first.id == 0, "ids are zero based");
|
||||
}
|
||||
|
||||
// Shape A': the same producer, but text_output is punctuated and cased while
|
||||
// the word timestamps are not. qwen3_asr rebuilds text_output from its word
|
||||
// list only when timestamps are requested, so the two genuinely differ; this
|
||||
// pins the sole segment's text to its words rather than to the top-level text.
|
||||
static void test_words_only_with_punctuated_text_output() {
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 8000}, "hello"},
|
||||
{{8000, 16000}, "world"},
|
||||
};
|
||||
const auto out = assemble_transcript("Hello, world!", {}, {}, words, kRate);
|
||||
|
||||
check(out.text == "Hello, world!", "punctuated text_output is untouched");
|
||||
check(out.segments.size() == 1, "one segment");
|
||||
check(segment_at(out, 0, "punctuated segment").text == "hello world",
|
||||
"a segment with words takes its text from the words, not text_output");
|
||||
}
|
||||
|
||||
// Shape A'': a merged word list whose last word is not the one that ends
|
||||
// latest. audio.cpp concatenates per-chunk word lists in chunk order
|
||||
// (append_chunk_word_timestamps in framework/audio/chunking.cpp). It drops a
|
||||
// word whose global start falls before the chunk's keep span, but it never
|
||||
// clips a word's end to that boundary, so the last word kept from one chunk can
|
||||
// outlast the first word kept from the next. The covering span must therefore
|
||||
// be the extent of every word, not the span from the first to the last.
|
||||
static void test_covering_span_spans_every_word() {
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 4000}, "a"},
|
||||
// Kept from the earlier chunk, ending past the chunk boundary.
|
||||
{{4000, 10000}, "b"},
|
||||
// First word of the next chunk, shorter, so it ends earlier.
|
||||
{{8000, 9000}, "c"},
|
||||
};
|
||||
const auto out = assemble_transcript("a b c", {}, {}, words, kRate);
|
||||
|
||||
check(out.segments.size() == 1, "one covering segment");
|
||||
check(segment_at(out, 0, "covering segment").end_ns == 625000000LL,
|
||||
"the covering span reaches the latest word end, not the last word's");
|
||||
}
|
||||
|
||||
// Shape B: speech segments, no words. Emitted by ASR families that report
|
||||
// utterance boundaries without word-level timing.
|
||||
static void test_segments_without_words() {
|
||||
const std::vector<Span> segments = {{0, 16000}, {16000, 32000}};
|
||||
const auto out = assemble_transcript("one two three", segments, {}, {}, kRate);
|
||||
|
||||
// The regression: this must NOT be empty.
|
||||
check(out.text == "one two three", "multi-segment text is not empty");
|
||||
check(out.segments.size() == 2, "both segments survive");
|
||||
check(segment_at(out, 0, "first segment").end_ns == 1000000000LL,
|
||||
"first segment ends at 1s");
|
||||
check(segment_at(out, 1, "second segment").start_ns == 1000000000LL,
|
||||
"second segment starts at 1s");
|
||||
check(segment_at(out, 0, "first segment").text.empty(),
|
||||
"per-segment text stays empty when there are no words to split by");
|
||||
check(segment_at(out, 1, "second segment").id == 1, "ids increment");
|
||||
}
|
||||
|
||||
// Shape C: speech segments plus speaker turns, no words. This is the real
|
||||
// VibeVoice diarized ASR shape that broke the earlier attempt.
|
||||
static void test_segments_with_speaker_turns_no_words() {
|
||||
const std::vector<Span> segments = {{0, 16000}, {16000, 32000}};
|
||||
const std::vector<SpeakerSpan> turns = {
|
||||
{{0, 16000}, "SPEAKER_00"},
|
||||
{{16000, 32000}, "SPEAKER_01"},
|
||||
};
|
||||
const auto out = assemble_transcript("hi there", segments, turns, {}, kRate);
|
||||
|
||||
check(out.text == "hi there", "diarized multi-segment text is not empty");
|
||||
check(out.segments.size() == 2, "two segments");
|
||||
check(segment_at(out, 0, "first diarized segment").speaker == "SPEAKER_00",
|
||||
"first speaker assigned");
|
||||
check(segment_at(out, 1, "second diarized segment").speaker == "SPEAKER_01",
|
||||
"second speaker assigned");
|
||||
}
|
||||
|
||||
// Shape C': the same producer, but the utterance boundaries and the speaker
|
||||
// boundaries disagree. VibeVoice chunk merging shifts both lists independently,
|
||||
// so they need not line up. speech_segments is the segment source and speaker
|
||||
// turns only label it, which the matching-span fixture above cannot show.
|
||||
static void test_speech_segments_outrank_speaker_turns() {
|
||||
const std::vector<Span> segments = {{0, 32000}};
|
||||
const std::vector<SpeakerSpan> turns = {
|
||||
{{0, 16000}, "SPEAKER_00"},
|
||||
{{16000, 32000}, "SPEAKER_01"},
|
||||
};
|
||||
const auto out = assemble_transcript("hi there", segments, turns, {}, kRate);
|
||||
|
||||
check(out.segments.size() == 1,
|
||||
"speech segments decide the segmentation, not speaker turns");
|
||||
check(segment_at(out, 0, "single utterance").end_ns == 2000000000LL,
|
||||
"the utterance keeps its own span");
|
||||
}
|
||||
|
||||
// Speaker turns outrank word timestamps as a segment source: a diarized result
|
||||
// is segmented by who spoke, and words only fill the turns in.
|
||||
static void test_speaker_turns_outrank_words() {
|
||||
const std::vector<SpeakerSpan> turns = {
|
||||
{{0, 16000}, "SPEAKER_00"},
|
||||
{{16000, 32000}, "SPEAKER_01"},
|
||||
};
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 8000}, "hi"},
|
||||
{{16000, 24000}, "there"},
|
||||
};
|
||||
const auto out = assemble_transcript("hi there", {}, turns, words, kRate);
|
||||
|
||||
check(out.segments.size() == 2, "the two turns segment the result");
|
||||
check(segment_at(out, 0, "turn 0").text == "hi", "first turn takes its word");
|
||||
check(segment_at(out, 1, "turn 1").text == "there", "second turn takes its word");
|
||||
}
|
||||
|
||||
// Shape D: text only. Emitted by ASR families that report no timing at all,
|
||||
// such as hviske_asr and citrinet_asr.
|
||||
static void test_text_only() {
|
||||
const auto out = assemble_transcript("just text", {}, {}, {}, kRate);
|
||||
|
||||
check(out.text == "just text", "text survives");
|
||||
check(out.segments.size() == 1, "a single synthetic segment is emitted");
|
||||
const OutSegment &only = segment_at(out, 0, "synthetic segment");
|
||||
check(only.start_ns == 0 && only.end_ns == 0,
|
||||
"synthetic segment has zero span, not a fabricated duration");
|
||||
check(only.text == "just text", "the sole segment carries the full text");
|
||||
}
|
||||
|
||||
// Shape E: speaker turns only, no speech segments and no text. This is
|
||||
// sortformer_diar, reached through the Diarize RPC.
|
||||
static void test_speaker_turns_only() {
|
||||
const std::vector<SpeakerSpan> turns = {
|
||||
{{0, 24000}, "0"},
|
||||
{{24000, 48000}, "1"},
|
||||
};
|
||||
const auto out = assemble_transcript("", {}, turns, {}, kRate);
|
||||
|
||||
check(out.text.empty(), "no text is reported when the model produced none");
|
||||
check(out.segments.size() == 2, "turns become segments");
|
||||
check(segment_at(out, 0, "turn 0").speaker == "0",
|
||||
"speaker label preserved verbatim");
|
||||
check(segment_at(out, 1, "turn 1").start_ns == 1500000000LL,
|
||||
"second turn starts at 1.5s");
|
||||
}
|
||||
|
||||
// Shape F: nothing at all. A model that ran but produced no output must not
|
||||
// crash or fabricate a segment.
|
||||
static void test_empty() {
|
||||
const auto out = assemble_transcript("", {}, {}, {}, kRate);
|
||||
check(out.text.empty(), "empty stays empty");
|
||||
check(out.segments.empty(), "no segments are invented");
|
||||
}
|
||||
|
||||
// Shape H: speech segments with no text and no words at all. This is the VAD
|
||||
// path, silero_vad and marblenet_vad, which fill speech_segments and never
|
||||
// touch text_output. It reaches the lone-segment rule with nothing to carry.
|
||||
static void test_vad_segments_without_text() {
|
||||
const std::vector<Span> segments = {{0, 16000}, {24000, 32000}};
|
||||
const auto out = assemble_transcript("", segments, {}, {}, kRate);
|
||||
|
||||
check(out.text.empty(), "VAD reports no text");
|
||||
check(out.segments.size() == 2, "both speech regions survive");
|
||||
check(segment_at(out, 1, "second speech region").start_ns == 1500000000LL,
|
||||
"second region starts at 1.5s");
|
||||
check(segment_at(out, 0, "first speech region").text.empty(),
|
||||
"a VAD segment carries no text");
|
||||
|
||||
const std::vector<Span> one = {{0, 16000}};
|
||||
const auto single = assemble_transcript("", one, {}, {}, kRate);
|
||||
check(single.segments.size() == 1, "a single speech region survives");
|
||||
check(segment_at(single, 0, "lone speech region").text.empty(),
|
||||
"a lone VAD segment does not fabricate text");
|
||||
}
|
||||
|
||||
// Shape G: segments and words together. Words are assigned by midpoint so a
|
||||
// word straddling a boundary lands in exactly one segment.
|
||||
static void test_words_distributed_into_segments() {
|
||||
const std::vector<Span> segments = {{0, 16000}, {16000, 32000}};
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 4000}, "alpha"},
|
||||
{{4000, 8000}, "beta"},
|
||||
// Straddles the boundary; midpoint 16000 falls in the second segment.
|
||||
{{12000, 20000}, "gamma"},
|
||||
{{20000, 28000}, "delta"},
|
||||
};
|
||||
const auto out = assemble_transcript("alpha beta gamma delta", segments, {},
|
||||
words, kRate);
|
||||
|
||||
check(out.text == "alpha beta gamma delta", "top level text unchanged");
|
||||
check(out.segments.size() == 2, "two segments");
|
||||
check(segment_at(out, 0, "first segment").words.size() == 2,
|
||||
"first segment takes two words");
|
||||
check(segment_at(out, 1, "second segment").words.size() == 2,
|
||||
"second segment takes two words");
|
||||
check(segment_at(out, 0, "first segment").text == "alpha beta",
|
||||
"first segment text");
|
||||
check(segment_at(out, 1, "second segment").text == "gamma delta",
|
||||
"boundary-straddling word lands by midpoint");
|
||||
}
|
||||
|
||||
// The midpoint rule is not the same as either endpoint rule. "early" starts in
|
||||
// the first segment but ends in the second, and "late" the other way round;
|
||||
// each must land where its midpoint says, which no start-only or end-only rule
|
||||
// reproduces.
|
||||
static void test_words_assigned_by_midpoint_not_endpoint() {
|
||||
const std::vector<Span> segments = {{0, 16000}, {16000, 32000}};
|
||||
const std::vector<WordSpan> words = {
|
||||
// Midpoint 12000 -> first segment, although it ends in the second.
|
||||
{{4000, 20000}, "early"},
|
||||
// Midpoint 20000 -> second segment, although it starts in the first.
|
||||
{{12000, 28000}, "late"},
|
||||
};
|
||||
const auto out = assemble_transcript("early late", segments, {}, words, kRate);
|
||||
|
||||
check(segment_at(out, 0, "first segment").text == "early",
|
||||
"a word ending past the boundary stays where its midpoint is");
|
||||
check(segment_at(out, 1, "second segment").text == "late",
|
||||
"a word starting before the boundary follows its midpoint");
|
||||
}
|
||||
|
||||
// A word outside every segment must still be reachable rather than dropped
|
||||
// silently, so it attaches to the nearest segment by midpoint distance.
|
||||
static void test_word_outside_all_segments() {
|
||||
const std::vector<Span> segments = {{0, 16000}};
|
||||
const std::vector<WordSpan> words = {
|
||||
{{0, 8000}, "inside"},
|
||||
{{40000, 48000}, "outside"},
|
||||
};
|
||||
const auto out = assemble_transcript("inside outside", segments, {}, words,
|
||||
kRate);
|
||||
check(out.segments.size() == 1, "one segment");
|
||||
check(segment_at(out, 0, "sole segment").words.size() == 2,
|
||||
"the stray word is not dropped");
|
||||
}
|
||||
|
||||
// The fallback picks the nearest segment, which is not the same as picking the
|
||||
// first. With one segment the two are indistinguishable, so this uses three and
|
||||
// puts the stray word past the last one.
|
||||
static void test_stray_word_goes_to_the_nearest_segment() {
|
||||
const std::vector<Span> segments = {{0, 8000}, {8000, 16000}, {16000, 24000}};
|
||||
const std::vector<WordSpan> words = {
|
||||
// Midpoint 44000, nearest the third segment.
|
||||
{{40000, 48000}, "trailing"},
|
||||
};
|
||||
const auto out = assemble_transcript("trailing", segments, {}, words, kRate);
|
||||
|
||||
check(segment_at(out, 0, "first segment").words.empty(),
|
||||
"the stray word does not fall back to the first segment");
|
||||
check(segment_at(out, 2, "third segment").text == "trailing",
|
||||
"the stray word attaches to the nearest segment");
|
||||
}
|
||||
|
||||
// Speaker assignment uses greatest overlap, not first match, so a turn that
|
||||
// barely touches a segment does not win over one that covers it.
|
||||
static void test_speaker_assigned_by_greatest_overlap() {
|
||||
const std::vector<Span> segments = {{8000, 24000}};
|
||||
const std::vector<SpeakerSpan> turns = {
|
||||
{{0, 9000}, "brief"}, // overlaps 1000 samples
|
||||
{{9000, 24000}, "main"} // overlaps 15000 samples
|
||||
};
|
||||
const auto out = assemble_transcript("x", segments, turns, {}, kRate);
|
||||
check(out.segments.size() == 1, "one segment");
|
||||
check(segment_at(out, 0, "sole segment").speaker == "main",
|
||||
"greatest overlap wins");
|
||||
}
|
||||
|
||||
// A segment no turn touches gets no speaker rather than the label of whichever
|
||||
// turn happened to be listed first.
|
||||
static void test_segment_without_any_overlapping_turn_has_no_speaker() {
|
||||
const std::vector<Span> segments = {{0, 8000}, {40000, 48000}};
|
||||
const std::vector<SpeakerSpan> turns = {{{0, 8000}, "SPEAKER_00"}};
|
||||
const auto out = assemble_transcript("x", segments, turns, {}, kRate);
|
||||
|
||||
check(segment_at(out, 0, "overlapped segment").speaker == "SPEAKER_00",
|
||||
"the overlapped segment is labelled");
|
||||
check(segment_at(out, 1, "unlabelled segment").speaker.empty(),
|
||||
"a segment no turn overlaps is left unlabelled");
|
||||
}
|
||||
|
||||
static void test_zero_sample_rate_is_safe() {
|
||||
const std::vector<Span> segments = {{0, 16000}};
|
||||
const auto out = assemble_transcript("x", segments, {}, {}, 0);
|
||||
check(out.segments.size() == 1, "a zero sample rate still yields the segment");
|
||||
const OutSegment &only = segment_at(out, 0, "sole segment");
|
||||
check(only.start_ns == 0 && only.end_ns == 0,
|
||||
"unknown sample rate yields zero timings rather than garbage");
|
||||
}
|
||||
|
||||
int main() {
|
||||
test_words_only();
|
||||
test_words_only_with_punctuated_text_output();
|
||||
test_covering_span_spans_every_word();
|
||||
test_segments_without_words();
|
||||
test_segments_with_speaker_turns_no_words();
|
||||
test_speech_segments_outrank_speaker_turns();
|
||||
test_speaker_turns_outrank_words();
|
||||
test_text_only();
|
||||
test_speaker_turns_only();
|
||||
test_empty();
|
||||
test_vad_segments_without_text();
|
||||
test_words_distributed_into_segments();
|
||||
test_words_assigned_by_midpoint_not_endpoint();
|
||||
test_word_outside_all_segments();
|
||||
test_stray_word_goes_to_the_nearest_segment();
|
||||
test_speaker_assigned_by_greatest_overlap();
|
||||
test_segment_without_any_overlapping_turn_has_no_speaker();
|
||||
test_zero_sample_rate_is_safe();
|
||||
if (failures) {
|
||||
fprintf(stderr, "%d check(s) failed\n", failures);
|
||||
return 1;
|
||||
}
|
||||
fprintf(stderr, "all transcript_assembly checks passed\n");
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user