#include "transcript_assembly.h" #include "audio_units.h" #include #include #include namespace audiocpp_backend { namespace { std::int64_t midpoint(const Span &span) { return span.start_sample + (span.end_sample - span.start_sample) / 2; } bool contains(const Span &span, std::int64_t sample) { return sample >= span.start_sample && sample < span.end_sample; } std::int64_t overlap(const Span &a, const Span &b) { const std::int64_t begin = std::max(a.start_sample, b.start_sample); const std::int64_t end = std::min(a.end_sample, b.end_sample); return end > begin ? end - begin : 0; } // Joins a segment's words into that segment's text. // // THE SEPARATOR IS NOT ALWAYS A SPACE, and getting it wrong is visible to every // caller rather than cosmetic: core/http/endpoints/openai/transcription.go // routes response_format text, srt, vtt and lrc through // schema.TranscriptionResponse, which builds the entire body out of // Segments[].Text and never reads the top-level text. For those four formats // the segment text IS the response. // // Two producer conventions have to be told apart: // // whole words "Some", "call", "me" -> join with a space // subword pieces "So", "me", " call" -> concatenate // // The second is SentencePiece, where a word boundary is carried as a LEADING // SPACE on the piece; nemotron_asr emits one entry per token in exactly that // form. Space-joining those produced "So me call me na ture ,", which is // what response_format=text returned while the correct sentence sat unread in // the top-level field. Concatenating them reproduces text_output exactly. // // The convention is read off the words themselves, because nothing else in the // result declares it. One leading space anywhere is enough to decide: a // whole-word producer has no reason to emit one, and a subword producer emits // one at every word boundary, so the two populations do not overlap. A producer // that mixed both conventions inside one segment could not be served correctly // by any single separator; this picks concatenation for it. // // This does NOT touch the top-level text, which stays text_output verbatim. The // rule that forbids deriving the transcript from the segments is about the // direction segments -> text. Segment text has no source other than its words // and is necessarily derived. std::string join_words(const std::vector &words) { const bool subword_pieces = std::any_of(words.begin(), words.end(), [](const OutWord &word) { return !word.text.empty() && word.text.front() == ' '; }); std::string out; for (const auto &word : words) { if (word.text.empty()) { continue; } if (!subword_pieces && !out.empty()) { out += " "; } out += word.text; } return out; } std::string speaker_for(const Span &segment, const std::vector &turns) { std::string best; std::int64_t best_overlap = 0; for (const auto &turn : turns) { const std::int64_t shared = overlap(segment, turn.span); if (shared > best_overlap) { best_overlap = shared; best = turn.speaker; } } return best; } // The chosen segmentation. labels is empty unless the spans were sourced from // the speaker turns themselves, in which case it is parallel to spans and holds // the label each span arrived with. struct SegmentSource { std::vector spans; std::vector labels; }; // Chooses the segment spans, per the documented precedence. SegmentSource choose_segment_spans(const std::string &text_output, const std::vector &speech_segments, const std::vector &speaker_turns, const std::vector &words) { if (!speech_segments.empty()) { return {speech_segments, {}}; } if (!speaker_turns.empty()) { // The labels are carried out rather than re-derived by overlap later. A // turn wholly contained in another speaker's turn overlaps its own span // completely, which is the largest overlap possible, so it can only tie // with the containing turn and would then lose that tie on order. // sortformer_diar binarizes each speaker's track independently and // sorts the result by start sample, so the container always comes // first, and the interjecting speaker would be silently relabelled to // the speaker it interrupted. SegmentSource source; source.spans.reserve(speaker_turns.size()); source.labels.reserve(speaker_turns.size()); for (const auto &turn : speaker_turns) { source.spans.push_back(turn.span); source.labels.push_back(turn.speaker); } return source; } if (!words.empty()) { Span covering = words.front().span; for (const auto &word : words) { covering.start_sample = std::min(covering.start_sample, word.span.start_sample); covering.end_sample = std::max(covering.end_sample, word.span.end_sample); } return {{covering}, {}}; } if (!text_output.empty()) { // A zero span rather than a fabricated duration: the model reported no // timing, and inventing one would be a lie the caller cannot detect. return {{Span{0, 0}}, {}}; } return {}; } // Returns the index of the segment a word belongs to, or the nearest segment // when the word falls outside all of them. size_t segment_index_for_word(const std::vector &spans, const Span &word) { const std::int64_t centre = midpoint(word); for (size_t i = 0; i < spans.size(); ++i) { if (contains(spans[i], centre)) { return i; } } size_t nearest = 0; std::int64_t best_distance = std::numeric_limits::max(); for (size_t i = 0; i < spans.size(); ++i) { const std::int64_t distance = std::llabs(midpoint(spans[i]) - centre); if (distance < best_distance) { best_distance = distance; nearest = i; } } return nearest; } } // namespace AssembledTranscript assemble_transcript(const std::string &text_output, const std::vector &speech_segments, const std::vector &speaker_turns, const std::vector &words, int sample_rate) { AssembledTranscript assembled; // THE RULE. Never derived from spans. assembled.text = text_output; const SegmentSource source = choose_segment_spans(text_output, speech_segments, speaker_turns, words); const std::vector &spans = source.spans; if (spans.empty()) { return assembled; } assembled.segments.resize(spans.size()); for (size_t i = 0; i < spans.size(); ++i) { OutSegment &segment = assembled.segments[i]; segment.id = static_cast(i); segment.start_ns = samples_to_nanoseconds(spans[i].start_sample, sample_rate); segment.end_ns = samples_to_nanoseconds(spans[i].end_sample, sample_rate); // A segment that came from a speaker turn already knows its speaker. // Only the other three sources have to look one up by overlap. segment.speaker = source.labels.empty() ? speaker_for(spans[i], speaker_turns) : source.labels[i]; } for (const auto &word : words) { const size_t index = segment_index_for_word(spans, word.span); OutWord out; out.start_ns = samples_to_nanoseconds(word.span.start_sample, sample_rate); out.end_ns = samples_to_nanoseconds(word.span.end_sample, sample_rate); out.text = word.word; assembled.segments[index].words.push_back(out); } for (auto &segment : assembled.segments) { segment.text = join_words(segment.words); } // A single segment with no word timing carries the whole transcript. With // several segments there is no defensible way to split the text, so their // per-segment text stays empty and only the top-level text is authoritative. if (assembled.segments.size() == 1 && assembled.segments[0].words.empty()) { assembled.segments[0].text = text_output; } return assembled; } } // namespace audiocpp_backend