diff --git a/backend/backend.proto b/backend/backend.proto index 54255528e..9ec5f862f 100644 --- a/backend/backend.proto +++ b/backend/backend.proto @@ -647,6 +647,7 @@ message TranscriptWord { int64 start = 1; int64 end = 2; string text = 3; + string speaker = 4; // backend speaker label when diarizing; empty otherwise } message TranscriptSegment { diff --git a/core/backend/transcript.go b/core/backend/transcript.go index 0ddee3add..27b13bc55 100644 --- a/core/backend/transcript.go +++ b/core/backend/transcript.go @@ -213,9 +213,10 @@ func transcriptResultFromProto(r *proto.TranscriptResult) *schema.TranscriptionR var words []schema.TranscriptionWord for _, w := range s.Words { var word = schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, } words = append(words, word) tr.Words = append(tr.Words, word) diff --git a/core/backend/transcript_live.go b/core/backend/transcript_live.go index 0ad6d72e1..d0dc59d87 100644 --- a/core/backend/transcript_live.go +++ b/core/backend/transcript_live.go @@ -298,9 +298,10 @@ func liveEventFromProto(r *proto.TranscriptLiveResponse) LiveTranscriptionEvent } for _, w := range r.GetWords() { ev.Words = append(ev.Words, schema.TranscriptionWord{ - Start: time.Duration(w.Start), - End: time.Duration(w.End), - Text: w.Text, + Start: time.Duration(w.Start), + End: time.Duration(w.End), + Text: w.Text, + Speaker: w.Speaker, }) } if r.GetFinalResult() != nil { diff --git a/core/backend/transcript_live_internal_test.go b/core/backend/transcript_live_internal_test.go index 6f6bed6a4..975ac844b 100644 --- a/core/backend/transcript_live_internal_test.go +++ b/core/backend/transcript_live_internal_test.go @@ -54,6 +54,20 @@ var _ = Describe("liveEventFromProto", func() { Expect(ev.Final).To(BeNil()) }) + It("carries word speakers and final segment speakers from a diarizing backend", func() { + ev := liveEventFromProto(&proto.TranscriptLiveResponse{ + Words: []*proto.TranscriptWord{{Text: "hi", Speaker: "1"}}, + }) + Expect(ev.Words[0].Speaker).To(Equal("1")) + ev = liveEventFromProto(&proto.TranscriptLiveResponse{ + FinalResult: &proto.TranscriptResult{ + Text: "hi there", + Segments: []*proto.TranscriptSegment{{Text: "hi", Speaker: "0"}, {Text: "there", Speaker: "1"}}, + }, + }) + Expect(ev.Final.Segments[1].Speaker).To(Equal("1")) + }) + It("maps the eob backchannel flag separately from eou", func() { ev := liveEventFromProto(&proto.TranscriptLiveResponse{Delta: "uh-huh", Eob: true}) Expect(ev.Eob).To(BeTrue()) diff --git a/core/cli/transcript.go b/core/cli/transcript.go index 06764f4dd..e187453a6 100644 --- a/core/cli/transcript.go +++ b/core/cli/transcript.go @@ -93,18 +93,20 @@ func (t *TranscriptCMD) Run(ctx *cliContext.Context) error { } for _, word := range(tr.Words) { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range(tr.Segments) { segWords := []schema.TranscriptionWordSeconds{} for _, word := range(seg.Words) { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ diff --git a/core/http/endpoints/openai/transcription.go b/core/http/endpoints/openai/transcription.go index 920094f84..a7e01ba95 100644 --- a/core/http/endpoints/openai/transcription.go +++ b/core/http/endpoints/openai/transcription.go @@ -191,18 +191,20 @@ func TranscriptEndpoint(cl *config.ModelConfigLoader, ml *model.ModelLoader, app } for _, word := range tr.Words { trs.Words = append(trs.Words, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } for _, seg := range tr.Segments { segWords := []schema.TranscriptionWordSeconds{} for _, word := range seg.Words { segWords = append(segWords, schema.TranscriptionWordSeconds{ - Start: word.Start.Seconds(), - End: word.End.Seconds(), - Text: word.Text, + Start: word.Start.Seconds(), + End: word.End.Seconds(), + Text: word.Text, + Speaker: word.Speaker, }) } trs.Segments = append(trs.Segments, schema.TranscriptionSegmentSeconds{ @@ -309,12 +311,16 @@ func streamTranscription(c echo.Context, req backend.TranscriptionRequest, ml *m if len(finalResult.Segments) > 0 { segs := make([]map[string]any, 0, len(finalResult.Segments)) for _, seg := range finalResult.Segments { - segs = append(segs, map[string]any{ + entry := map[string]any{ "id": seg.Id, "start": seg.Start.Seconds(), "end": seg.End.Seconds(), "text": seg.Text, - }) + } + if seg.Speaker != "" { + entry["speaker"] = seg.Speaker + } + segs = append(segs, entry) } doneEvent["segments"] = segs } diff --git a/core/schema/transcription.go b/core/schema/transcription.go index 8414fd0ba..aab914f70 100644 --- a/core/schema/transcription.go +++ b/core/schema/transcription.go @@ -13,9 +13,10 @@ type TranscriptionSegment struct { } type TranscriptionWord struct { - Start time.Duration `json:"start"` - End time.Duration `json:"end"` - Text string `json:"text"` + Start time.Duration `json:"start"` + End time.Duration `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResult struct { @@ -42,9 +43,10 @@ type TranscriptionSegmentSeconds struct { } type TranscriptionWordSeconds struct { - Start float64 `json:"start"` - End float64 `json:"end"` - Text string `json:"text"` + Start float64 `json:"start"` + End float64 `json:"end"` + Text string `json:"text"` + Speaker string `json:"speaker,omitempty"` } type TranscriptionResultSeconds struct {