mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
localai-org-maint-botandEttore Di Giacinto
6b794651a4
* feat(diarization): return sound events with include_sounds
A client that wants text, speakers, voice prints and sound events had to
make a diarization call and a separate sound call. Add an include_sounds
request field to /v1/audio/diarization that adds a sounds array of closed
events {start, end, label, confidence}, in seconds.
The parakeet-cpp backend runs a tagger-only scene stream over the clip,
the same stream and thresholds the live path uses, so a clip gives the
same events offline and live. A model with no sound_model companion, or a
backend that does not report sound events, fails with 501 and the stable
code include_sounds_unsupported instead of an empty list. The proto
carries sounds_included so an empty list still means "nothing heard".
The localai-proxy backend forwards the field. Swagger, docs and the
e2e mock backend are updated.
Assisted-by: Claude:claude-sonnet-5-5 [protoc swag go]
* feat(gallery): add parakeet-cpp-multilingual-diarization-speakers-sounds
Same as parakeet-cpp-multilingual-diarization-speakers (TDT 0.6B v3,
Nemotron-3-Diarization, WeSpeaker) plus a CED-Tiny sound_model, so one
model name serves /v1/audio/diarization with include_text,
include_speaker_profiles and include_sounds. It declares the
sound_classification usecase like the realtime scene entries.
Assisted-by: Claude:claude-sonnet-5-5
---------
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
41 lines
1.5 KiB
Go
41 lines
1.5 KiB
Go
// SPDX-License-Identifier: MIT
|
|
package backend
|
|
|
|
import (
|
|
"encoding/json"
|
|
|
|
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
)
|
|
|
|
var _ = Describe("diarization sound events", func() {
|
|
It("sends include_sounds to the backend only when asked", func() {
|
|
Expect((&DiarizationRequest{IncludeSounds: true}).toProto(1, "m").IncludeSounds).To(BeTrue())
|
|
Expect((&DiarizationRequest{}).toProto(1, "m").IncludeSounds).To(BeFalse())
|
|
})
|
|
|
|
It("maps proto sounds to the result and keeps an empty list when the backend heard nothing", func() {
|
|
out := diarizationResultFromProto(&pb.DiarizeResponse{
|
|
SoundsIncluded: true,
|
|
Sounds: []*pb.DiarizeSound{{Start: 1, End: 2.5, Label: "Cough", Confidence: 0.5}},
|
|
})
|
|
Expect(out.Sounds).To(HaveLen(1))
|
|
Expect(out.Sounds[0].Label).To(Equal("Cough"))
|
|
Expect(out.Sounds[0].Start).To(BeNumerically("~", 1, 1e-6))
|
|
Expect(out.Sounds[0].End).To(BeNumerically("~", 2.5, 1e-6))
|
|
Expect(out.Sounds[0].Confidence).To(BeNumerically("~", 0.5, 1e-6))
|
|
|
|
empty := diarizationResultFromProto(&pb.DiarizeResponse{SoundsIncluded: true})
|
|
raw, err := json.Marshal(empty)
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(string(raw)).To(ContainSubstring(`"sounds":[]`))
|
|
})
|
|
|
|
It("leaves sounds out of the payload when they were not requested", func() {
|
|
raw, err := json.Marshal(diarizationResultFromProto(&pb.DiarizeResponse{}))
|
|
Expect(err).ToNot(HaveOccurred())
|
|
Expect(string(raw)).ToNot(ContainSubstring("sounds"))
|
|
})
|
|
})
|