mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-05 04:24:39 -04:00
feat(parakeet-cpp): VAD (Silero and the Moondream head), vad_model option and gallery entries (#12463)
* feat(parakeet-cpp): implement the VAD call and add the vad_model option The backend now serves the VAD gRPC call (POST /vad and /v1/vad) with the standalone VAD of libparakeet. It accepts a Silero VAD GGUF as the model file, or an ASR model with a VAD head (Moondream Ultra and Redux). The request audio is float32 PCM at 16 kHz; the response lists the speech segments in seconds, like the silero-vad backend. A model with neither fails the request with the library message. The vad_threshold, vad_min_pause, vad_min_speech, vad_speech_pad and vad_max_segment options tune the segmenter. Unset values keep the defaults of the detector in use, and a bad value fails the load. The vad_model option names a Silero GGUF, resolved against the models directory like the other companion files. It lets any ASR model cut long audio at pauses through parakeet_capi_transcribe_path_json_vad_with, and it implies vad. vad:true alone still uses the model's own head. The new symbols are probed like the existing optional ones. A library without them still loads; the feature that needs one fails with a clear message only when it is used. Assisted-by: Claude:claude-sonnet-5-5 [go test] * feat(gallery): add parakeet-cpp VAD entries and a v3 plus Silero example Add VAD-only entries for the parakeet-cpp backend: the VAD heads of Moondream Redux (packed, CPU) and Ultra (Q8_0), which share their files with the existing ASR entries, and Silero VAD v6.2.3 as a GGUF (MIT, Silero Team). The parakeet-cpp-vad entry installs Silero; it has no variants, because variant ranking prefers the larger build that fits and these are different detectors. Add parakeet-cpp-tdt-0.6b-v3-silero-vad, a v3 entry that sets vad_model so long audio is cut at pauses by Silero. The Silero GGUF entries point at the intended Hugging Face URL of the file; the existing silero-vad entries are unchanged. A test checks the usecases, the shared files and the default entry and the vad_model reference. Assisted-by: Claude:claude-sonnet-5-5 [go test] * docs: describe parakeet-cpp VAD and the vad_model option Document the VAD endpoint on the parakeet-cpp backend (Silero GGUF and the VAD heads of Moondream Ultra and Redux), the vad_* tuning options, and the vad_model option that lets an ASR model without a VAD head cut long audio with Silero. Assisted-by: Claude:claude-sonnet-5-5 * chore(parakeet-cpp): bump parakeet.cpp to 6165e3d Pin the release that adds the standalone VAD (Ultra/Redux head and Silero) and the C API calls the backend now uses. Assisted-by: Claude:claude-sonnet-5-5 --------- Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
1 parent
00faf4ec26
commit
6c718fdda7
10 files changed
+807
-12
No files matched your search
@@ -0,0 +1,94 @@
|
||||
package gallery_test
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
|
||||
"github.com/mudler/LocalAI/core/gallery"
|
||||
)
|
||||
|
||||
// The parakeet-cpp VAD entries serve the VAD endpoint through known_usecases
|
||||
// and the vad_model option points at a file the same entry downloads. A typo in
|
||||
// either one installs cleanly and fails at the first request.
|
||||
var _ = Describe("gallery/index.yaml parakeet-cpp VAD entries", func() {
|
||||
byName := func() map[string]gallery.GalleryModel {
|
||||
entries, err := loadGalleryIndex()
|
||||
Expect(err).ToNot(HaveOccurred())
|
||||
m := map[string]gallery.GalleryModel{}
|
||||
for _, e := range entries {
|
||||
m[e.Name] = e
|
||||
}
|
||||
return m
|
||||
}
|
||||
filenames := func(e gallery.GalleryModel) []string {
|
||||
var out []string
|
||||
for _, f := range e.AdditionalFiles {
|
||||
out = append(out, f.Filename)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
It("declares the VAD usecase and an existing model file on each VAD-only entry", func() {
|
||||
entries := byName()
|
||||
for _, name := range []string{
|
||||
"parakeet-cpp-silero-vad-f16",
|
||||
"parakeet-cpp-vad-moondream-redux-packed",
|
||||
"parakeet-cpp-vad-moondream-ultra-q8_0",
|
||||
} {
|
||||
e, ok := entries[name]
|
||||
Expect(ok).To(BeTrue(), name)
|
||||
Expect(e.Overrides["backend"]).To(Equal("parakeet-cpp"), name)
|
||||
Expect(e.Overrides["known_usecases"]).To(ConsistOf("vad"), name)
|
||||
params, _ := e.Overrides["parameters"].(map[string]any)
|
||||
Expect(filenames(e)).To(ContainElement(params["model"]), name)
|
||||
for _, f := range e.AdditionalFiles {
|
||||
Expect(f.SHA256).To(HaveLen(64), name)
|
||||
}
|
||||
}
|
||||
})
|
||||
|
||||
It("shares the VAD head files with the matching ASR entries", func() {
|
||||
entries := byName()
|
||||
pairs := map[string]string{
|
||||
"parakeet-cpp-vad-moondream-redux-packed": "parakeet-cpp-moondream-redux-packed",
|
||||
"parakeet-cpp-vad-moondream-ultra-q8_0": "parakeet-cpp-moondream-ultra-q8_0",
|
||||
}
|
||||
for vad, asr := range pairs {
|
||||
Expect(entries[vad].AdditionalFiles[0].Filename).To(Equal(entries[asr].AdditionalFiles[0].Filename))
|
||||
Expect(entries[vad].AdditionalFiles[0].SHA256).To(Equal(entries[asr].AdditionalFiles[0].SHA256))
|
||||
}
|
||||
})
|
||||
|
||||
It("installs Silero from the parakeet-cpp-vad entry and declares no variants", func() {
|
||||
// Variant ranking prefers the larger build that fits, and these are
|
||||
// different detectors, so the entry must not offer a choice.
|
||||
meta, ok := byName()["parakeet-cpp-vad"]
|
||||
Expect(ok).To(BeTrue())
|
||||
Expect(meta.Variants).To(BeEmpty())
|
||||
Expect(filenames(meta)).To(ConsistOf("parakeet-cpp/silero-vad-f16.gguf"))
|
||||
Expect(meta.Overrides["parameters"]).To(HaveKeyWithValue("model", "parakeet-cpp/silero-vad-f16.gguf"))
|
||||
})
|
||||
|
||||
It("downloads the Silero file that the vad_model option of the v3 entry names", func() {
|
||||
e, ok := byName()["parakeet-cpp-tdt-0.6b-v3-silero-vad"]
|
||||
Expect(ok).To(BeTrue())
|
||||
opts, _ := e.Overrides["options"].([]any)
|
||||
var vadModel string
|
||||
for _, o := range opts {
|
||||
if s, ok := o.(string); ok && strings.HasPrefix(s, "vad_model:") {
|
||||
vadModel = strings.TrimPrefix(s, "vad_model:")
|
||||
}
|
||||
}
|
||||
Expect(vadModel).ToNot(BeEmpty())
|
||||
Expect(filenames(e)).To(ContainElement(filepath.ToSlash(vadModel)))
|
||||
})
|
||||
|
||||
It("leaves the existing Silero entries on their own backends", func() {
|
||||
entries := byName()
|
||||
Expect(entries["silero-vad-ggml"].Overrides["backend"]).To(Equal("whisper"))
|
||||
Expect(entries["silero-vad"].Overrides["backend"]).To(Equal("silero-vad"))
|
||||
})
|
||||
})
|
||||
Reference in new issue
Block a user