diff --git a/backend/go/parakeet-cpp/Makefile b/backend/go/parakeet-cpp/Makefile index 9a4ce573b..d1b02c5bf 100644 --- a/backend/go/parakeet-cpp/Makefile +++ b/backend/go/parakeet-cpp/Makefile @@ -1,6 +1,6 @@ # parakeet-cpp backend Makefile. # -# Upstream pin lives below as PARAKEET_VERSION?=e53a2539bd7fc3290951037696b953b34a9c4b9c +# Upstream pin lives below as PARAKEET_VERSION?=781a973e755bec5562ac2a0b04770ddffa36cad2 # (.github/bump_deps.sh) can find and update it - matches the # whisper.cpp / ds4 / vibevoice-cpp convention. # @@ -15,7 +15,7 @@ # That's what the L0 smoke test uses. The default target below does the # proper clone-at-pin + cmake build so CI doesn't need a side-checkout. -PARAKEET_VERSION?=e53a2539bd7fc3290951037696b953b34a9c4b9c +PARAKEET_VERSION?=781a973e755bec5562ac2a0b04770ddffa36cad2 PARAKEET_REPO?=https://github.com/mudler/parakeet.cpp GOCMD?=go diff --git a/backend/go/parakeet-cpp/bundle.go b/backend/go/parakeet-cpp/bundle.go new file mode 100644 index 000000000..d3bede81a --- /dev/null +++ b/backend/go/parakeet-cpp/bundle.go @@ -0,0 +1,189 @@ +package main + +import ( + "encoding/json" + "errors" + "fmt" + "strings" +) + +// Bundle GGUF support. A bundle is one GGUF file holding several models +// ("components"), each with its own kind and licence (docs/bundle.md in +// parakeet.cpp). The library opens one component per context; this file works +// out which component each role of a loaded model uses. +// +// Model options (key:value, all optional): +// +// bundle_asr: the ASR component of the primary file (or of the +// asr_model: file), when the bundle has several +// vad_component: the VAD component; with no vad_model: the VAD of the +// primary file +// diar_component: the diarization component; with no diarization_model: +// the one of the primary file +// sound_component: the sound (CED) component; likewise for sound_model: +// speaker_component: the speaker encoder component; likewise for speaker_model: +// +// A companion option (diarization_model: and the others) may name a bundle: the +// only component of the wanted kind is used, and a *_component option picks one +// when there are several. A Silero VAD component of the primary bundle is loaded +// without any option, because it costs little and serves the VAD endpoint. + +// Component kinds as the C-API reports them in the bundle component list. +const ( + componentASR = "asr" + componentVAD = "vad" + componentDiar = "diar" + componentSound = "ced" + componentVoice = "voice" +) + +// bundleComponent is one entry of parakeet_capi_bundle_components_json. The +// library reports more fields (sources, hashes); only the ones this backend +// reads are decoded. +type bundleComponent struct { + Name string `json:"name"` + Kind string `json:"kind"` + License string `json:"license"` + Bytes int64 `json:"bytes"` +} + +// componentKindModel maps a component kind to the model kind the loaded context +// must report. +func componentKindModel(kind string) (int32, bool) { + switch kind { + case componentASR: + return modelKindASR, true + case componentVAD: + return modelKindVAD, true + case componentDiar: + return modelKindDiarization, true + case componentSound: + return modelKindSound, true + case componentVoice: + return modelKindSpeaker, true + } + return 0, false +} + +// kindForModel is the reverse of componentKindModel. +func kindForModel(model int32) string { + switch model { + case modelKindASR: + return componentASR + case modelKindVAD: + return componentVAD + case modelKindDiarization: + return componentDiar + case modelKindSound: + return componentSound + case modelKindSpeaker: + return componentVoice + } + return "" +} + +// bundleComponents lists the components of the file at path. It returns nil +// when the file is not a bundle, and also when the library has no bundle +// support (the three bundle symbols are probed together in main.go), so callers +// treat both as a plain file. Only the header of the file is read. +func bundleComponents(path string) ([]bundleComponent, error) { + if CppBundleComponentsJSON == nil { + return nil, nil + } + cstr := CppBundleComponentsJSON(path) + if cstr == 0 { + return nil, nil + } + raw := goStringFromCPtr(cstr) + CppFreeString(cstr) + var comps []bundleComponent + if err := json.Unmarshal([]byte(raw), &comps); err != nil { + return nil, fmt.Errorf("parakeet-cpp: decode bundle component list of %q: %w", path, err) + } + return comps, nil +} + +// describeComponents renders a component list for error messages, for example +// "asr (asr), vad (vad)": name, then kind. +func describeComponents(comps []bundleComponent) string { + parts := make([]string, 0, len(comps)) + for _, c := range comps { + parts = append(parts, fmt.Sprintf("%s (%s)", c.Name, c.Kind)) + } + return strings.Join(parts, ", ") +} + +// componentsOfKind returns the names of the components of one kind. +func componentsOfKind(comps []bundleComponent, kind string) []string { + var names []string + for _, c := range comps { + if c.Kind == kind { + names = append(names, c.Name) + } + } + return names +} + +// pickComponent chooses the component of kind wantKind in a bundle. A name set +// by the user must exist and have that kind; with no name the bundle must hold +// exactly one component of the kind. what names the role for the messages and +// nameOpt the option that picks a component. +func pickComponent(path string, comps []bundleComponent, name, wantKind, what, nameOpt string) (string, error) { + if name != "" { + for _, c := range comps { + if c.Name != name { + continue + } + if c.Kind != wantKind { + return "", fmt.Errorf("parakeet-cpp: component %q of %q has kind %q, but %s needs kind %q", + name, path, c.Kind, what, wantKind) + } + return name, nil + } + return "", fmt.Errorf("parakeet-cpp: %q has no component %q (components: %s)", + path, name, describeComponents(comps)) + } + names := componentsOfKind(comps, wantKind) + switch len(names) { + case 1: + return names[0], nil + case 0: + return "", fmt.Errorf("parakeet-cpp: %s needs a %q component, but the bundle %q has none (components: %s)", + what, wantKind, path, describeComponents(comps)) + default: + return "", fmt.Errorf("parakeet-cpp: the bundle %q has several %q components (%s); set %s to pick one", + path, wantKind, strings.Join(names, ", "), nameOpt) + } +} + +// loadComponent opens one component of a bundle and returns the library's +// reason on failure. +func loadComponent(path, name string) (uintptr, error) { + if CppLoadComponent == nil { + return 0, errors.New("parakeet-cpp: bundle components need a libparakeet.so with parakeet_capi_load_component; rebuild the backend against a newer parakeet.cpp") + } + ctx := CppLoadComponent(path, name) + if ctx == 0 { + reason := "" + if CppLoadError != nil { + reason = CppLoadError() + } + return 0, fmt.Errorf("parakeet-cpp: failed to load component %q of %q: %s", name, path, reason) + } + return ctx, nil +} + +// roleHint ends the error of an RPC that needs a role the loaded model does not +// have. For a bundle it says whether the bundle has a component of that kind +// (and which option loads it) or lacks one; for any other model it is empty. +func (p *ParakeetCpp) roleHint(kind, option string) string { + if len(p.bundle) == 0 { + return "" + } + if names := componentsOfKind(p.bundle, kind); len(names) > 0 { + return fmt.Sprintf(" (the model file is a bundle with a %q component %s: set %s:%s to load it)", + kind, strings.Join(names, ", "), option, names[0]) + } + return fmt.Sprintf(" (the model file is a bundle without a %q component; components: %s)", + kind, describeComponents(p.bundle)) +} diff --git a/backend/go/parakeet-cpp/bundle_test.go b/backend/go/parakeet-cpp/bundle_test.go new file mode 100644 index 000000000..2925e070e --- /dev/null +++ b/backend/go/parakeet-cpp/bundle_test.go @@ -0,0 +1,349 @@ +package main + +import ( + "context" + "encoding/json" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// bundleLib extends fakeLib with bundle files. Paths registered with withBundle +// answer the component list and open components by name; the plain +// CppLoad on a bundle behaves like the library (the only ASR component, else +// the only component). +type bundleLib struct { + *fakeLib + bundles map[string][]bundleComponent + pool liveCstrPool + loadedComp []string + loadErr string +} + +func newBundleLib() *bundleLib { + return &bundleLib{fakeLib: newFakeLib(), bundles: map[string][]bundleComponent{}} +} + +func (b *bundleLib) withBundle(path string, comps ...bundleComponent) *bundleLib { + b.bundles[path] = comps + return b +} + +func (b *bundleLib) withPlain(path string, kind int32) *bundleLib { + b.fakeLib.withModel(path, kind) + return b +} + +func (b *bundleLib) open(kind string) uintptr { + mk, _ := componentKindModel(kind) + ctx := b.next + b.next++ + b.ctxKind[ctx] = mk + return ctx +} + +func (b *bundleLib) install() (restore func()) { + restoreBase := b.fakeLib.install() + savedJSON, savedComp, savedErr, savedFreeStr, savedLoad := CppBundleComponentsJSON, CppLoadComponent, CppLoadError, CppFreeString, CppLoad + baseLoad := CppLoad + CppFreeString = func(uintptr) {} + CppBundleComponentsJSON = func(path string) uintptr { + comps, ok := b.bundles[path] + if !ok { + return 0 + } + raw, _ := json.Marshal(comps) + return b.pool.cstr(string(raw)) + } + CppLoadError = func() string { return b.loadErr } + CppLoadComponent = func(path, name string) uintptr { + for _, c := range b.bundles[path] { + if c.Name == name { + b.loadedComp = append(b.loadedComp, path+"#"+name) + return b.open(c.Kind) + } + } + b.loadErr = "no such component " + name + return 0 + } + CppLoad = func(path string) uintptr { + comps, ok := b.bundles[path] + if !ok { + return baseLoad(path) + } + asr := componentsOfKind(comps, componentASR) + switch { + case len(asr) == 1: + return CppLoadComponent(path, asr[0]) + case len(comps) == 1: + return CppLoadComponent(path, comps[0].Name) + } + b.loadErr = "bundle has several candidates" + return 0 + } + return func() { + restoreBase() + CppBundleComponentsJSON, CppLoadComponent, CppLoadError, CppFreeString, CppLoad = savedJSON, savedComp, savedErr, savedFreeStr, savedLoad + } +} + +var smallBundle = []bundleComponent{ + {Name: "asr", Kind: "asr", License: "CC-BY-4.0"}, + {Name: "diar", Kind: "diar", License: "OpenMDW-1.1"}, + {Name: "ced", Kind: "ced", License: "Apache-2.0"}, + {Name: "voice", Kind: "voice", License: "CC-BY-4.0"}, + {Name: "vad", Kind: "vad", License: "MIT"}, +} + +var _ = Describe("bundle GGUF files (stubbed C API)", func() { + var restore func() + var savedAdd func(uintptr, string, *float32, int32) int32 + var savedDim func(uintptr) int32 + var savedBegin func(asr, diar, tagger, speaker, reg uintptr, o *cSceneOpts) uintptr + + BeforeEach(func() { + savedAdd, savedDim, savedBegin = CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSceneStreamBeginSpeaker + CppSpeakerRegistryAddEmbedding = func(uintptr, string, *float32, int32) int32 { return 0 } + CppSpeakerDim = func(uintptr) int32 { return 256 } + CppSceneStreamBeginSpeaker = func(_, _, _, _, _ uintptr, _ *cSceneOpts) uintptr { return 0 } + }) + AfterEach(func() { + if restore != nil { + restore() + restore = nil + } + CppSpeakerRegistryAddEmbedding, CppSpeakerDim, CppSceneStreamBeginSpeaker = savedAdd, savedDim, savedBegin + }) + + load := func(b *bundleLib, opts ...string) (*ParakeetCpp, error) { + restore = b.install() + p := &ParakeetCpp{} + return p, p.Load(&pb.ModelOptions{ModelFile: "small.gguf", Options: opts}) + } + + It("opens the ASR component and the Silero component of a bundle with no options", func() { + b := newBundleLib().withBundle("small.gguf", smallBundle...) + p, err := load(b) + Expect(err).ToNot(HaveOccurred()) + Expect(b.loadedComp).To(Equal([]string{"small.gguf#asr", "small.gguf#vad"})) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(p.diarCtx).To(BeZero(), "diarization is opt-in") + Expect(p.tagCtx).To(BeZero()) + Expect(p.bundle).To(HaveLen(5)) + Expect(p.vad).To(BeFalse(), "vad:true stays opt-in") + }) + + It("loads every role of a bundle from the component options and frees each context once", func() { + CppTranscribePathJSONVad = func(uintptr, string, int32) uintptr { return 0 } + DeferCleanup(func() { CppTranscribePathJSONVad = nil }) + b := newBundleLib().withBundle("small.gguf", smallBundle...) + p, err := load(b, "diar_component:diar", "sound_component:ced", "speaker_component:voice", "vad:true") + Expect(err).ToNot(HaveOccurred()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.tagCtx).ToNot(BeZero()) + Expect(p.spkCtx).ToNot(BeZero()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(p.vad).To(BeTrue()) + Expect(p.Free()).To(Succeed()) + Expect(b.freed).To(HaveLen(5)) + }) + + It("takes the same file for a role through the companion option", func() { + b := newBundleLib().withBundle("small.gguf", smallBundle...) + p, err := load(b, "diarization_model:small.gguf", "sound_model:small.gguf") + Expect(err).ToNot(HaveOccurred()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.tagCtx).ToNot(BeZero()) + Expect(b.loadedComp).To(ContainElements("small.gguf#diar", "small.gguf#ced")) + }) + + It("uses a bundle as the companion of a plain primary", func() { + b := newBundleLib(). + withBundle("small.gguf", smallBundle...). + withPlain("asr.gguf", modelKindASR) + restore = b.install() + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "asr.gguf", Options: []string{"diarization_model:small.gguf"}}) + Expect(err).ToNot(HaveOccurred()) + Expect(p.diarCtx).ToNot(BeZero()) + Expect(p.bundle).To(BeNil(), "the primary is a plain file") + Expect(p.vadCtx).To(BeZero(), "the bundle VAD is only picked up from the primary file") + }) + + It("chooses an ASR component by bundle_asr", func() { + two := append([]bundleComponent{{Name: "fast", Kind: "asr"}, {Name: "big", Kind: "asr"}}, smallBundle[4]) + b := newBundleLib().withBundle("small.gguf", two...) + _, err := load(b) + Expect(err).To(MatchError(And(ContainSubstring("several"), ContainSubstring("fast, big"), ContainSubstring("bundle_asr")))) + Expect(b.loadedComp).To(BeEmpty()) + + b2 := newBundleLib().withBundle("small.gguf", two...) + p, err := load(b2, "bundle_asr:big") + Expect(err).ToNot(HaveOccurred()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(b2.loadedComp).To(ContainElement("small.gguf#big")) + }) + + It("rejects an unknown or wrong-kind component name and frees what it opened", func() { + b := newBundleLib().withBundle("small.gguf", smallBundle...) + _, err := load(b, "bundle_asr:nope") + Expect(err).To(MatchError(And(ContainSubstring(`no component "nope"`), ContainSubstring("vad (vad)")))) + + b2 := newBundleLib().withBundle("small.gguf", smallBundle...) + _, err = load(b2, "diar_component:vad") + Expect(err).To(MatchError(ContainSubstring(`kind "vad"`))) + Expect(b2.freed).To(HaveLen(len(b2.loadedComp)), "every opened context is freed") + }) + + It("gives a clear error for a role the bundle has no component for", func() { + noDiar := []bundleComponent{smallBundle[0], smallBundle[4]} + b := newBundleLib().withBundle("small.gguf", noDiar...) + _, err := load(b, "diar_component:diar") + Expect(err).To(MatchError(ContainSubstring(`no component "diar"`))) + + b2 := newBundleLib().withBundle("small.gguf", noDiar...) + _, err = load(b2, "diarization_model:small.gguf") + Expect(err).To(MatchError(And(ContainSubstring(`needs a "diar" component`), ContainSubstring("asr (asr), vad (vad)")))) + }) + + It("names the role mismatch in the RPC errors of a bundle", func() { + noDiar := []bundleComponent{smallBundle[0], smallBundle[4]} + b := newBundleLib().withBundle("small.gguf", noDiar...) + p, err := load(b) + Expect(err).ToNot(HaveOccurred()) + + _, err = p.Diarize(&pb.DiarizeRequest{Dst: "x.wav"}) + Expect(err).To(MatchError(And(ContainSubstring("not a diarization model"), ContainSubstring(`without a "diar" component`)))) + + _, err = p.SoundDetection(context.Background(), &pb.SoundDetectionRequest{Src: "x.wav"}) + Expect(err).To(MatchError(ContainSubstring(`without a "ced" component`))) + + full := newBundleLib().withBundle("small.gguf", smallBundle...) + restore() + p, err = load(full) + Expect(err).ToNot(HaveOccurred()) + _, err = p.Diarize(&pb.DiarizeRequest{Dst: "x.wav"}) + Expect(err).To(MatchError(ContainSubstring("diar_component:diar"))) + }) + + It("does not load the VAD component when vad_model or vad_component is given", func() { + b := newBundleLib(). + withBundle("small.gguf", smallBundle...). + withPlain("silero.gguf", modelKindVAD) + restore = b.install() + p := &ParakeetCpp{} + CppTranscribePathJSONVadWith = func(ctx, vadCtx uintptr, wav string, dec int32, o string) uintptr { return 0 } + DeferCleanup(func() { CppTranscribePathJSONVadWith = nil }) + Expect(p.Load(&pb.ModelOptions{ModelFile: "small.gguf", Options: []string{"vad_model:silero.gguf"}})).To(Succeed()) + Expect(b.loadedComp).To(Equal([]string{"small.gguf#asr"})) + Expect(p.vad).To(BeTrue()) + }) + + It("skips an ambiguous bundle VAD silently, and refuses it when vad_component is set without a name match", func() { + two := append(append([]bundleComponent{}, smallBundle...), bundleComponent{Name: "vad2", Kind: "vad"}) + b := newBundleLib().withBundle("small.gguf", two...) + p, err := load(b) + Expect(err).ToNot(HaveOccurred()) + Expect(p.vadCtx).To(BeZero()) + + CppTranscribePathJSONVadWith = func(ctx, vadCtx uintptr, wav string, dec int32, o string) uintptr { return 0 } + DeferCleanup(func() { CppTranscribePathJSONVadWith = nil }) + restore() + b2 := newBundleLib().withBundle("small.gguf", two...) + p, err = load(b2, "vad_component:vad2") + Expect(err).ToNot(HaveOccurred()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(b2.loadedComp).To(ContainElement("small.gguf#vad2")) + }) + + It("rejects a component option on a file that is not a bundle", func() { + b := newBundleLib().withPlain("small.gguf", modelKindASR) + _, err := load(b, "diar_component:diar") + Expect(err).To(MatchError(ContainSubstring("not one"))) + Expect(b.freed).To(HaveLen(1), "the primary is freed") + + b2 := newBundleLib().withPlain("small.gguf", modelKindASR) + _, err = load(b2, "bundle_asr:asr") + Expect(err).To(MatchError(ContainSubstring("bundle_asr needs a bundle"))) + }) + + It("loads a plain file exactly as before when the library has the bundle symbols", func() { + b := newBundleLib().withPlain("small.gguf", modelKindASR) + p, err := load(b) + Expect(err).ToNot(HaveOccurred()) + Expect(p.ctxPtr).ToNot(BeZero()) + Expect(p.bundle).To(BeNil()) + Expect(b.loadedComp).To(BeEmpty()) + }) + + It("loads a bundle without an ASR component through the library default", func() { + b := newBundleLib().withBundle("small.gguf", bundleComponent{Name: "vad", Kind: "vad"}) + p, err := load(b) + Expect(err).ToNot(HaveOccurred()) + Expect(p.vadCtx).ToNot(BeZero()) + Expect(p.ctxPtr).To(BeZero()) + }) + + It("reports the library reason when a bundle component does not open", func() { + b := newBundleLib().withBundle("small.gguf", smallBundle...) + restore = b.install() + saved := CppLoadComponent + CppLoadComponent = func(string, string) uintptr { b.loadErr = "component asr is damaged"; return 0 } + DeferCleanup(func() { CppLoadComponent = saved }) + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "small.gguf"}) + Expect(err).To(MatchError(ContainSubstring("component asr is damaged"))) + }) + + Describe("a libparakeet.so without the bundle symbols", func() { + It("loads a plain file as before, treating the file as ASR", func() { + f := newFakeLib().withModel("a.gguf", modelKindASR) + restore = f.install() + savedJSON, savedComp := CppBundleComponentsJSON, CppLoadComponent + CppBundleComponentsJSON, CppLoadComponent = nil, nil + DeferCleanup(func() { CppBundleComponentsJSON, CppLoadComponent = savedJSON, savedComp }) + p := &ParakeetCpp{} + Expect(p.Load(&pb.ModelOptions{ModelFile: "a.gguf"})).To(Succeed()) + Expect(p.bundle).To(BeNil()) + }) + + It("refuses the component options", func() { + f := newFakeLib().withModel("a.gguf", modelKindASR) + restore = f.install() + savedJSON, savedComp := CppBundleComponentsJSON, CppLoadComponent + CppBundleComponentsJSON, CppLoadComponent = nil, nil + DeferCleanup(func() { CppBundleComponentsJSON, CppLoadComponent = savedJSON, savedComp }) + for _, opt := range []string{"bundle_asr:asr", "diar_component:diar", "sound_component:ced", "vad_component:vad"} { + p := &ParakeetCpp{} + err := p.Load(&pb.ModelOptions{ModelFile: "a.gguf", Options: []string{opt}}) + Expect(err).To(HaveOccurred(), opt) + Expect(err.Error()).To(Or(ContainSubstring("not one"), ContainSubstring("needs a bundle"), ContainSubstring("need")), opt) + } + }) + }) +}) + +var _ = Describe("bundle helpers", func() { + comps := []bundleComponent{{Name: "a", Kind: "asr"}, {Name: "v", Kind: "vad"}} + + It("maps kinds both ways", func() { + for _, k := range []string{"asr", "vad", "diar", "ced", "voice"} { + m, ok := componentKindModel(k) + Expect(ok).To(BeTrue(), k) + Expect(kindForModel(m)).To(Equal(k)) + } + _, ok := componentKindModel("future") + Expect(ok).To(BeFalse()) + }) + + It("picks the only component of a kind and names the alternatives otherwise", func() { + name, err := pickComponent("f", comps, "", "vad", "vad_model", "vad_component") + Expect(err).ToNot(HaveOccurred()) + Expect(name).To(Equal("v")) + _, err = pickComponent("f", comps, "", "ced", "sound_model", "sound_component") + Expect(err).To(MatchError(ContainSubstring("a (asr), v (vad)"))) + }) +}) diff --git a/backend/go/parakeet-cpp/diarize.go b/backend/go/parakeet-cpp/diarize.go index 8cbfd33f6..128c42007 100644 --- a/backend/go/parakeet-cpp/diarize.go +++ b/backend/go/parakeet-cpp/diarize.go @@ -116,7 +116,7 @@ func (p *ParakeetCpp) Diarize(req *pb.DiarizeRequest) (pb.DiarizeResponse, error } if p.diarCtx == 0 { return pb.DiarizeResponse{}, status.Error(codes.FailedPrecondition, - "parakeet-cpp: model is not a diarization model") + "parakeet-cpp: model is not a diarization model"+p.roleHint(componentDiar, "diar_component")) } if CppDiarizePCM == nil { return pb.DiarizeResponse{}, status.Error(codes.Unimplemented, diff --git a/backend/go/parakeet-cpp/goparakeetcpp.go b/backend/go/parakeet-cpp/goparakeetcpp.go index 931018326..270678da4 100644 --- a/backend/go/parakeet-cpp/goparakeetcpp.go +++ b/backend/go/parakeet-cpp/goparakeetcpp.go @@ -39,6 +39,18 @@ var ( CppFreeString func(s uintptr) CppLastError func(ctx uintptr) string + // Bundle GGUF (additive in the C-API, no ABI bump; see bundle.go). All three + // are registered together and nil on an older libparakeet.so, where a + // bundle file is loaded like any other file (the library then refuses it) + // and the *_component options are rejected. + // CppLoadComponent opens one named component of a bundle. + // CppBundleComponentsJSON returns the component list as a malloc'd JSON + // array (uintptr, freed via CppFreeString), or 0 when the file is not a bundle. + // CppLoadError is the reason of the last failed load on the calling thread. + CppLoadComponent func(ggufPath, component string) uintptr + CppBundleComponentsJSON func(ggufPath string) uintptr + CppLoadError func() string + // CppTranscribePathJSONVad is CppTranscribePathJSON with long audio cut at // pauses by the model's own VAD head (segments of at most 30 s; the document // has the same shape, times are relative to the whole file). Returns 0 and @@ -254,9 +266,12 @@ type ParakeetCpp struct { // primary (asr_model:/diarization_model:/sound_model: options), so Free // can release them after the primary. companions []uintptr - engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) - bat *batcher - batStop chan struct{} + // bundle lists the components of the primary model file when it is a bundle + // GGUF (bundle.go); nil for a plain file. + bundle []bundleComponent + engineMu sync.Mutex // sole guard of the one C engine (dispatcher + streaming) + bat *batcher + batStop chan struct{} // segmentGapFrames is NeMo's segment_gap_threshold in ENCODER FRAMES (model // YAML option, default 0=off). When >0 it adds NeMo's silence-gap split on // top of the punctuation split; converted to seconds via the JSON frame_sec. @@ -295,11 +310,11 @@ func (p *ParakeetCpp) Load(opts *pb.ModelOptions) error { return err } p.vadOptions = vadOpts - if optString(opts, "vad_model") != "" { + if optString(opts, "vad_model") != "" || optString(opts, "vad_component") != "" { if CppTranscribePathJSONVadWith == nil { - return errors.New("parakeet-cpp: vad_model needs a libparakeet.so with parakeet_capi_transcribe_path_json_vad_with; rebuild the backend against a newer parakeet.cpp") + return errors.New("parakeet-cpp: vad_model and vad_component need a libparakeet.so with parakeet_capi_transcribe_path_json_vad_with; rebuild the backend against a newer parakeet.cpp") } - // vad_model implies vad: a Silero model is only useful to cut audio. + // vad_model and vad_component imply vad: a Silero model is only useful to cut audio. p.vad = true } diff --git a/backend/go/parakeet-cpp/main.go b/backend/go/parakeet-cpp/main.go index 9022c0a9a..fe1bb8ce6 100644 --- a/backend/go/parakeet-cpp/main.go +++ b/backend/go/parakeet-cpp/main.go @@ -89,6 +89,19 @@ func main() { purego.RegisterLibFunc(&CppVadPcmJSON, lib, "parakeet_capi_vad_pcm_json") } + // Bundle GGUF (one file with several models; docs/bundle.md in parakeet.cpp). + // Additive in the C-API. The three entry points come together, so one probe + // decides; without them a bundle file is handled like any other file. + if sym, err := purego.Dlsym(lib, "parakeet_capi_load_component"); err == nil && sym != 0 { + if sym2, err2 := purego.Dlsym(lib, "parakeet_capi_bundle_components_json"); err2 == nil && sym2 != 0 { + if sym3, err3 := purego.Dlsym(lib, "parakeet_capi_load_error"); err3 == nil && sym3 != 0 { + purego.RegisterLibFunc(&CppLoadComponent, lib, "parakeet_capi_load_component") + purego.RegisterLibFunc(&CppBundleComponentsJSON, lib, "parakeet_capi_bundle_components_json") + purego.RegisterLibFunc(&CppLoadError, lib, "parakeet_capi_load_error") + } + } + } + // Per-request language variants (multilingual nemotron). Same probe pattern: // present only in libparakeet.so built with multilingual support, so the // backend still loads against an older library and falls back to the diff --git a/backend/go/parakeet-cpp/roles.go b/backend/go/parakeet-cpp/roles.go index 42ff6353c..de31f7cb9 100644 --- a/backend/go/parakeet-cpp/roles.go +++ b/backend/go/parakeet-cpp/roles.go @@ -103,6 +103,11 @@ type companionSpec struct { wantKind int32 assign func(*ParakeetCpp, uintptr) current func(*ParakeetCpp) uintptr + // component and compOpt are the bundle component name for this role and the + // option that sets it (bundle.go). With a component set and no value, the + // role is loaded from the primary model file, which must be a bundle. + component string + compOpt string } // indefiniteArticle returns "an" for a word starting with a vowel sound and @@ -137,17 +142,23 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { soundModelOpt := optString(opts, "sound_model") speakerModelOpt := optString(opts, "speaker_model") vadModelOpt := optString(opts, "vad_model") - hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" || vadModelOpt != "" + bundleASR := optString(opts, "bundle_asr") + vadComp := optString(opts, "vad_component") + diarComp := optString(opts, "diar_component") + soundComp := optString(opts, "sound_component") + speakerComp := optString(opts, "speaker_component") + hasCompanionOpts := diarModelOpt != "" || asrModelOpt != "" || soundModelOpt != "" || speakerModelOpt != "" || vadModelOpt != "" || + bundleASR != "" || vadComp != "" || diarComp != "" || soundComp != "" || speakerComp != "" if hasCompanionOpts && CppModelKind == nil { - return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model/vad_model options need " + + return errors.New("parakeet-cpp: asr_model/diarization_model/sound_model/speaker_model/vad_model and the bundle component options need " + "parakeet_capi_model_kind (ABI v8) to verify what they load; the loaded libparakeet.so " + "is too old to report companion model roles") } - if speakerModelOpt != "" { + if speakerModelOpt != "" || speakerComp != "" { if CppSpeakerRegistryAddEmbedding == nil || CppSpeakerDim == nil || CppSceneStreamBeginSpeaker == nil { - return errors.New("parakeet-cpp: speaker_model needs libparakeet.so ABI 10 " + + return errors.New("parakeet-cpp: speaker_model and speaker_component need libparakeet.so ABI 10 " + "(parakeet_capi_speaker_registry_add_embedding); the loaded library is older") } } @@ -165,18 +176,11 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { return err } - primary := CppLoad(opts.ModelFile) - if primary == 0 { - // No ctx to ask for last_error (the C-API's last-error buffer lives on - // the ctx that was never returned). Surface the path so the operator - // at least knows which load failed. - // - // A packed ternary Redux GGUF (redux-packed) is the likeliest cause on a - // GPU build: the library refuses it with a message that goes to its own - // log, not through the C-API, so name the cause here. - return fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q (see the backend log for the library message; "+ - "a packed ternary Redux model is CPU only and is refused on a GPU backend, use the redux-f16 or redux-q8_0 file there)", opts.ModelFile) + primary, comps, err := loadPrimary(opts.ModelFile, bundleASR) + if err != nil { + return err } + p.bundle = comps loaded := []uintptr{primary} // freeLoaded undoes everything loadRoles opened this call: every context // it freed AND every ParakeetCpp field it may have assigned (the primary @@ -189,6 +193,7 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { CppFree(c) } p.ctxPtr, p.diarCtx, p.tagCtx, p.spkCtx, p.vadCtx = 0, 0, 0, 0, 0 + p.bundle = nil p.companions = nil } @@ -218,23 +223,42 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { specs := []companionSpec{ {"diarization_model", diarModelOpt, modelKindDiarization, func(pp *ParakeetCpp, c uintptr) { pp.diarCtx = c }, - func(pp *ParakeetCpp) uintptr { return pp.diarCtx }}, + func(pp *ParakeetCpp) uintptr { return pp.diarCtx }, + diarComp, "diar_component"}, {"asr_model", asrModelOpt, modelKindASR, func(pp *ParakeetCpp, c uintptr) { pp.ctxPtr = c }, - func(pp *ParakeetCpp) uintptr { return pp.ctxPtr }}, + func(pp *ParakeetCpp) uintptr { return pp.ctxPtr }, + bundleASR, "bundle_asr"}, {"sound_model", soundModelOpt, modelKindSound, func(pp *ParakeetCpp, c uintptr) { pp.tagCtx = c }, - func(pp *ParakeetCpp) uintptr { return pp.tagCtx }}, + func(pp *ParakeetCpp) uintptr { return pp.tagCtx }, + soundComp, "sound_component"}, {"speaker_model", speakerModelOpt, modelKindSpeaker, func(pp *ParakeetCpp, c uintptr) { pp.spkCtx = c }, - func(pp *ParakeetCpp) uintptr { return pp.spkCtx }}, + func(pp *ParakeetCpp) uintptr { return pp.spkCtx }, + speakerComp, "speaker_component"}, {"vad_model", vadModelOpt, modelKindVAD, func(pp *ParakeetCpp, c uintptr) { pp.vadCtx = c }, - func(pp *ParakeetCpp) uintptr { return pp.vadCtx }}, + func(pp *ParakeetCpp) uintptr { return pp.vadCtx }, + vadComp, "vad_component"}, } for _, spec := range specs { + path := resolveModelPath(opts.ModelPath, spec.value) + fromPrimary := false if spec.value == "" { - continue + // bundle_asr names the ASR of the primary file, so on its own it is + // not a request for an asr_model companion. + if spec.component == "" || spec.optName == "asr_model" { + continue + } + // A component option with no companion file: the role comes from the + // primary file, which must be a bundle. + if comps == nil { + freeLoaded() + return fmt.Errorf("parakeet-cpp: %s needs the model file or a %s to be a bundle GGUF; %q is not one", + spec.compOpt, spec.optName, opts.ModelFile) + } + path, fromPrimary = opts.ModelFile, true } // A companion whose role the primary already occupies (e.g. asr_model: // on an already-ASR primary) would overwrite that role field below, @@ -245,30 +269,46 @@ func (p *ParakeetCpp) loadRoles(opts *pb.ModelOptions) error { return fmt.Errorf("parakeet-cpp: %s is not allowed on %s %s model", spec.optName, indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) } - resolved := resolveModelPath(opts.ModelPath, spec.value) - cctx := CppLoad(resolved) - if cctx == 0 { - freeLoaded() - return fmt.Errorf("parakeet-cpp: failed to load %s %q", spec.optName, resolved) + cctx, err := loadCompanion(spec, path, fromPrimary, comps) + if cctx != 0 { + loaded = append(loaded, cctx) } - loaded = append(loaded, cctx) - if gotKind := CppModelKind(cctx); gotKind != spec.wantKind { + if err != nil { freeLoaded() - return fmt.Errorf("parakeet-cpp: %s %q is %s %s model, expected %s %s model", - spec.optName, resolved, indefiniteArticle(modelKindName(gotKind)), modelKindName(gotKind), - indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) + return err } spec.assign(p, cctx) p.companions = append(p.companions, cctx) } - if vadModelOpt != "" && p.ctxPtr == 0 { + // A Silero VAD component of the primary bundle is the VAD source without any + // option, as in the parakeet-cli rules: it is small and serves the VAD + // endpoint and vad:true. Not finding one, or not being able to load it, is + // not an error here: the ASR head (if any) stays the fallback. + if vadModelOpt == "" && vadComp == "" && p.vadCtx == 0 && p.ctxPtr != 0 && comps != nil && CppLoadComponent != nil { + if names := componentsOfKind(comps, componentVAD); len(names) == 1 { + vctx, verr := loadComponent(opts.ModelFile, names[0]) + switch { + case verr != nil: + xlog.Warn("parakeet-cpp: bundle VAD component not loaded", "error", verr) + case CppModelKind(vctx) != modelKindVAD: + CppFree(vctx) + xlog.Warn("parakeet-cpp: bundle VAD component is not a Silero model; using the ASR VAD head if any", "component", names[0]) + default: + loaded = append(loaded, vctx) + p.vadCtx = vctx + p.companions = append(p.companions, vctx) + } + } + } + + if (vadModelOpt != "" || vadComp != "") && p.ctxPtr == 0 { freeLoaded() - return errors.New("parakeet-cpp: vad_model cuts audio for transcription and needs an ASR model (the primary or asr_model:)") + return errors.New("parakeet-cpp: vad_model and vad_component cut audio for transcription and needs an ASR model (the primary or asr_model:)") } if p.spkCtx != 0 && p.diarCtx == 0 { freeLoaded() - return errors.New("parakeet-cpp: speaker_model needs a diarization model (the primary or diarization_model:)") + return errors.New("parakeet-cpp: speaker_model needs a diarization model (the primary, diarization_model: or diar_component:)") } p.speakerAccept, p.speakerMargin = accept, margin p.diarLatency = latency @@ -296,3 +336,106 @@ func (p *ParakeetCpp) notASRError() error { return nil } } + +// loadPrimary opens the primary model file. A bundle is opened by component: the +// bundle_asr one, else its only ASR component. A bundle with several ASR +// components and no bundle_asr is refused with their names. A bundle with no ASR +// component is left to the library, which opens its only loadable component. +// Any other file is opened as before. comps lists the components of a bundle +// (nil for a plain file); nothing is loaded when it returns an error. +func loadPrimary(path, bundleASR string) (ctx uintptr, comps []bundleComponent, err error) { + comps, err = bundleComponents(path) + if err != nil { + return 0, nil, err + } + if comps == nil { + if bundleASR != "" { + return 0, nil, fmt.Errorf("parakeet-cpp: bundle_asr needs a bundle GGUF; %q is not one (or libparakeet.so has no bundle support)", path) + } + ctx = CppLoad(path) + if ctx == 0 { + // No ctx to ask for last_error (the C-API's last-error buffer lives on + // the ctx that was never returned). Surface the path so the operator + // at least knows which load failed. + // + // A packed ternary Redux GGUF (redux-packed) is the likeliest cause on a + // GPU build: the library refuses it with a message that goes to its own + // log, not through the C-API, so name the cause here. + return 0, nil, fmt.Errorf("parakeet-cpp: parakeet_capi_load failed for %q (see the backend log for the library message; "+ + "a packed ternary Redux model is CPU only and is refused on a GPU backend, use the redux-f16 or redux-q8_0 file there)", path) + } + return ctx, nil, nil + } + if bundleASR == "" && len(componentsOfKind(comps, componentASR)) == 0 { + ctx = CppLoad(path) + if ctx == 0 { + return 0, nil, fmt.Errorf("parakeet-cpp: failed to load the bundle %q: %s", path, loadErrorText()) + } + return ctx, comps, nil + } + name, err := pickComponent(path, comps, bundleASR, componentASR, "the primary model", "bundle_asr") + if err != nil { + return 0, nil, err + } + ctx, err = loadComponent(path, name) + if err != nil { + return 0, nil, withRedux(err) + } + return ctx, comps, nil +} + +// withRedux adds the CPU-only hint of a packed Redux component to a failed +// component load: the library refuses it on a GPU backend. +func withRedux(err error) error { + return fmt.Errorf("%w (a packed ternary Redux model is CPU only and is refused on a GPU backend)", err) +} + +func loadErrorText() string { + if CppLoadError == nil { + return "" + } + return CppLoadError() +} + +// loadCompanion loads the model of one companion role from path. A bundle gives +// its component of the wanted kind (named by the role's component option, else +// the only one); a plain file is loaded as before and refuses a component +// option. It returns the context even with an error when the context was +// opened but is not usable, so the caller can free it. fromPrimary tells that +// path is the primary file, already known to be a bundle with components comps. +func loadCompanion(spec companionSpec, path string, fromPrimary bool, comps []bundleComponent) (uintptr, error) { + if !fromPrimary { + var err error + comps, err = bundleComponents(path) + if err != nil { + return 0, err + } + } + var cctx uintptr + if comps == nil { + if spec.component != "" { + return 0, fmt.Errorf("parakeet-cpp: %s needs a bundle GGUF; %q is not one (or libparakeet.so has no bundle support)", + spec.compOpt, path) + } + cctx = CppLoad(path) + if cctx == 0 { + return 0, fmt.Errorf("parakeet-cpp: failed to load %s %q", spec.optName, path) + } + } else { + wantKind := kindForModel(spec.wantKind) + name, err := pickComponent(path, comps, spec.component, wantKind, spec.optName, spec.compOpt) + if err != nil { + return 0, err + } + cctx, err = loadComponent(path, name) + if err != nil { + return 0, err + } + } + if gotKind := CppModelKind(cctx); gotKind != spec.wantKind { + return cctx, fmt.Errorf("parakeet-cpp: %s %q is %s %s model, expected %s %s model", + spec.optName, path, indefiniteArticle(modelKindName(gotKind)), modelKindName(gotKind), + indefiniteArticle(modelKindName(spec.wantKind)), modelKindName(spec.wantKind)) + } + return cctx, nil +} diff --git a/backend/go/parakeet-cpp/sound.go b/backend/go/parakeet-cpp/sound.go index 1a3a358ad..3b4cb1249 100644 --- a/backend/go/parakeet-cpp/sound.go +++ b/backend/go/parakeet-cpp/sound.go @@ -51,7 +51,7 @@ type classAvg struct { func (p *ParakeetCpp) SoundDetection(ctx context.Context, req *pb.SoundDetectionRequest) (*pb.SoundDetectionResponse, error) { if p.tagCtx == 0 { return nil, status.Error(codes.FailedPrecondition, - "parakeet-cpp: model is not a sound (CED) model") + "parakeet-cpp: model is not a sound (CED) model"+p.roleHint(componentSound, "sound_component")) } if CppSoundStreamBegin == nil || CppSoundStreamFeed == nil || CppSoundStreamDrainScoresJSON == nil || CppSoundStreamFree == nil || CppSoundOptsDefault == nil || CppNumClasses == nil { diff --git a/core/gallery/parakeet_bundle_entries_test.go b/core/gallery/parakeet_bundle_entries_test.go new file mode 100644 index 000000000..f7558ae72 --- /dev/null +++ b/core/gallery/parakeet_bundle_entries_test.go @@ -0,0 +1,113 @@ +package gallery_test + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" +) + +// The parakeet-cpp bundle entries install one GGUF file that holds several +// models. The component options name components inside that file, so a typo +// installs cleanly and fails when the model loads. The sizes and checksums are +// those of the files published in mudler/parakeet-cpp-gguf. +var _ = Describe("gallery/index.yaml parakeet-cpp bundle entries", func() { + type bundle struct { + file string + sha256 string + usecases []string + options []string + } + full := []string{"vad:true", "diar_component:diar", "sound_component:ced", "speaker_component:voice"} + want := map[string]bundle{ + "parakeet-cpp-bundle-small": { + "parakeet-cpp/parakeet-bundle-small.gguf", + "5f2c6697eb08d4d67e858692d4223a9d911aacc51ae16cf2aee9e760f8a2269a", + []string{"transcript", "vad", "diarization", "sound_classification"}, full, + }, + "parakeet-cpp-bundle-standard": { + "parakeet-cpp/parakeet-bundle-standard.gguf", + "85304e3bb49de04b84a7d0c76baaa7dedd1a0c02c0920078a99f92c00cffbdf6", + []string{"transcript", "vad", "diarization", "sound_classification"}, full, + }, + "parakeet-cpp-bundle-moondream-redux": { + "parakeet-cpp/parakeet-bundle-moondream-redux.gguf", + "d7c5bff66b1dbc02f4c7a83148b0836b57805676c844193c214c751249853d7b", + []string{"transcript", "vad"}, []string{"vad:true"}, + }, + } + + entries := func() map[string]gallery.GalleryModel { + list, err := loadGalleryIndex() + Expect(err).ToNot(HaveOccurred()) + m := map[string]gallery.GalleryModel{} + for _, e := range list { + m[e.Name] = e + } + return m + } + + It("installs one bundle file with the published checksum and serves it as the model", func() { + all := entries() + for name, w := range want { + e, ok := all[name] + Expect(ok).To(BeTrue(), name) + Expect(e.Overrides["backend"]).To(Equal("parakeet-cpp"), name) + Expect(e.Overrides["name"]).To(Equal(name), name) + Expect(e.Overrides["parameters"]).To(HaveKeyWithValue("model", w.file), name) + Expect(e.AdditionalFiles).To(HaveLen(1), name) + Expect(e.AdditionalFiles[0].Filename).To(Equal(w.file), name) + Expect(e.AdditionalFiles[0].SHA256).To(Equal(w.sha256), name) + Expect(e.AdditionalFiles[0].URI).To(Equal("huggingface://mudler/parakeet-cpp-gguf/"+strings.TrimPrefix(w.file, "parakeet-cpp/")), name) + Expect(e.Variants).To(BeEmpty(), name) + } + }) + + It("declares the usecases and options of the roles each bundle covers", func() { + all := entries() + for name, w := range want { + e := all[name] + Expect(e.Overrides["known_usecases"]).To(ConsistOf(toAny(w.usecases)...), name) + Expect(e.Overrides["options"]).To(ConsistOf(toAny(w.options)...), name) + } + }) + + It("states that the licence is per component and names each one", func() { + all := entries() + for name := range want { + e := all[name] + Expect(e.License).To(Equal("other"), name) + Expect(e.Description).To(ContainSubstring("no single license"), name) + Expect(e.Description).To(ContainSubstring("CC-BY-4.0"), name) + Expect(e.Description).To(ContainSubstring("MIT"), name) + } + for _, name := range []string{"parakeet-cpp-bundle-small", "parakeet-cpp-bundle-standard"} { + d := all[name].Description + Expect(d).To(ContainSubstring("OpenMDW-1.1"), name) + Expect(d).To(ContainSubstring("Apache-2.0"), name) + Expect(d).To(ContainSubstring("not consistent upstream"), name) + Expect(d).To(ContainSubstring("converted, not trained"), name) + } + Expect(all["parakeet-cpp-bundle-moondream-redux"].Description).To(ContainSubstring("CPU only")) + }) + + It("keeps the single-purpose parakeet-cpp entries", func() { + all := entries() + for _, name := range []string{ + "parakeet-cpp-tdt_ctc-110m", "parakeet-cpp-tdt-0.6b-v3", "parakeet-cpp-moondream-redux-packed", + "parakeet-cpp-silero-vad-f16", "parakeet-cpp-nemotron-3-diarization", "parakeet-cpp-ced-tiny", + } { + Expect(all).To(HaveKey(name)) + } + }) +}) + +func toAny(in []string) []any { + out := make([]any, len(in)) + for i, s := range in { + out[i] = s + } + return out +} diff --git a/docs/content/features/audio-classification.md b/docs/content/features/audio-classification.md index 4f9c58fb9..572983651 100644 --- a/docs/content/features/audio-classification.md +++ b/docs/content/features/audio-classification.md @@ -80,6 +80,8 @@ curl http://localhost:8080/v1/audio/classification \ -F top_k=10 ``` +The bundle entries `parakeet-cpp-bundle-small` and `parakeet-cpp-bundle-standard` also serve this endpoint: they hold CED-Small next to the transcription, VAD and diarization models (`sound_component:ced`). See [Bundle GGUF files]({{% relref "audio-to-text" %}}#bundle-gguf-files-several-models-in-one-file). + ## See also - [Audio to Text]({{% relref "audio-to-text" %}}) - speech transcription diff --git a/docs/content/features/audio-diarization.md b/docs/content/features/audio-diarization.md index 72bc0cdf1..3d6b9c702 100644 --- a/docs/content/features/audio-diarization.md +++ b/docs/content/features/audio-diarization.md @@ -196,6 +196,8 @@ The complete `-asr-speakers` entry downloads Nemotron-3-Diarization, Parakeet TD It configures both `asr_model` and `speaker_model`; no custom gallery configuration is needed. See [Remember speakers in the Web UI](#remember-speakers-in-the-web-ui) for installation and enrollment. +The entries `parakeet-cpp-bundle-small` and `parakeet-cpp-bundle-standard` hold Nemotron-3-Diarization, an ASR model and the WeSpeaker speaker encoder in one file (`diar_component:diar` and `speaker_component:voice`), so one install serves the transcript and the identification options above. See [Bundle GGUF files]({{% relref "audio-to-text" %}}#bundle-gguf-files-several-models-in-one-file). + For manual configuration, this example pairs Sortformer with ASR: ```yaml diff --git a/docs/content/features/audio-to-text.md b/docs/content/features/audio-to-text.md index 08e5beed6..4ecdbc109 100644 --- a/docs/content/features/audio-to-text.md +++ b/docs/content/features/audio-to-text.md @@ -308,6 +308,57 @@ The segmenter options below apply to both `vad:true` and `vad_model`. Each is op `vad_speech_pad` (seconds) pads each region and only affects the [VAD endpoint]({{%relref "features/voice-activity-detection" %}}). `vad_model` needs a `libparakeet.so` that exports `parakeet_capi_transcribe_path_json_vad_with`; an older library fails the load with a message that names it. +### Bundle GGUF files (several models in one file) + +A bundle is one GGUF file that holds several models, called components. Each component keeps its own licence. The backend opens the components it needs from the one file, so a single model YAML can serve transcription, VAD, diarization, speaker naming and sound events. A bundle needs a `libparakeet.so` from parakeet.cpp with bundle support (pin `781a973` or newer); the format is described in the [parakeet.cpp bundle documentation](https://github.com/mudler/parakeet.cpp/blob/master/docs/bundle.md). Single-model files and every existing option work as before. + +The gallery has three bundles, built from [`mudler/parakeet-cpp-gguf`](https://huggingface.co/mudler/parakeet-cpp-gguf): + +| Gallery entry | Size | Components | Serves | +|---|---|---|---| +| `parakeet-cpp-bundle-small` | 338 MB | Parakeet TDT+CTC 110M (Q8_0), Nemotron-3-Diarization (Q8_0), CED-Small (Q8_0), WeSpeaker ResNet34-LM (F32), Silero VAD (F16) | transcription, VAD, diarization, speaker naming, sound events | +| `parakeet-cpp-bundle-standard` | 1.1 GB | Parakeet TDT 0.6B v3 (Q8_0), plus the same four components | the same, with the multilingual 0.6B model | +| `parakeet-cpp-bundle-moondream-redux` | 215 MB | Moondream Redux (packed ternary), Silero VAD (F16) | transcription and VAD; CPU only and offline only | + +The component that each role uses, and the option that picks another one: + +| Role | Component used | Option | +|---|---|---| +| Transcription | the only `asr` component | `bundle_asr:` picks one when the bundle has several | +| VAD (`/v1/vad`, and `vad:true` for long audio) | the `vad` (Silero) component, loaded with no option; without one the VAD head of the ASR model | `vad_component:` picks one, and implies `vad:true` | +| Diarization | the `diar` component, only when asked for | `diar_component:` | +| Sound events | the `ced` component, only when asked for | `sound_component:` | +| Speaker naming | the `voice` component, only when asked for | `speaker_component:` (needs a diarization component) | + +A `*_component` option without the matching companion option takes the component from the model file itself. The companion options (`diarization_model:`, `sound_model:`, `speaker_model:`, `vad_model:`, `asr_model:`) can also name a bundle file, even the same file as the model: the only component of the wanted kind is used, and the `*_component` option picks one when there are several. This YAML loads the same file for four roles: + +```yaml +name: parakeet-bundle +backend: parakeet-cpp +parameters: + model: parakeet-cpp/parakeet-bundle-small.gguf +options: +- vad:true # cut long audio at pauses, with the Silero component +- diar_component:diar # same as diarization_model:parakeet-cpp/parakeet-bundle-small.gguf +- sound_component:ced +- speaker_component:voice +``` + +A role that the bundle cannot fill fails with a message that lists the components, for example `parakeet-cpp: diarization_model needs a "diar" component, but the bundle "" has none (components: asr (asr), vad (vad))`. A request for a role the loaded model does not have (diarization from a bundle without a `diar` component) returns `parakeet-cpp: model is not a diarization model (the model file is a bundle without a "diar" component; ...)`. A `*_component` option on a file that is not a bundle, or on a library without bundle support, fails the load. + +Licences: a bundle has no single licence, so the gallery entries use `license: other`. The licence, source and credit of every component are in the file header, and the NOTICE file next to each bundle in the repository (`NOTICE-.txt`) has the credits and the full licence texts. Keep it with any copy of the file you pass on. + +| Component | Licence | Credit | +|---|---|---| +| Parakeet TDT+CTC 110M, Parakeet TDT 0.6B v3 | CC-BY-4.0 | NVIDIA | +| Moondream Redux | CC-BY-4.0 | Moondream, derived from Parakeet TDT 0.6B v3 by NVIDIA | +| Nemotron-3-Diarization | OpenMDW-1.1 | NVIDIA | +| CED-Small | Apache-2.0, as stated on the model card | Heinrich Dinkel et al., Xiaomi (mispeech) | +| WeSpeaker ResNet34-LM | CC-BY-4.0 | the WeSpeaker project | +| Silero VAD | MIT | Silero Team | + +The licence of the CED weights is not consistent upstream: the model card says Apache-2.0, the upstream code repository is GPL-3.0 and the original checkpoint records say CC-BY-4.0. The file here is converted, not trained, and follows the model card. The weights of all components were converted to GGUF, and quantised where the table shows it; nothing was retrained. + ## See also - [Audio Transform]({{< relref "audio-transform.md" >}}) - clean up the audio (echo cancellation, noise suppression, dereverberation) before passing it to a transcription model. diff --git a/docs/content/features/voice-activity-detection.md b/docs/content/features/voice-activity-detection.md index 75e7c358d..b4bef8e53 100644 --- a/docs/content/features/voice-activity-detection.md +++ b/docs/content/features/voice-activity-detection.md @@ -125,6 +125,8 @@ The `parakeet-cpp` backend serves the same endpoint. It runs one of two detector - **The VAD head** of a full Moondream Ultra or Redux model (gallery entries `parakeet-cpp-vad-moondream-ultra-q8_0` and `parakeet-cpp-vad-moondream-redux-packed`). One probability per 80 ms. The packed Redux file runs on CPU only. - **A VAD-only slice** of that head (gallery entries `parakeet-cpp-vad-moondream-redux`, 9.9 MB, and `parakeet-cpp-vad-moondream-ultra`, 6.0 MB). The slice is cut out of the full model without retraining, so the segments are byte-identical to the full model's head, and the speed is the same. Compared with loading the whole model (213 MB to 1.4 GB), the file is 6 to 10 MB, loads in a few milliseconds instead of 0.1 to 0.7 s, and needs about 245 MiB of peak memory for a 33 s clip instead of 0.6 to 1.6 GiB. A slice cannot transcribe, and it needs a parakeet.cpp build with VAD-only GGUF support (pin e53a253 or newer). +- **A bundle GGUF** (gallery entries `parakeet-cpp-bundle-small`, `parakeet-cpp-bundle-standard` and `parakeet-cpp-bundle-moondream-redux`). The bundle holds a Silero component next to the ASR model, and the backend uses it for this endpoint with no option. See [Bundle GGUF files]({{%relref "features/audio-to-text" %}}#bundle-gguf-files-several-models-in-one-file). + The entry `parakeet-cpp-vad` installs Silero. The detectors differ and are not variants of one model, so install the entry of the VAD head by name if you want it (`parakeet-cpp-vad-moondream-redux` or `parakeet-cpp-vad-moondream-ultra` for the small files). The request is the same as above: `audio` is 16 kHz mono float32 PCM, and the response lists `segments` with `start` and `end` in seconds. An ASR model that has no VAD head fails the request with `model has no VAD head`. ```yaml diff --git a/gallery/index.yaml b/gallery/index.yaml index 29d68410d..14f06c66b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -54685,6 +54685,178 @@ - filename: parakeet-cpp/ced-base-q8_0.gguf uri: huggingface://mudler/ced-gguf/ced-base-q8_0.gguf sha256: bd34a7710169f0047fea17267965d211f967828ab25ba6fb9d3768481393f6e2 +- name: parakeet-cpp-bundle-small + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + - https://huggingface.co/nvidia/parakeet-tdt_ctc-110m + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-small + - https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet34-LM + - https://github.com/snakers4/silero-vad + description: | + Parakeet TDT+CTC 110M with diarization, sound events, speaker naming and VAD. One bundle GGUF file (about 338 MB) that holds five models for the parakeet-cpp backend + (C++/ggml port of NVIDIA NeMo Parakeet): Parakeet TDT+CTC 110M (English, Q8_0), Nemotron-3-Diarization (Q8_0), CED-Small sound + events (Q8_0), WeSpeaker ResNet34-LM speaker encoder (F32) and Silero VAD (F16). One install serves + transcription (/v1/audio/transcriptions), voice activity detection (/v1/vad), speaker diarization with + text and named speakers (/v1/audio/diarization) and sound events (/v1/audio/classification). The vad:true + option cuts long audio at pauses found by Silero before it transcribes. The bundle options + (diar_component, sound_component, speaker_component) load each model from the same file; see the + audio-to-text docs for the option list. + A bundle has no single license: each model keeps its own, listed in the file header and in the NOTICE file + next to the file. Parakeet TDT+CTC 110M by NVIDIA is CC-BY-4.0, Nemotron-3-Diarization by NVIDIA is OpenMDW-1.1, CED-Small is Apache-2.0 + (as stated on the model card; the upstream code repository is GPL-3.0 and the original checkpoint records say + CC-BY-4.0, so the licence of the weights is not consistent upstream; the model here is converted, not trained), + WeSpeaker ResNet34-LM by the WeSpeaker project is CC-BY-4.0, Silero VAD by the Silero Team is MIT. The + weights were converted to GGUF and quantised where stated; nothing was retrained. + license: other + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - silero + - bundle + - asr + - speech-recognition + - stt + - vad + - diarization + - speaker-diarization + - sound-classification + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - vad + - diarization + - sound_classification + name: parakeet-cpp-bundle-small + options: + - vad:true + - diar_component:diar + - sound_component:ced + - speaker_component:voice + parameters: + model: parakeet-cpp/parakeet-bundle-small.gguf + files: + - filename: parakeet-cpp/parakeet-bundle-small.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/parakeet-bundle-small.gguf + sha256: 5f2c6697eb08d4d67e858692d4223a9d911aacc51ae16cf2aee9e760f8a2269a +- name: parakeet-cpp-bundle-standard + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + - https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3 + - https://huggingface.co/nvidia/Nemotron-3-Diarization + - https://huggingface.co/mispeech/ced-small + - https://huggingface.co/Wespeaker/wespeaker-voxceleb-resnet34-LM + - https://github.com/snakers4/silero-vad + description: | + Parakeet TDT 0.6B v3 (multilingual) with diarization, sound events, speaker naming and VAD. One bundle GGUF file (about 1101 MB) that holds five models for the parakeet-cpp backend + (C++/ggml port of NVIDIA NeMo Parakeet): Parakeet TDT 0.6B v3 (multilingual, Q8_0), Nemotron-3-Diarization (Q8_0), CED-Small sound + events (Q8_0), WeSpeaker ResNet34-LM speaker encoder (F32) and Silero VAD (F16). One install serves + transcription (/v1/audio/transcriptions), voice activity detection (/v1/vad), speaker diarization with + text and named speakers (/v1/audio/diarization) and sound events (/v1/audio/classification). The vad:true + option cuts long audio at pauses found by Silero before it transcribes. The bundle options + (diar_component, sound_component, speaker_component) load each model from the same file; see the + audio-to-text docs for the option list. + A bundle has no single license: each model keeps its own, listed in the file header and in the NOTICE file + next to the file. Parakeet TDT 0.6B v3 by NVIDIA is CC-BY-4.0, Nemotron-3-Diarization by NVIDIA is OpenMDW-1.1, CED-Small is Apache-2.0 + (as stated on the model card; the upstream code repository is GPL-3.0 and the original checkpoint records say + CC-BY-4.0, so the licence of the weights is not consistent upstream; the model here is converted, not trained), + WeSpeaker ResNet34-LM by the WeSpeaker project is CC-BY-4.0, Silero VAD by the Silero Team is MIT. The + weights were converted to GGUF and quantised where stated; nothing was retrained. + license: other + tags: + - parakeet + - parakeet-cpp + - nemotron + - sortformer + - ced + - silero + - bundle + - asr + - speech-recognition + - stt + - vad + - diarization + - speaker-diarization + - sound-classification + - gguf + - ggml + - quantized + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - vad + - diarization + - sound_classification + name: parakeet-cpp-bundle-standard + options: + - vad:true + - diar_component:diar + - sound_component:ced + - speaker_component:voice + parameters: + model: parakeet-cpp/parakeet-bundle-standard.gguf + files: + - filename: parakeet-cpp/parakeet-bundle-standard.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/parakeet-bundle-standard.gguf + sha256: 85304e3bb49de04b84a7d0c76baaa7dedd1a0c02c0920078a99f92c00cffbdf6 +- name: parakeet-cpp-bundle-moondream-redux + url: github:mudler/LocalAI/gallery/virtual.yaml@master + urls: + - https://huggingface.co/mudler/parakeet-cpp-gguf + - https://github.com/mudler/parakeet.cpp + - https://huggingface.co/moondream/parakeet-redux + - https://github.com/snakers4/silero-vad + description: | + Moondream Redux with Silero VAD in one bundle GGUF file (about 215 MB) for the parakeet-cpp backend + (C++/ggml port of NVIDIA NeMo Parakeet). Redux is Moondream's ternary-encoder derivative of NVIDIA + parakeet-tdt-0.6b-v3 (TDT, 0.6B), kept as published with the packed ternary weights. The bundle also holds + Silero VAD (F16). One install serves transcription (/v1/audio/transcriptions) and voice activity detection + (/v1/vad). The vad:true option cuts long audio at pauses before it transcribes, with Silero as the VAD source. + CPU only and offline only: the library refuses the packed Redux model on a GPU backend and does not stream it. + For a GPU or for streaming, use the redux-f16 or redux-q8_0 entry instead. + A bundle has no single license: each model keeps its own, listed in the file header and in the NOTICE file + next to the file. Parakeet Redux by Moondream, derived from Parakeet TDT 0.6B v3 by NVIDIA, is CC-BY-4.0 + (credit Moondream and NVIDIA); Silero VAD by the Silero Team is MIT. The weights were converted to GGUF; + nothing was retrained. + license: other + tags: + - parakeet + - parakeet-cpp + - moondream + - silero + - bundle + - asr + - speech-recognition + - stt + - vad + - gguf + - ggml + overrides: + backend: parakeet-cpp + known_usecases: + - transcript + - vad + name: parakeet-cpp-bundle-moondream-redux + options: + - vad:true + parameters: + model: parakeet-cpp/parakeet-bundle-moondream-redux.gguf + files: + - filename: parakeet-cpp/parakeet-bundle-moondream-redux.gguf + uri: huggingface://mudler/parakeet-cpp-gguf/parakeet-bundle-moondream-redux.gguf + sha256: d7c5bff66b1dbc02f4c7a83148b0836b57805676c844193c214c751249853d7b - name: moss-transcribe-cpp-0.9b url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: