mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-28 09:08:54 -04:00
feat(omnivoice): add bounded duration controls (#12637)
* test(omnivoice): expose ignored duration controls Compile the real wrapper with a model-free engine fixture. Four failing assertions expose ignored duration, postprocessing, and validation. Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * feat(omnivoice): add bounded duration controls Validate request controls before reference reads or native output. Preserve engine defaults when controls are omitted. Bound forced single-shot duration to 30 seconds as wrapper resource policy. Exercise the real native boundary, streaming rejection, and defaults with synthetic PCM. Document quantization and postprocessing limits. Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * test(omnivoice): require fixture-free ordinary tests Expose the ordinary suite regression before isolating native fixtures. Require selected native tests to reject a missing fixture explicitly. Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto <mudler@localai.io> * fix(omnivoice): isolate native fixture tests Ordinary Go tests must run without a compiled native fixture. Select fixture specs explicitly in the dedicated gate while keeping parser tests ordinary and native checks mandatory. Assisted-by: nib:gpt-6-astra Signed-off-by: Ettore Di Giacinto <mudler@localai.io> --------- Signed-off-by: Ettore Di Giacinto <mudler@localai.io> Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
1 parent
c66b1a81ef
commit
440225a724
13 files changed
+892
-14
No files matched your search
@@ -62,8 +62,12 @@ int omni_load(const char *model_path, const char *codec_path, int use_fa,
|
||||
static void fill_params(ov_tts_params *tp, const char *text, const char *lang,
|
||||
const char *instruct, const float *ref_samples,
|
||||
int ref_n, const char *ref_text, long long seed,
|
||||
int denoise) {
|
||||
int denoise, int duration_frames, int postprocess) {
|
||||
ov_tts_default_params(tp);
|
||||
if (duration_frames > 0)
|
||||
tp->T_override = duration_frames;
|
||||
if (postprocess >= 0)
|
||||
tp->postproc = postprocess != 0;
|
||||
tp->text = text ? text : "";
|
||||
tp->lang = lang ? lang : "";
|
||||
if (instruct && instruct[0] != '\0')
|
||||
@@ -81,9 +85,11 @@ static void fill_params(ov_tts_params *tp, const char *text, const char *lang,
|
||||
|
||||
float *omni_tts(const char *text, const char *lang, const char *instruct,
|
||||
const float *ref_samples, int ref_n, const char *ref_text,
|
||||
long long seed, int denoise, int *out_n) {
|
||||
long long seed, int denoise, int duration_frames, int postprocess, int *out_n) {
|
||||
if (out_n)
|
||||
*out_n = 0;
|
||||
if (duration_frames < 0 || duration_frames > 750 || postprocess < -1 || postprocess > 1)
|
||||
return nullptr;
|
||||
if (!g_ctx) {
|
||||
fprintf(stderr, "[omnivoice-cpp] ERROR: model not loaded\n");
|
||||
return nullptr;
|
||||
@@ -94,7 +100,7 @@ float *omni_tts(const char *text, const char *lang, const char *instruct,
|
||||
}
|
||||
ov_tts_params tp;
|
||||
fill_params(&tp, text, lang, instruct, ref_samples, ref_n, ref_text, seed,
|
||||
denoise);
|
||||
denoise, duration_frames, postprocess);
|
||||
|
||||
ov_audio out = {0};
|
||||
enum ov_status rc = ov_synthesize(g_ctx, &tp, &out);
|
||||
@@ -123,8 +129,10 @@ float *omni_tts(const char *text, const char *lang, const char *instruct,
|
||||
|
||||
int omni_tts_stream(const char *text, const char *lang, const char *instruct,
|
||||
const float *ref_samples, int ref_n, const char *ref_text,
|
||||
long long seed, int denoise, omni_pcm_chunk_cb cb,
|
||||
long long seed, int denoise, int duration_frames, int postprocess, omni_pcm_chunk_cb cb,
|
||||
void *user_data) {
|
||||
if (duration_frames < 0 || duration_frames > 750 || postprocess < -1 || postprocess > 1 || postprocess == 0)
|
||||
return 5;
|
||||
if (!g_ctx) {
|
||||
fprintf(stderr, "[omnivoice-cpp] ERROR: model not loaded\n");
|
||||
return 1;
|
||||
@@ -139,7 +147,7 @@ int omni_tts_stream(const char *text, const char *lang, const char *instruct,
|
||||
}
|
||||
ov_tts_params tp;
|
||||
fill_params(&tp, text, lang, instruct, ref_samples, ref_n, ref_text, seed,
|
||||
denoise);
|
||||
denoise, duration_frames, postprocess);
|
||||
// ov_audio_chunk_cb has the identical signature to omni_pcm_chunk_cb
|
||||
// (bool vs int return are ABI-compatible; non-zero == true).
|
||||
tp.on_chunk = (ov_audio_chunk_cb)cb;
|
||||
|
||||
@@ -18,16 +18,19 @@ int omni_load(const char *model_path, const char *codec_path, int use_fa,
|
||||
// ref_samples != null && ref_n > 0 => voice cloning (ref_text optional).
|
||||
// instruct != null && non-empty => voice design. seed < 0 keeps the default
|
||||
// MaskGIT seed. denoise toggles the <|denoise|> marker (only with a reference).
|
||||
// duration_frames: 0 keeps defaults; 1..750 overrides generation frames.
|
||||
// postprocess: -1 keeps defaults; 0 disables, 1 enables post filtering.
|
||||
// The frame cap is wrapper resource policy, not a success guarantee.
|
||||
// Writes the sample count to *out_n. Returns NULL on failure (out_n set to 0).
|
||||
float *omni_tts(const char *text, const char *lang, const char *instruct,
|
||||
const float *ref_samples, int ref_n, const char *ref_text,
|
||||
long long seed, int denoise, int *out_n);
|
||||
long long seed, int denoise, int duration_frames, int postprocess, int *out_n);
|
||||
|
||||
// Streaming synthesis: cb is invoked per PCM chunk as audio is produced.
|
||||
// Same reference/design/seed semantics as omni_tts. Returns 0 on success.
|
||||
// Same semantics as omni_tts, but postprocess=0 is rejected. Returns 0 on success.
|
||||
int omni_tts_stream(const char *text, const char *lang, const char *instruct,
|
||||
const float *ref_samples, int ref_n, const char *ref_text,
|
||||
long long seed, int denoise, omni_pcm_chunk_cb cb,
|
||||
long long seed, int denoise, int duration_frames, int postprocess, omni_pcm_chunk_cb cb,
|
||||
void *user_data);
|
||||
|
||||
// Free a buffer returned by omni_tts.
|
||||
|
||||
@@ -16,12 +16,12 @@ import (
|
||||
var (
|
||||
// omni_load(model_path, codec_path, use_fa, clamp_fp16) int
|
||||
CppLoad func(modelPath, codecPath string, useFA, clampFP16 int) int
|
||||
// omni_tts(text, lang, instruct, ref_samples, ref_n, ref_text, seed, denoise, out_n) -> float* (uintptr)
|
||||
// omni_tts(text, lang, instruct, ref_samples, ref_n, ref_text, seed, denoise, duration_frames, postprocess, out_n) -> float* (uintptr)
|
||||
CppTTS func(text, lang, instruct string, refSamples unsafe.Pointer, refN int,
|
||||
refText string, seed int64, denoise int, outN unsafe.Pointer) uintptr
|
||||
// omni_tts_stream(text, lang, instruct, ref_samples, ref_n, ref_text, seed, denoise, cb, user) int
|
||||
refText string, seed int64, denoise int, durationFrames, postprocess int32, outN unsafe.Pointer) uintptr
|
||||
// omni_tts_stream(text, lang, instruct, ref_samples, ref_n, ref_text, seed, denoise, duration_frames, postprocess, cb, user) int
|
||||
CppTTSStream func(text, lang, instruct string, refSamples unsafe.Pointer, refN int,
|
||||
refText string, seed int64, denoise int, cb uintptr, user uintptr) int
|
||||
refText string, seed int64, denoise int, durationFrames, postprocess int32, cb uintptr, user uintptr) int
|
||||
CppPCMFree func(ptr uintptr)
|
||||
CppUnload func()
|
||||
)
|
||||
@@ -150,6 +150,10 @@ func optStr(p *string) string {
|
||||
}
|
||||
|
||||
func (o *OmnivoiceCpp) TTS(req *pb.TTSRequest) error {
|
||||
controls, err := parseRequestControls(req.Params, false)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if req.Dst == "" {
|
||||
return fmt.Errorf("omnivoice: TTS requires a destination path")
|
||||
}
|
||||
@@ -169,7 +173,7 @@ func (o *OmnivoiceCpp) TTS(req *pb.TTSRequest) error {
|
||||
|
||||
var n int32
|
||||
ptr := CppTTS(req.Text, lang, instruct, refPtr, len(ref), refText, seed,
|
||||
boolToInt(o.opts.denoise), unsafe.Pointer(&n))
|
||||
boolToInt(o.opts.denoise), controls.durationFrames, controls.postprocess, unsafe.Pointer(&n))
|
||||
runtimeKeepAlive(ref)
|
||||
if ptr == 0 || n <= 0 {
|
||||
return fmt.Errorf("omnivoice: synthesis failed")
|
||||
@@ -206,6 +210,10 @@ func streamCallback(samples *float32, nSamples int32, _ uintptr) uintptr {
|
||||
|
||||
func (o *OmnivoiceCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error {
|
||||
defer close(results)
|
||||
controls, err := parseRequestControls(req.Params, true)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if req.Text == "" {
|
||||
return fmt.Errorf("omnivoice: TTSStream requires text")
|
||||
}
|
||||
@@ -234,7 +242,7 @@ func (o *OmnivoiceCpp) TTSStream(req *pb.TTSRequest, results chan []byte) error
|
||||
streamMu.Lock()
|
||||
streamChan = results
|
||||
rc := CppTTSStream(req.Text, lang, instruct, refPtr, len(ref), refText, seed,
|
||||
boolToInt(o.opts.denoise), streamCbPtr, 0)
|
||||
boolToInt(o.opts.denoise), controls.durationFrames, controls.postprocess, streamCbPtr, 0)
|
||||
streamChan = nil
|
||||
streamMu.Unlock()
|
||||
runtimeKeepAlive(ref)
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
//go:build native_controls
|
||||
|
||||
// SPDX-License-Identifier: MIT
|
||||
package main
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
"unsafe"
|
||||
|
||||
"github.com/ebitengine/purego"
|
||||
pb "github.com/mudler/LocalAI/pkg/grpc/proto"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("native request controls", Label("native-controls"), Serial, func() {
|
||||
var backend *OmnivoiceCpp
|
||||
var req *pb.TTSRequest
|
||||
var reset func()
|
||||
var calls, frees, defaults, frames, postproc, otherDefaults func() int
|
||||
var seed func() uint64
|
||||
var denoise, refN func() int
|
||||
var refFirst func() float32
|
||||
var text, lang, instruct, refText func() string
|
||||
BeforeEach(func() {
|
||||
path := os.Getenv("OMNIVOICE_TEST_LIBRARY")
|
||||
Expect(path).NotTo(BeEmpty(), "run verify-controls.sh to compile the actual wrapper fixture")
|
||||
lib, err := purego.Dlopen(path, purego.RTLD_NOW|purego.RTLD_LOCAL)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
oldLoad, oldTTS, oldStream, oldFree, oldUnload := CppLoad, CppTTS, CppTTSStream, CppPCMFree, CppUnload
|
||||
DeferCleanup(func() {
|
||||
CppUnload()
|
||||
reset()
|
||||
CppLoad, CppTTS, CppTTSStream, CppPCMFree, CppUnload = oldLoad, oldTTS, oldStream, oldFree, oldUnload
|
||||
Expect(purego.Dlclose(lib)).To(Succeed())
|
||||
})
|
||||
purego.RegisterLibFunc(&CppLoad, lib, "omni_load")
|
||||
purego.RegisterLibFunc(&CppTTS, lib, "omni_tts")
|
||||
purego.RegisterLibFunc(&CppTTSStream, lib, "omni_tts_stream")
|
||||
purego.RegisterLibFunc(&CppPCMFree, lib, "omni_pcm_free")
|
||||
purego.RegisterLibFunc(&CppUnload, lib, "omni_unload")
|
||||
purego.RegisterLibFunc(&reset, lib, "fixture_reset")
|
||||
purego.RegisterLibFunc(&calls, lib, "fixture_calls")
|
||||
purego.RegisterLibFunc(&frees, lib, "fixture_frees")
|
||||
purego.RegisterLibFunc(&defaults, lib, "fixture_defaults")
|
||||
purego.RegisterLibFunc(&frames, lib, "fixture_frames")
|
||||
purego.RegisterLibFunc(&postproc, lib, "fixture_postproc")
|
||||
purego.RegisterLibFunc(&otherDefaults, lib, "fixture_other_defaults")
|
||||
purego.RegisterLibFunc(&seed, lib, "fixture_seed")
|
||||
purego.RegisterLibFunc(&denoise, lib, "fixture_denoise")
|
||||
purego.RegisterLibFunc(&refN, lib, "fixture_ref_n")
|
||||
purego.RegisterLibFunc(&refFirst, lib, "fixture_ref_first")
|
||||
purego.RegisterLibFunc(&text, lib, "fixture_text")
|
||||
purego.RegisterLibFunc(&lang, lib, "fixture_lang")
|
||||
purego.RegisterLibFunc(&instruct, lib, "fixture_instruct")
|
||||
purego.RegisterLibFunc(&refText, lib, "fixture_ref_text")
|
||||
reset()
|
||||
Expect(CppLoad("fixture-model", "fixture-codec", 1, 0)).To(BeZero())
|
||||
backend = &OmnivoiceCpp{opts: parseOptions(nil)}
|
||||
dir, err := os.MkdirTemp("", "omnivoice-controls-")
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
DeferCleanup(func() { Expect(os.RemoveAll(dir)).To(Succeed()) })
|
||||
req = &pb.TTSRequest{Text: "Fixture speech", Dst: filepath.Join(dir, "out.wav")}
|
||||
})
|
||||
It("preserves native defaults and frees engine and wrapper PCM", func() {
|
||||
nativeFree := CppPCMFree
|
||||
wrapperFrees := 0
|
||||
CppPCMFree = func(p uintptr) { wrapperFrees++; nativeFree(p) }
|
||||
for _, params := range []map[string]string{nil, {}, {"unrelated": "ignored"}} {
|
||||
req.Params = params
|
||||
Expect(backend.TTS(req)).To(Succeed())
|
||||
Expect(frames()).To(BeZero())
|
||||
Expect(postproc()).To(Equal(1))
|
||||
Expect(otherDefaults()).To(Equal(1))
|
||||
audio, err := readWAVAsFloat(req.Dst)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(audio).To(HaveLen(2))
|
||||
Expect(audio[0]).To(BeNumerically("~", .25, .001))
|
||||
}
|
||||
Expect(calls()).To(Equal(3))
|
||||
Expect(defaults()).To(Equal(3))
|
||||
Expect(frees()).To(Equal(3))
|
||||
Expect(wrapperFrees).To(Equal(3))
|
||||
})
|
||||
It("passes duration to the actual engine struct", func() {
|
||||
req.Params = map[string]string{"duration": "2.5"}
|
||||
Expect(backend.TTS(req)).To(Succeed())
|
||||
Expect(frames()).To(Equal(62))
|
||||
})
|
||||
It("passes buffered false without duration", func() {
|
||||
req.Params = map[string]string{"postprocess": "false"}
|
||||
Expect(backend.TTS(req)).To(Succeed())
|
||||
Expect(postproc()).To(BeZero())
|
||||
})
|
||||
It("rejects invalid duration before synthesis or output", func() {
|
||||
req.Params = map[string]string{"duration": "NaN"}
|
||||
Expect(backend.TTS(req)).NotTo(Succeed())
|
||||
Expect(calls()).To(BeZero())
|
||||
_, err := os.Stat(req.Dst)
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
It("rejects streaming false without header or native calls", func() {
|
||||
req.Params = map[string]string{"postprocess": "false"}
|
||||
results := make(chan []byte, 4)
|
||||
Expect(backend.TTSStream(req, results)).NotTo(Succeed())
|
||||
Expect(calls()).To(BeZero())
|
||||
_, open := <-results
|
||||
Expect(open).To(BeFalse())
|
||||
})
|
||||
It("preserves argument positions and resets controls between calls", func() {
|
||||
language, instruction := "English", "soft"
|
||||
req.Language, req.Instructions = &language, &instruction
|
||||
req.Voice = filepath.Join(filepath.Dir(req.Dst), "ref.wav")
|
||||
Expect(writeWAV24k(req.Voice, []float32{.5, -.5, .25})).To(Succeed())
|
||||
backend.opts.denoise = false
|
||||
for _, flag := range []string{"false", "true"} {
|
||||
req.Params = map[string]string{"duration": "30", "postprocess": flag, "seed": "12345678901", "ref_text": "Reference transcript"}
|
||||
Expect(backend.TTS(req)).To(Succeed())
|
||||
Expect(frames()).To(Equal(750))
|
||||
Expect(postproc()).To(Equal(boolToInt(flag == "true")))
|
||||
Expect(seed()).To(Equal(uint64(12345678901)))
|
||||
Expect(denoise()).To(BeZero())
|
||||
Expect(refN()).To(Equal(3))
|
||||
Expect(refFirst()).To(BeNumerically("~", .5, .001))
|
||||
Expect(text()).To(Equal(req.Text))
|
||||
Expect(lang()).To(Equal("en"))
|
||||
Expect(instruct()).To(Equal(instruction))
|
||||
Expect(refText()).To(Equal("Reference transcript"))
|
||||
Expect(otherDefaults()).To(Equal(1))
|
||||
}
|
||||
req.Params = nil
|
||||
req.Voice = ""
|
||||
Expect(backend.TTS(req)).To(Succeed())
|
||||
Expect(frames()).To(BeZero())
|
||||
Expect(postproc()).To(Equal(1))
|
||||
Expect(seed()).To(Equal(uint64(42)))
|
||||
Expect(denoise()).To(Equal(1))
|
||||
Expect(refN()).To(BeZero())
|
||||
Expect(defaults()).To(Equal(3))
|
||||
})
|
||||
It("streams duration and true while preserving defaults and PCM callbacks", func() {
|
||||
for _, params := range []map[string]string{{"duration": "2.5", "postprocess": "true"}, {"duration": "1"}, nil} {
|
||||
req.Params = params
|
||||
results := make(chan []byte, 4)
|
||||
Expect(backend.TTSStream(req, results)).To(Succeed())
|
||||
var chunks [][]byte
|
||||
for chunk := range results {
|
||||
chunks = append(chunks, chunk)
|
||||
}
|
||||
Expect(chunks).To(HaveLen(2))
|
||||
Expect(chunks[0]).To(Equal(wavHeader24k()))
|
||||
Expect(chunks[1]).To(Equal(floatToPCM16LE([]float32{.25, -.25})))
|
||||
want := 0
|
||||
if params["duration"] == "2.5" {
|
||||
want = 62
|
||||
} else if params["duration"] == "1" {
|
||||
want = 25
|
||||
}
|
||||
Expect(frames()).To(Equal(want))
|
||||
Expect(postproc()).To(Equal(1))
|
||||
Expect(otherDefaults()).To(Equal(1))
|
||||
}
|
||||
Expect(calls()).To(Equal(3))
|
||||
Expect(frees()).To(Equal(3))
|
||||
})
|
||||
It("rejects invalid controls before reference reads, callbacks, or output", func() {
|
||||
req.Voice = filepath.Join(filepath.Dir(req.Dst), "invalid.wav")
|
||||
Expect(os.WriteFile(req.Voice, []byte("not a WAV"), 0600)).To(Succeed())
|
||||
savedPtr := streamCbPtr
|
||||
savedChan := streamChan
|
||||
// Preserve the once state without copying its mutex.
|
||||
initialized := false
|
||||
streamCbOnce.Do(func() { initialized = true })
|
||||
streamCbOnce = sync.Once{}
|
||||
streamCbPtr = 0
|
||||
DeferCleanup(func() {
|
||||
streamCbPtr, streamChan = savedPtr, savedChan
|
||||
streamCbOnce = sync.Once{}
|
||||
if !initialized {
|
||||
streamCbOnce.Do(func() {})
|
||||
}
|
||||
})
|
||||
cases := []map[string]string{}
|
||||
for _, value := range invalidDurations {
|
||||
cases = append(cases, map[string]string{"duration": value})
|
||||
}
|
||||
for _, value := range invalidPostprocess {
|
||||
cases = append(cases, map[string]string{"postprocess": value})
|
||||
}
|
||||
for _, params := range cases {
|
||||
req.Params = params
|
||||
key := "duration"
|
||||
if _, ok := params["postprocess"]; ok {
|
||||
key = "postprocess"
|
||||
}
|
||||
Expect(backend.TTS(req)).To(MatchError(ContainSubstring(key)))
|
||||
results := make(chan []byte, 4)
|
||||
Expect(backend.TTSStream(req, results)).To(MatchError(ContainSubstring(key)))
|
||||
_, open := <-results
|
||||
Expect(open).To(BeFalse())
|
||||
Expect(streamCbPtr).To(BeZero())
|
||||
Expect(streamChan).To(BeNil())
|
||||
}
|
||||
req.Params = map[string]string{"postprocess": "false"}
|
||||
results := make(chan []byte, 4)
|
||||
Expect(backend.TTSStream(req, results)).To(MatchError(ContainSubstring("postprocess=false")))
|
||||
_, open := <-results
|
||||
Expect(open).To(BeFalse())
|
||||
Expect(streamCbPtr).To(BeZero())
|
||||
Expect(calls()).To(BeZero())
|
||||
Expect(defaults()).To(BeZero())
|
||||
_, err := os.Stat(req.Dst)
|
||||
Expect(os.IsNotExist(err)).To(BeTrue())
|
||||
})
|
||||
It("defensively rejects direct native invalid controls", func() {
|
||||
streamCbOnce.Do(func() { streamCbPtr = purego.NewCallback(streamCallback) })
|
||||
for _, pair := range [][2]int32{{-1, -1}, {751, -1}, {2147483647, -1}, {0, -2}, {0, 2}} {
|
||||
n := int32(99)
|
||||
ptr := CppTTS("text", "", "", nil, 0, "", -1, 1, pair[0], pair[1], unsafe.Pointer(&n))
|
||||
Expect(ptr).To(BeZero())
|
||||
Expect(n).To(BeZero())
|
||||
Expect(CppTTSStream("text", "", "", nil, 0, "", -1, 1, pair[0], pair[1], streamCbPtr, 0)).NotTo(BeZero())
|
||||
}
|
||||
Expect(CppTTSStream("text", "", "", nil, 0, "", -1, 1, 25, 0, streamCbPtr, 0)).NotTo(BeZero())
|
||||
Expect(calls()).To(BeZero())
|
||||
Expect(defaults()).To(BeZero())
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,44 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type requestControls struct {
|
||||
durationFrames int32
|
||||
postprocess int32
|
||||
}
|
||||
|
||||
func parseRequestControls(params map[string]string, streaming bool) (requestControls, error) {
|
||||
controls := requestControls{postprocess: -1}
|
||||
if value, present := params["duration"]; present {
|
||||
seconds, err := strconv.ParseFloat(value, 64)
|
||||
// Forced duration bypasses chunking. This cap is wrapper resource policy,
|
||||
// not an engine positional limit or an allocation/success guarantee.
|
||||
if err != nil || math.IsNaN(seconds) || math.IsInf(seconds, 0) || seconds <= 0 || seconds > 30 {
|
||||
return controls, fmt.Errorf("omnivoice: duration must be finite seconds greater than 0 and at most 30")
|
||||
}
|
||||
frames := math.Max(1, math.Floor(seconds*25)) // bundled codec: 24000 / 960
|
||||
if frames < 1 || frames > 750 {
|
||||
return controls, fmt.Errorf("omnivoice: duration frame count out of range")
|
||||
}
|
||||
controls.durationFrames = int32(frames)
|
||||
}
|
||||
if value, present := params["postprocess"]; present {
|
||||
switch value {
|
||||
case "true":
|
||||
controls.postprocess = 1
|
||||
case "false":
|
||||
controls.postprocess = 0
|
||||
default:
|
||||
return controls, fmt.Errorf("omnivoice: postprocess must be exactly true or false")
|
||||
}
|
||||
}
|
||||
if streaming && controls.postprocess == 0 {
|
||||
return controls, fmt.Errorf("omnivoice: streaming does not support postprocess=false")
|
||||
}
|
||||
return controls, nil
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package main
|
||||
|
||||
import (
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
"math"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func durationString(v float64) string { return strconv.FormatFloat(v, 'g', -1, 64) }
|
||||
|
||||
var invalidDurations = []string{"", " ", " 1", "1 ", "bad", "0", "-0", "-1", "NaN", "Inf", "+Inf", "-Inf", "1e309", "1e300", "2147483647", "85899346", "1e-999", durationString(math.Nextafter(30, math.Inf(1)))}
|
||||
var invalidPostprocess = []string{"", "TRUE", "False", "True", "FALSE", " true", "false ", "1", "0", "yes", "no", "garbage"}
|
||||
|
||||
var _ = Describe("request controls", func() {
|
||||
It("preserves omission and ignores unrelated parameters", func() {
|
||||
for _, p := range []map[string]string{nil, {}, {"unrelated": "value"}} {
|
||||
for _, stream := range []bool{false, true} {
|
||||
c, err := parseRequestControls(p, stream)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(c).To(Equal(requestControls{postprocess: -1}))
|
||||
}
|
||||
}
|
||||
})
|
||||
It("quantizes finite positive durations with float64 arithmetic", func() {
|
||||
cases := map[string]int32{"1": 25, "2.5": 62, "0.04": 1, "30": 750, "5e-324": 1, "0x1p+0": 25, "+1": 25}
|
||||
for _, v := range []float64{math.Nextafter(.08, 0), .08, math.Nextafter(.08, 1), math.Nextafter(1, 0), math.Nextafter(1, 2), math.Nextafter(30, 0)} {
|
||||
cases[durationString(v)] = int32(math.Max(1, math.Floor(v*25)))
|
||||
}
|
||||
for value, frames := range cases {
|
||||
for _, stream := range []bool{false, true} {
|
||||
c, err := parseRequestControls(map[string]string{"duration": value}, stream)
|
||||
Expect(err).NotTo(HaveOccurred(), value)
|
||||
Expect(c.durationFrames).To(Equal(frames), value)
|
||||
Expect(c.postprocess).To(Equal(int32(-1)))
|
||||
}
|
||||
}
|
||||
})
|
||||
It("rejects invalid duration and exact-boolean violations", func() {
|
||||
for _, stream := range []bool{false, true} {
|
||||
for _, value := range invalidDurations {
|
||||
_, err := parseRequestControls(map[string]string{"duration": value}, stream)
|
||||
Expect(err).To(HaveOccurred(), value)
|
||||
}
|
||||
for _, value := range invalidPostprocess {
|
||||
_, err := parseRequestControls(map[string]string{"postprocess": value}, stream)
|
||||
Expect(err).To(HaveOccurred(), value)
|
||||
}
|
||||
}
|
||||
})
|
||||
It("distinguishes omission, true, and false", func() {
|
||||
for _, value := range []string{"true", "false"} {
|
||||
c, err := parseRequestControls(map[string]string{"postprocess": value}, false)
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
Expect(c.postprocess).To(Equal(int32(boolToInt(value == "true"))))
|
||||
_, err = parseRequestControls(map[string]string{"postprocess": value}, true)
|
||||
if value == "false" {
|
||||
Expect(err).To(HaveOccurred())
|
||||
} else {
|
||||
Expect(err).NotTo(HaveOccurred())
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,21 @@
|
||||
MIT License
|
||||
|
||||
Copyright (c) 2023-2026 The omnivoice.cpp authors
|
||||
|
||||
Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
of this software and associated documentation files (the "Software"), to deal
|
||||
in the Software without restriction, including without limitation the rights
|
||||
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
copies of the Software, and to permit persons to whom the Software is
|
||||
furnished to do so, subject to the following conditions:
|
||||
|
||||
The above copyright notice and this permission notice shall be included in all
|
||||
copies or substantial portions of the Software.
|
||||
|
||||
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
SOFTWARE.
|
||||
@@ -0,0 +1,21 @@
|
||||
# Model-free native fixture
|
||||
|
||||
`omnivoice.h` is an unmodified public header from ServeurpersoCom/omnivoice.cpp,
|
||||
commit `b549668434fbe0a8592ae26adc08c1aa4e329805` (the backend's pinned engine).
|
||||
Source: https://raw.githubusercontent.com/ServeurpersoCom/omnivoice.cpp/b549668434fbe0a8592ae26adc08c1aa4e329805/src/omnivoice.h
|
||||
SHA-256: `891bbccd2ab63b7ce2e2cb8a35b0f57a347198086d9f003dbd98a164f0b1261a`.
|
||||
|
||||
`LICENSE.omnivoice` retains the upstream MIT license from the same commit:
|
||||
https://raw.githubusercontent.com/ServeurpersoCom/omnivoice.cpp/b549668434fbe0a8592ae26adc08c1aa4e329805/LICENSE
|
||||
SHA-256: `cddbecd5db98ec5bd44af3e7221b6d44c3e9fed1c92940a3be5b0080c9b86475`.
|
||||
|
||||
The verification script compiles the production wrapper against this header.
|
||||
`engine.cpp` implements only the engine calls the wrapper needs. Its initializer
|
||||
uses the documented defaults. Captures copy borrowed strings and reference data
|
||||
before returning. Its fake context never opens a model. Synthetic PCM exercises
|
||||
allocation, copying, freeing, and callbacks, not inference or audio quality.
|
||||
`ggml-backend.h` supplies only the wrapper's logging and backend declarations.
|
||||
|
||||
Run `bash backend/go/omnivoice-cpp/verify-controls.sh` from an offline Go setup
|
||||
with generated protobuf files. The script requires Linux and a C++17 compiler.
|
||||
No engine build, models, services, or downloads run during verification.
|
||||
@@ -0,0 +1,90 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#include "omnivoice.h"
|
||||
#include "ggml-backend.h"
|
||||
#include <cstdlib>
|
||||
#include <cstring>
|
||||
#include <string>
|
||||
|
||||
struct ov_context {};
|
||||
static ov_context context;
|
||||
static ov_tts_params captured;
|
||||
static std::string text, lang, instruct, ref_text;
|
||||
static int calls, frees, defaults;
|
||||
static float ref_first;
|
||||
void ggml_log_set(void (*)(ggml_log_level, const char *, void *), void *) {}
|
||||
void ggml_backend_load_all() {}
|
||||
|
||||
extern "C" {
|
||||
const char *ov_version() { return "model-free fixture"; }
|
||||
const char *ov_last_error() { return "fixture error"; }
|
||||
void ov_init_default_params(ov_init_params *p) {
|
||||
*p = {};
|
||||
p->abi_version = OV_ABI_VERSION;
|
||||
p->use_fa = true;
|
||||
}
|
||||
ov_context *ov_init(const ov_init_params *) { return &context; }
|
||||
void ov_free(ov_context *) {}
|
||||
void ov_tts_default_params(ov_tts_params *p) {
|
||||
++defaults;
|
||||
*p = {};
|
||||
p->abi_version = OV_ABI_VERSION;
|
||||
p->chunk_duration_sec = 15;
|
||||
p->chunk_threshold_sec = 30;
|
||||
p->denoise = p->preprocess_prompt = p->postproc = true;
|
||||
p->mg_num_step = 32;
|
||||
p->mg_guidance_scale = 2;
|
||||
p->mg_t_shift = 0.1f;
|
||||
p->mg_layer_penalty_factor = p->mg_position_temperature = 5;
|
||||
p->mg_seed = 42;
|
||||
}
|
||||
ov_status ov_synthesize(ov_context *, const ov_tts_params *p, ov_audio *out) {
|
||||
++calls;
|
||||
captured = *p;
|
||||
text = p->text ? p->text : "";
|
||||
lang = p->lang ? p->lang : "";
|
||||
instruct = p->instruct ? p->instruct : "";
|
||||
ref_text = p->ref_text ? p->ref_text : "";
|
||||
ref_first = p->ref_n_samples ? p->ref_audio_24k[0] : 0;
|
||||
float pcm[] = {0.25f, -0.25f};
|
||||
if (p->on_chunk) {
|
||||
return p->on_chunk(pcm, 2, p->on_chunk_user_data) ? OV_STATUS_OK : OV_STATUS_CANCELLED;
|
||||
}
|
||||
out->samples = static_cast<float *>(malloc(sizeof(pcm)));
|
||||
memcpy(out->samples, pcm, sizeof(pcm));
|
||||
out->n_samples = 2;
|
||||
out->sample_rate = 24000;
|
||||
return OV_STATUS_OK;
|
||||
}
|
||||
void ov_audio_free(ov_audio *out) {
|
||||
++frees;
|
||||
free(out->samples);
|
||||
*out = {};
|
||||
}
|
||||
void fixture_reset() {
|
||||
captured = {};
|
||||
calls = frees = defaults = 0;
|
||||
text.clear(); lang.clear(); instruct.clear(); ref_text.clear();
|
||||
ref_first = 0;
|
||||
}
|
||||
int fixture_calls() { return calls; }
|
||||
int fixture_frees() { return frees; }
|
||||
int fixture_defaults() { return defaults; }
|
||||
int fixture_frames() { return captured.T_override; }
|
||||
int fixture_postproc() { return captured.postproc; }
|
||||
int fixture_denoise() { return captured.denoise; }
|
||||
int fixture_ref_n() { return captured.ref_n_samples; }
|
||||
float fixture_ref_first() { return ref_first; }
|
||||
uint64_t fixture_seed() { return captured.mg_seed; }
|
||||
const char *fixture_text() { return text.c_str(); }
|
||||
const char *fixture_lang() { return lang.c_str(); }
|
||||
const char *fixture_instruct() { return instruct.c_str(); }
|
||||
const char *fixture_ref_text() { return ref_text.c_str(); }
|
||||
int fixture_other_defaults() {
|
||||
return captured.abi_version == OV_ABI_VERSION &&
|
||||
captured.chunk_duration_sec == 15 && captured.chunk_threshold_sec == 30 &&
|
||||
captured.preprocess_prompt && captured.mg_num_step == 32 &&
|
||||
captured.mg_guidance_scale == 2 && captured.mg_t_shift == 0.1f &&
|
||||
captured.mg_layer_penalty_factor == 5 && captured.mg_position_temperature == 5 &&
|
||||
captured.mg_class_temperature == 0 && !captured.cancel && !captured.dump_dir;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
#pragma once
|
||||
// Only the logging/backend declarations used by the production wrapper.
|
||||
enum ggml_log_level { GGML_LOG_LEVEL_DEBUG, GGML_LOG_LEVEL_INFO,
|
||||
GGML_LOG_LEVEL_WARN, GGML_LOG_LEVEL_ERROR };
|
||||
void ggml_log_set(void (*)(ggml_log_level, const char *, void *), void *);
|
||||
void ggml_backend_load_all();
|
||||
@@ -0,0 +1,324 @@
|
||||
#pragma once
|
||||
// omnivoice.h: public ABI for omnivoice.cpp.
|
||||
//
|
||||
// Single-header public API. Pure C99, consumable from C and C++ alike.
|
||||
// Bindings (Python ctypes, Rust bindgen, Go cgo) parse this file directly.
|
||||
// Style follows whisper.h / llama.h: extern "C" linkage on every entry,
|
||||
// POD structs only, const char * UTF-8 strings, ov_status enum returns.
|
||||
//
|
||||
// The opaque ov_context handle aggregates every module the synthesis path
|
||||
// needs (LM weights, audio tokenizer codec, BPE tokenizer, voice-design
|
||||
// vocabulary, GGML backend pair). One init, one free, one synthesize call
|
||||
// covers the full TTS path. The lower-level pipeline_*_load /
|
||||
// pipeline_*_free entries declared in pipeline-tts.h / pipeline-codec.h
|
||||
// stay available for the debug paths that need partial init, but they
|
||||
// are intentionally not part of this public ABI.
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
// Symbol visibility. Three Windows cases: building the SHARED target
|
||||
// (OMNIVOICE_BUILD set, dllexport), consuming the SHARED target from
|
||||
// outside (nothing set, dllimport), consuming the STATIC archive
|
||||
// (OMNIVOICE_STATIC set by the static target's INTERFACE definitions,
|
||||
// empty so the linker resolves the symbol directly without dllimport).
|
||||
// On GCC/Clang the default-visibility attribute is harmless on static
|
||||
// builds and required on shared builds.
|
||||
#if defined(_WIN32) || defined(__CYGWIN__)
|
||||
# if defined(OMNIVOICE_STATIC)
|
||||
# define OV_API
|
||||
# elif defined(OMNIVOICE_BUILD)
|
||||
# define OV_API __declspec(dllexport)
|
||||
# else
|
||||
# define OV_API __declspec(dllimport)
|
||||
# endif
|
||||
#elif defined(__GNUC__) || defined(__clang__)
|
||||
# define OV_API __attribute__((visibility("default")))
|
||||
#else
|
||||
# define OV_API
|
||||
#endif
|
||||
|
||||
// Struct ABI version. Incremented every time a public POD struct grows a
|
||||
// new field at the end. Callers fill `.abi_version = OV_ABI_VERSION` (or
|
||||
// let ov_*_default_params set it). Entries that consume those structs
|
||||
// reject inputs whose abi_version exceeds the build-time constant: this
|
||||
// guards a binary built against vN from receiving a struct laid out for
|
||||
// vN+1 by a freshly compiled binding. Adding fields stays backward-compat
|
||||
// because the new tail is zero-init in older callers and the lib reads
|
||||
// only what its abi_version permits.
|
||||
//
|
||||
// There is no separate semver triple. The runtime build identity is the
|
||||
// git short hash + commit date string returned by ov_version(); for
|
||||
// binding compat checks, OV_ABI_VERSION is the only number that matters.
|
||||
#define OV_ABI_VERSION 3
|
||||
|
||||
// Returns a static string of the form "<git-hash> (<date>)" identifying
|
||||
// the exact commit this binary was built from. Safe to call from any
|
||||
// thread, no allocation. Pointer stays valid for the process lifetime.
|
||||
OV_API const char * ov_version(void);
|
||||
|
||||
// Status code returned by every fallible entry. OV_STATUS_OK is always
|
||||
// zero so `if (rc)` reads as `if (rc != OV_STATUS_OK)`.
|
||||
enum ov_status {
|
||||
OV_STATUS_OK = 0,
|
||||
OV_STATUS_INVALID_PARAMS = -1,
|
||||
OV_STATUS_INSTRUCT_INVALID = -2,
|
||||
OV_STATUS_GENERATE_FAILED = -3,
|
||||
OV_STATUS_OOM = -4,
|
||||
OV_STATUS_CANCELLED = -5,
|
||||
};
|
||||
|
||||
// Returns the last error message produced on the calling thread by any
|
||||
// ov_* entry, as a NUL-terminated UTF-8 string. errno-style semantics: the
|
||||
// pointer is only meaningful right after a failure (ov_init returning NULL,
|
||||
// or any ov_* entry returning a negative ov_status); calling it after a
|
||||
// successful entry yields the previous message or an empty string. Storage
|
||||
// is thread-local so two threads running ov_synthesize concurrently never
|
||||
// race on each other's diagnostics. The pointer stays valid until the next
|
||||
// failing ov_* entry on the same thread.
|
||||
OV_API const char * ov_last_error(void);
|
||||
|
||||
// Output audio buffer. Plain POD: the samples pointer is malloc-allocated
|
||||
// by ov_synthesize, owned by the struct, released by ov_audio_free. Do not
|
||||
// free samples directly nor reassign without freeing first. Zero-initialise
|
||||
// before the first use: `struct ov_audio a = {0};`.
|
||||
struct ov_audio {
|
||||
float * samples; // mono PCM, malloc-allocated
|
||||
int n_samples; // length in samples
|
||||
int sample_rate; // 24000 for OmniVoice
|
||||
int channels; // 1 (mono)
|
||||
};
|
||||
|
||||
// Release the samples buffer and reset the struct to empty. Safe on a
|
||||
// zero-initialised struct (no double free, no NULL deref).
|
||||
OV_API void ov_audio_free(struct ov_audio * a);
|
||||
|
||||
// Opaque handle. Definition lives in omnivoice.cpp. Use ov_init / ov_free.
|
||||
struct ov_context;
|
||||
|
||||
// Initialisation parameters. model_path is required (the LM GGUF). When
|
||||
// codec_path is NULL the codec module is skipped and ov_synthesize fails
|
||||
// immediately with OV_STATUS_INVALID_PARAMS. use_fa enables flash
|
||||
// attention when a GPU backend is present; clamp_fp16 guards FP16
|
||||
// matmul accumulation on sub-Ampere CUDA. abi_version stays first so a
|
||||
// future struct growth keeps reading the version field at offset 0.
|
||||
struct ov_init_params {
|
||||
int abi_version;
|
||||
const char * model_path;
|
||||
const char * codec_path;
|
||||
bool use_fa;
|
||||
bool clamp_fp16;
|
||||
};
|
||||
|
||||
// Initialise to the standard defaults: codec_path NULL (caller must set
|
||||
// it for synthesis), use_fa true, clamp_fp16 false.
|
||||
OV_API void ov_init_default_params(struct ov_init_params * p);
|
||||
|
||||
// Allocate every module described by params. Returns NULL on any failure
|
||||
// after releasing whatever it has allocated so far. The returned handle
|
||||
// owns its GGML backend pair and must be released with ov_free.
|
||||
OV_API struct ov_context * ov_init(const struct ov_init_params * params);
|
||||
|
||||
// Release every module owned by the handle and free the handle itself.
|
||||
// Safe on NULL.
|
||||
OV_API void ov_free(struct ov_context * ov);
|
||||
|
||||
// Cooperative cancellation callback. Returns true to request the
|
||||
// synthesis to abort. Polled between chunks of long-form output, so the
|
||||
// cancel granularity is roughly chunk_duration_sec.
|
||||
typedef bool (*ov_cancel_cb)(void * user_data);
|
||||
|
||||
// Streaming output callback. When set on ov_tts_params, the synth pipeline
|
||||
// runs in streaming mode: audio is post processed and emitted chunk by
|
||||
// chunk through this callback rather than accumulated into the `out` buffer
|
||||
// of ov_synthesize. Returning false aborts the synthesis with
|
||||
// OV_STATUS_CANCELLED, identical to the ov_cancel_cb behaviour. The samples
|
||||
// pointer is mono float PCM at the codec sample rate; valid only for the
|
||||
// duration of the call. user_data is forwarded verbatim from on_chunk_user_data.
|
||||
//
|
||||
// Bit perfect against the buffered path for voice cloning. For voice design
|
||||
// (no reference) the streaming pipeline skips peak / 0.5 normalisation since
|
||||
// the global peak is unknowable before the last sample: output level then
|
||||
// runs roughly 6 to 12 dB below the buffered path. Logged at INFO when this
|
||||
// branch fires.
|
||||
typedef bool (*ov_audio_chunk_cb)(const float * samples, int n_samples, void * user_data);
|
||||
|
||||
// Log severity. Numerically ordered so a callback can filter with a
|
||||
// simple `if (level < threshold) return;`. ERROR is reserved for failure
|
||||
// reports that the lib also surfaces via ov_status / ov_last_error;
|
||||
// WARN for recoverable surprises; INFO for the normal load and
|
||||
// synthesis cadence; DEBUG for tensor-level cossim diagnostics.
|
||||
enum ov_log_level {
|
||||
OV_LOG_DEBUG = 0,
|
||||
OV_LOG_INFO = 1,
|
||||
OV_LOG_WARN = 2,
|
||||
OV_LOG_ERROR = 3,
|
||||
};
|
||||
|
||||
// Logging callback. msg is a NUL-terminated UTF-8 string already formatted
|
||||
// by the lib, with no trailing newline (the callback is free to add one).
|
||||
// user_data is forwarded verbatim from ov_log_set. Called from any thread
|
||||
// the lib runs on: the callback must be reentrant.
|
||||
typedef void (*ov_log_cb)(enum ov_log_level level, const char * msg, void * user_data);
|
||||
|
||||
// Install a global log callback. Passing cb == NULL restores the default
|
||||
// behaviour (write to stderr). Safe to call at any point; takes effect
|
||||
// immediately on subsequent log emissions across every thread. Storage
|
||||
// is process-wide, not per-handle, matching whisper_log_set / llama_log_set.
|
||||
OV_API void ov_log_set(ov_log_cb cb, void * user_data);
|
||||
|
||||
// Synthesis parameters. Strings are NULL-terminated UTF-8; NULL maps to
|
||||
// empty. Reference inputs are mutually exclusive: either pre-encoded
|
||||
// tokens [K, ref_T] OR raw 24 kHz mono samples. Passing both fails with
|
||||
// OV_STATUS_INVALID_PARAMS. The MaskGIT sampler config is flattened
|
||||
// directly into this struct (seven mg_* fields) to keep it fully POD.
|
||||
// abi_version stays first so the lib can route on it before reading any
|
||||
// field that may have shifted in a future minor.
|
||||
struct ov_tts_params {
|
||||
int abi_version;
|
||||
|
||||
// Input text and language hint. lang accepts "" or "none" for auto, an
|
||||
// ISO id ("fr") or a language name ("french"), resolved against the table
|
||||
// the model was trained with; ov_n_languages enumerates it, and anything
|
||||
// else is OV_STATUS_INVALID_PARAMS. instruct is the raw attribute string
|
||||
// ("female young adult moderate"), validated and normalised internally
|
||||
// against the bundled VoiceDesign.
|
||||
const char * text;
|
||||
const char * lang;
|
||||
const char * instruct;
|
||||
|
||||
// Duration control. T_override > 0 forces single-shot at the exact
|
||||
// frame count and bypasses both the estimator and the chunker.
|
||||
// T_override == 0 lets the pipeline decide between single-shot and
|
||||
// chunked output. chunk_duration_sec <= 0 disables chunking entirely.
|
||||
int T_override;
|
||||
float chunk_duration_sec;
|
||||
float chunk_threshold_sec;
|
||||
|
||||
// Generation flags. denoise toggles the <|denoise|> marker emitted
|
||||
// only when a reference is present. preprocess_prompt mirrors the
|
||||
// upstream Python flag: when true, applies add_punctuation to
|
||||
// ref_text and silence-trims the raw reference waveform.
|
||||
bool denoise;
|
||||
bool preprocess_prompt;
|
||||
|
||||
// MaskGIT sampler config. Defaults match the reference :
|
||||
// num_step=32, guidance_scale=2.0, t_shift=0.1,
|
||||
// layer_penalty_factor=5.0, position_temperature=5.0,
|
||||
// class_temperature=0.0, seed=42.
|
||||
int mg_num_step;
|
||||
float mg_guidance_scale;
|
||||
float mg_t_shift;
|
||||
float mg_layer_penalty_factor;
|
||||
float mg_position_temperature;
|
||||
float mg_class_temperature;
|
||||
uint64_t mg_seed;
|
||||
|
||||
// Optional voice reference. Two mutually exclusive ways to supply it.
|
||||
const int32_t * ref_audio_tokens;
|
||||
int ref_T;
|
||||
const float * ref_audio_24k;
|
||||
int ref_n_samples;
|
||||
const char * ref_text;
|
||||
|
||||
// Intermediate tensor dump directory. NULL disables dumps.
|
||||
const char * dump_dir;
|
||||
|
||||
// Cooperative cancellation. cancel NULL disables the feature.
|
||||
// cancel_user_data is forwarded to the callback verbatim.
|
||||
ov_cancel_cb cancel;
|
||||
void * cancel_user_data;
|
||||
|
||||
// Streaming output. When on_chunk is non NULL, ov_synthesize runs the
|
||||
// streaming pipeline: audio chunks emit through on_chunk and `out`
|
||||
// stays empty on success. on_chunk NULL keeps the buffered path.
|
||||
ov_audio_chunk_cb on_chunk;
|
||||
void * on_chunk_user_data;
|
||||
|
||||
// Output post filtering toggle for the buffered path. true (default)
|
||||
// keeps the reference behaviour: silence removal, per utterance peak
|
||||
// normalisation when no reference is set, and edge fade plus padding.
|
||||
// false returns the raw decode at exactly T_override * hop samples, so a
|
||||
// caller assembling its own timeline onto fixed slots (dubbing) gets a
|
||||
// predictable segment length and owns its own edge shaping. Reference
|
||||
// loudness matching (ref_rms scaling) is part of voice cloning and runs
|
||||
// either way. Tail field: kept last for ABI growth, read only when
|
||||
// abi_version >= 3. The streaming path always post filters.
|
||||
bool postproc;
|
||||
};
|
||||
|
||||
// Initialise to the standard defaults. Strings NULL, T_override 0,
|
||||
// chunk_duration_sec 15, chunk_threshold_sec 30, denoise true,
|
||||
// preprocess_prompt true, MaskGIT defaults as above, every reference
|
||||
// pointer NULL, dump_dir NULL, cancel NULL, postproc true.
|
||||
OV_API void ov_tts_default_params(struct ov_tts_params * p);
|
||||
|
||||
// Run the full TTS synthesis. Resolves the instruct against the bundled
|
||||
// VoiceDesign vocabulary, picks between single-shot, chunked auto-voice
|
||||
// and voice-cloning paths from the params struct, and fills `out` with
|
||||
// mono float PCM at 24 kHz. Returns OV_STATUS_OK on success; on any
|
||||
// failure returns a negative ov_status describing the cause and leaves
|
||||
// `out` empty. Requires a codec-loaded handle.
|
||||
OV_API enum ov_status ov_synthesize(struct ov_context * ov, const struct ov_tts_params * params, struct ov_audio * out);
|
||||
|
||||
// Convert a duration in seconds to a frame count using the bundled codec
|
||||
// frame rate (sample_rate / hop_length). Clamps to a minimum of one
|
||||
// frame. Requires a codec-loaded handle.
|
||||
OV_API int ov_duration_sec_to_tokens(const struct ov_context * ov, float duration_sec);
|
||||
|
||||
// Number of RVQ codebooks (K) of the loaded model. Pre-encoded reference
|
||||
// tokens passed via ref_audio_tokens are laid out [K, ref_T] row-major;
|
||||
// callers reading a packed .rvq stream need K to derive ref_T from the
|
||||
// code count. Returns 0 on a NULL handle.
|
||||
OV_API int ov_num_codebooks(const struct ov_context * ov);
|
||||
|
||||
// Languages the synthesis accepts as a hint, in the order of the bundled
|
||||
// table. An empty hint leaves the model to infer one from the text, so "auto"
|
||||
// is not an entry here. The id is what the prompt carries ("fr"), the name is
|
||||
// what a user reads ("french"); both pointers are static and outlive any
|
||||
// context.
|
||||
OV_API int ov_n_languages(void);
|
||||
OV_API const char * ov_language_id(int i);
|
||||
OV_API const char * ov_language_name(int i);
|
||||
|
||||
// Pre-encoded voice reference codes, the in-process equivalent of an
|
||||
// omnivoice-codec CLI encode. Plain POD: ref_codes is malloc allocated
|
||||
// by ov_extract_voice_ref, owned by the struct, released by
|
||||
// ov_voice_ref_free. Do not free the pointer directly nor reassign
|
||||
// without freeing first. Zero initialise before first use:
|
||||
// `struct ov_voice_ref ref = {0};`.
|
||||
//
|
||||
// ref_codes is the RVQ code matrix equivalent to a raw .rvq file, laid
|
||||
// out [num_codebooks, ref_T] row-major (T fastest), ready to feed back
|
||||
// through ov_tts_params.ref_audio_tokens / ref_T.
|
||||
struct ov_voice_ref {
|
||||
int32_t * ref_codes;
|
||||
int ref_T;
|
||||
int num_codebooks;
|
||||
};
|
||||
|
||||
// Extract reusable voice-clone codes from a decoded reference audio
|
||||
// buffer: mono float32 PCM at 24 kHz. Requires a codec-loaded handle.
|
||||
// Applies the same preprocessing as the omnivoice-codec CLI and the
|
||||
// --ref-wav synth path (RMS auto-gain, silence trim, hop truncation),
|
||||
// so the codes are bit-identical to both and round-trip directly through
|
||||
// ov_tts_params.ref_audio_tokens.
|
||||
//
|
||||
// On success fills out with a malloc-owned buffer. On failure leaves out
|
||||
// empty and stores a diagnostic in ov_last_error().
|
||||
OV_API enum ov_status ov_extract_voice_ref(struct ov_context * ov,
|
||||
const float * ref_audio_24k,
|
||||
int ref_n_samples,
|
||||
struct ov_voice_ref * out);
|
||||
|
||||
// Release the RVQ code buffer and reset the struct to empty. Safe on a
|
||||
// zero initialised struct.
|
||||
OV_API void ov_voice_ref_free(struct ov_voice_ref * ref);
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
Executable
+34
@@ -0,0 +1,34 @@
|
||||
#!/usr/bin/env bash
|
||||
# SPDX-License-Identifier: MIT
|
||||
set -euo pipefail
|
||||
root=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)
|
||||
cd "$root"
|
||||
if [[ ! -f pkg/grpc/proto/backend.pb.go || ! -f pkg/grpc/proto/backend_grpc.pb.go ]]; then
|
||||
echo 'Missing generated protobuf Go files. Generate backend/backend.proto into pkg/grpc/proto with installed protoc and Go plugins before running this offline check.' >&2
|
||||
exit 1
|
||||
fi
|
||||
if [[ $(uname -s) != Linux ]]; then
|
||||
echo 'This model-free native verification currently requires Linux.' >&2
|
||||
exit 1
|
||||
fi
|
||||
build=$(mktemp -d "$root/.omnivoice-controls.XXXXXX")
|
||||
trap 'rm -rf "$build"' EXIT
|
||||
backend="$root/backend/go/omnivoice-cpp"
|
||||
# Ordinary package tests must not depend on the optional native fixture.
|
||||
env -u OMNIVOICE_TEST_LIBRARY go test -mod=readonly -count=1 -timeout=60s ./backend/go/omnivoice-cpp -ginkgo.label-filter='!e2e' -ginkgo.fail-on-empty
|
||||
# Selecting native specs without their fixture must fail, not silently skip.
|
||||
if env -u OMNIVOICE_TEST_LIBRARY go test -tags=native_controls -mod=readonly -count=1 -timeout=60s ./backend/go/omnivoice-cpp -ginkgo.label-filter='native-controls' -ginkgo.fail-on-empty -ginkgo.no-color >"$build/missing-fixture.log" 2>&1; then
|
||||
cat "$build/missing-fixture.log"
|
||||
echo 'Native controls unexpectedly passed without their fixture.' >&2
|
||||
exit 1
|
||||
fi
|
||||
cat "$build/missing-fixture.log"
|
||||
# Do not mistake compilation errors or an empty selection for the expected failure.
|
||||
grep -Fq 'run verify-controls.sh to compile the actual wrapper fixture' "$build/missing-fixture.log"
|
||||
"${CXX:-c++}" -std=c++17 -shared -fPIC -I"$backend/testdata/native" \
|
||||
"$backend/cpp/gomnivoicecpp.cpp" "$backend/testdata/native/engine.cpp" \
|
||||
-o "$build/libomnivoice-controls.so"
|
||||
export OMNIVOICE_TEST_LIBRARY="$build/libomnivoice-controls.so"
|
||||
go test -tags=native_controls -mod=readonly -count=1 -timeout=60s ./backend/go/omnivoice-cpp -ginkgo.label-filter='native-controls' -ginkgo.fail-on-empty
|
||||
go test -tags=native_controls -mod=readonly -count=1 -timeout=60s ./backend/go/omnivoice-cpp -ginkgo.label-filter='!e2e'
|
||||
git diff --check
|
||||
@@ -608,6 +608,29 @@ options:
|
||||
|
||||
A per-request `seed` can also be supplied through the `params` map alongside `ref_text`.
|
||||
|
||||
#### Duration and postprocessing
|
||||
|
||||
For `omnivoice-cpp`, set these request-local controls in `params`, not as top-level fields:
|
||||
|
||||
```json
|
||||
{
|
||||
"model": "omnivoice-cpp",
|
||||
"input": "Hello world.",
|
||||
"params": {"duration":"2.5","postprocess":"false"}
|
||||
}
|
||||
```
|
||||
|
||||
- `duration` accepts a finite numeric string of seconds greater than zero and at most `30`, inclusive. Whitespace, empty strings, and nonfinite values are invalid.
|
||||
- The backend uses `max(1, floor(seconds * 25))` generation frames. Each frame represents 40 milliseconds at the bundled codec rate. Thus, `2.5` selects 62 frames; tiny positive values select one frame.
|
||||
- `postprocess` accepts only the exact strings `"true"` and `"false"`. Buffered synthesis accepts either value, with or without `duration`.
|
||||
- Omitted controls preserve engine defaults. Omitted duration leaves duration estimation and chunking unchanged.
|
||||
|
||||
A supplied duration forces single-shot generation and bypasses chunking. The 30-second cap is a conservative wrapper resource policy, aligned with the engine's default chunk threshold. It is not an engine positional limit or a guarantee of available memory or successful synthesis.
|
||||
|
||||
Duration controls generation frames, not playback speed or guaranteed final WAV duration. Buffered postprocessing can trim or pad audio. With `postprocess="false"`, buffered output preserves the raw decoded frame length; reference loudness matching remains active. Quantization does not guarantee perceptual accuracy.
|
||||
|
||||
Streaming accepts `duration` and explicit `postprocess="true"`. It rejects `postprocess="false"` before reference reads, native synthesis, or audio output because the streaming engine always postprocesses audio.
|
||||
|
||||
### Pocket TTS
|
||||
|
||||
[Pocket TTS](https://github.com/kyutai-labs/pocket-tts) is a lightweight text-to-speech model designed to run efficiently on CPUs. It supports voice cloning through HuggingFace voice URLs or local audio files.
|
||||
|
||||
Reference in new issue
Block a user