From 0b7d0b40edbf4e5d167b856ba51ff7286d899d00 Mon Sep 17 00:00:00 2001 From: "nexxtmobile.de" Date: Sun, 25 Oct 2026 05:42:55 +0000 Subject: [PATCH] feat(qwen3-tts-cpp): warm up every batch width at load qwentts builds its code-predictor and codec-stream graphs lazily per batch width. With max_batch > 1 the first request that hits a new width (for example two requests finishing in the same step) stalled the whole batch for ~5 s on an RTX 5060 Ti; afterwards the same pattern took ~1 s. When max_batch > 1, Load now synthesizes N short utterances of increasing length concurrently through the streaming path and discards the audio, so the slots finish one by one and every width from N down to 1 is built before the first real request. `max_batch_warmup:false` skips it. Signed-off-by: nexxtmobile.de Co-Authored-By: Claude Opus 5.5 --- backend/go/qwen3-tts-cpp/README.md | 5 ++++ backend/go/qwen3-tts-cpp/goqwen3ttscpp.go | 30 ++++++++++++++++++++ backend/go/qwen3-tts-cpp/options.go | 7 ++++- backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go | 28 ++++++++++++++++++ 4 files changed, 69 insertions(+), 1 deletion(-) diff --git a/backend/go/qwen3-tts-cpp/README.md b/backend/go/qwen3-tts-cpp/README.md index b354b6ab97..cf137e6579 100644 --- a/backend/go/qwen3-tts-cpp/README.md +++ b/backend/go/qwen3-tts-cpp/README.md @@ -68,6 +68,11 @@ requests (time to the first PCM chunk): RTX 5060 Ti 0.02/0.36/0.70/1.04 s with `max_batch:1`, 0.06 s for all four with `max_batch:4`. VRAM grows with N (about 2.6 GB at 1, 6 GB at 4, 10.5 GB at 8 for that talker). +- With `max_batch > 1` the backend warms up at load: it synthesizes N short + utterances of increasing length concurrently, so qwentts builds its graphs + for every batch width before the first request (otherwise the first + request that hits a new width stalls the batch for seconds). + `max_batch_warmup:false` skips it. - Incremental synthesis (`TTSLive`, the realtime pipeline with `streaming.tts: true`) requires `max_batch:1` and is rejected otherwise (qwentts `docs/INCREMENTAL.md`). diff --git a/backend/go/qwen3-tts-cpp/goqwen3ttscpp.go b/backend/go/qwen3-tts-cpp/goqwen3ttscpp.go index b4e0a9cf92..9337799b01 100644 --- a/backend/go/qwen3-tts-cpp/goqwen3ttscpp.go +++ b/backend/go/qwen3-tts-cpp/goqwen3ttscpp.go @@ -7,6 +7,7 @@ import ( "strings" "sync" "sync/atomic" + "time" "unsafe" "github.com/ebitengine/purego" @@ -90,9 +91,38 @@ func (q *Qwen3TtsCpp) Load(opts *pb.ModelOptions) error { return fmt.Errorf("qwen3-tts: failed to load model (rc=%d)", rc) } q.maxBatch.Store(int32(q.opts.maxBatch)) + if q.opts.maxBatch > 1 && q.opts.batchWarmup { + warmupBatch(q.opts.maxBatch) + } return nil } +// warmupBatch synthesizes n short utterances of increasing length +// concurrently through the streaming path, discarding the audio. qt builds +// its code-predictor and codec-stream graphs lazily per batch width; the +// first request that hits a new width (e.g. two requests finishing in the +// same step) otherwise stalls the whole batch for seconds (~5 s per width on +// an RTX 5060 Ti). The staggered lengths make the slots finish one by one, so +// every width from n down to 1 is built here, at load. +func warmupBatch(n int) { + streamCbOnce.Do(func() { + streamCbPtr = purego.NewCallback(streamCallback) + }) + start := time.Now() + var wg sync.WaitGroup + for i := 0; i < n; i++ { + text := strings.TrimSpace(strings.Repeat("Hello there. ", i+1)) + wg.Add(1) + go func() { + defer wg.Done() + // user_data 0 is never a registered route: chunks are dropped. + CppTTSStream(text, "", "", "", nil, 0, "", 0, 0, 0, 0, 0, 96, streamCbPtr, 0) + }() + } + wg.Wait() + fmt.Fprintf(os.Stderr, "[qwen3-tts-cpp] batch warmup (%d concurrent) done in %s\n", n, time.Since(start).Round(time.Millisecond)) +} + // discoverTokenizer returns the first *tokenizer*.gguf in dir, or "". func discoverTokenizer(dir string) string { entries, err := os.ReadDir(dir) diff --git a/backend/go/qwen3-tts-cpp/options.go b/backend/go/qwen3-tts-cpp/options.go index e50b6972db..e2673f8141 100644 --- a/backend/go/qwen3-tts-cpp/options.go +++ b/backend/go/qwen3-tts-cpp/options.go @@ -14,6 +14,9 @@ type loadOptions struct { // maxBatch > 1 sizes qt's KV cache sets and lets its compute worker batch // concurrent TTS/TTSStream requests on one handle (qt_init_params.max_batch). maxBatch int + // batchWarmup runs a staggered concurrent synthesis at load so every + // batch width is built before the first real request (see warmupBatch). + batchWarmup bool } // sampling holds per-request generation parameters with qt defaults applied. @@ -40,7 +43,7 @@ func parseBool(v string) bool { return v == "true" || v == "1" } // ignored. Defaults: use_fa true (qt default; CPU still uses the F32 chain), // seed -1 (engine random). func parseOptions(opts []string) loadOptions { - o := loadOptions{useFA: true, seed: -1, maxBatch: 1} + o := loadOptions{useFA: true, seed: -1, maxBatch: 1, batchWarmup: true} for _, oo := range opts { key, value, ok := splitOption(oo) if !ok { @@ -61,6 +64,8 @@ func parseOptions(opts []string) loadOptions { if n, err := strconv.Atoi(value); err == nil && n >= 1 { o.maxBatch = n } + case "max_batch_warmup": + o.batchWarmup = parseBool(value) } } return o diff --git a/backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go b/backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go index 7b35a193d8..f750d9f434 100644 --- a/backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go +++ b/backend/go/qwen3-tts-cpp/qwen3ttscpp_test.go @@ -258,3 +258,31 @@ var _ = Describe("TTSLive with max_batch", func() { Expect(err).To(MatchError(ContainSubstring("requires max_batch=1"))) }) }) + +var _ = Describe("batch warmup", func() { + It("is on by default and can be disabled", func() { + Expect(parseOptions(nil).batchWarmup).To(BeTrue()) + Expect(parseOptions([]string{"max_batch_warmup:false"}).batchWarmup).To(BeFalse()) + }) + It("runs n concurrent streaming syntheses of increasing length", func() { + original := CppTTSStream + DeferCleanup(func() { CppTTSStream = original }) + var mu sync.Mutex + var texts []string + var inside sync.WaitGroup + inside.Add(3) + CppTTSStream = func(text, _, _, _ string, _ unsafe.Pointer, _ int, _ string, _ int64, _ float32, _ int, _, _ float32, _ int, _ uintptr, user uintptr) int { + Expect(user).To(Equal(uintptr(0))) + mu.Lock() + texts = append(texts, text) + mu.Unlock() + inside.Done() + inside.Wait() // all three must be in flight together + return 0 + } + done := make(chan struct{}) + go func() { warmupBatch(3); close(done) }() + Eventually(done).WithTimeout(5 * time.Second).Should(BeClosed()) + Expect(texts).To(ConsistOf("Hello there.", "Hello there. Hello there.", "Hello there. Hello there. Hello there.")) + }) +})