fix(llama-cpp): stop non-TTS models crashing on the new pin

Two regressions, both hit every ordinary llama-cpp model and neither was
caught locally because every test on this branch loaded a TTS model.

The first is a null dereference. server_slot::tts_ctx::reset() called
mtmd_helper_gen_audio_reset() unconditionally, but the gen-audio pipeline
is only allocated for models carrying a gen-audio mmproj, and upstream's
implementation reads ctx->pipeline before null-checking anything. Since
server_slot::reset() runs during slot initialization for every model, any
non-TTS model segfaulted the backend the moment it loaded. Guard the call
on the is_supported() predicate already defined beside it, and keep the
plain field resets unconditional.

The second is unrelated to TTS and came in with the pin bump.
PredictOptions.Penalty is a bare proto float, so a caller that names no
repetition penalty sends 0 rather than omitting the field. Since
9de0fcf2b, common_sampler_init() rejects a non-positive penalty_repeat
outright because it would divide logits by zero, turning every such
request into "Failed to initialize samplers". Treat 0 as unset and leave
llama.cpp's own neutral default in place.

Verified with the same suite CI runs, which is what caught both:
tests/e2e-backends passes 6 of 6 including the load and predict specs
that were red. Qwen3-TTS still synthesises on both paths, 24 kHz mono
16-bit WAV with exactly one RIFF header on the streamed output.

Assisted-by: Claude:claude-fable-5 [Claude Code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

backend/cpp/bonsai/Makefile

@@ -42,6 +42,7 @@ define bonsai-build
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
+	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:$(1)$(RESET))
 	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(BONSAI_VERSION) \
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build llama.cpp
@@ -79,6 +80,7 @@ bonsai-cpu-all:
 	rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
+	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
 	$(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET))
 	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(BONSAI_VERSION) \
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build llama.cpp

backend/cpp/llama-cpp/CMakeLists.txt

@@ -115,4 +115,9 @@ if(LLAMA_GRPC_BUILD_TESTS)
     target_include_directories(passthrough_options_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
     target_compile_features(passthrough_options_test PRIVATE cxx_std_17)
     add_test(NAME passthrough_options_test COMMAND passthrough_options_test)
+
+    add_executable(tts_request_options_test tts_request_options_test.cpp tts_request_options.h)
+    target_include_directories(tts_request_options_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
+    target_compile_features(tts_request_options_test PRIVATE cxx_std_17)
+    add_test(NAME tts_request_options_test COMMAND tts_request_options_test)
 endif()

backend/cpp/llama-cpp/Makefile

@@ -1,5 +1,5 @@
 
-LLAMA_VERSION?=221f0f6356efe2260023208365705ec5d5a7c8f5
+LLAMA_VERSION?=9de0fcf2b3e587a43f293d9a2b6ec0a32991f768
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
 
 CMAKE_ARGS?=

backend/cpp/llama-cpp/disable-tts-task.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+# Mark a copied gRPC server as targeting a llama.cpp fork that does not carry
+# LocalAI's SERVER_TASK_TYPE_TTS patch. The RPCs remain present in the shared
+# protobuf service, but respond with UNIMPLEMENTED instead of referencing
+# server task types and mtmd gen-audio APIs absent from those forks.
+
+set -euo pipefail
+
+if [[ $# -ne 1 ]]; then
+    echo "usage: $0 <grpc-server.cpp>" >&2
+    exit 2
+fi
+
+SRC=$1
+
+if [[ ! -f "$SRC" ]]; then
+    echo "grpc-server.cpp not found at $SRC" >&2
+    exit 2
+fi
+
+if grep -q '^#define LOCALAI_LLAMA_CPP_NO_TTS_TASK' "$SRC"; then
+    echo "==> $SRC already disables the LocalAI TTS task, skipping"
+    exit 0
+fi
+
+awk '
+    !done && /^#include/ {
+        print "#define LOCALAI_LLAMA_CPP_NO_TTS_TASK 1"
+        print "// ^ injected by disable-tts-task.sh for an unpatched llama.cpp fork"
+        print ""
+        done = 1
+    }
+    { print }
+    END {
+        if (!done) {
+            print "disable-tts-task.sh: no #include anchor found" > "/dev/stderr"
+            exit 1
+        }
+    }
+' "$SRC" > "$SRC.tmp"
+mv "$SRC.tmp" "$SRC"
+
+echo "==> LocalAI TTS task disabled in $SRC"

backend/cpp/llama-cpp/grpc-server.cpp

@@ -55,6 +55,7 @@
 #include "llama_compat.h" // fork-skew switches, generated by prepare.sh
 #include "message_content.h"
 #include "passthrough_options.h"
+#include "tts_request_options.h"
 #include <getopt.h>
 #include <grpcpp/ext/proto_server_reflection_plugin.h>
 #include <grpcpp/grpcpp.h>
@@ -65,6 +66,7 @@
 #include <atomic>
 #include <cmath>
 #include <cstdlib>
+#include <cstring>
 #include <fstream>
 #include <iterator>
 #include <list>
@@ -233,7 +235,15 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
     data["typical_p"] = predict->typicalp();
     data["temperature"] = predict->temperature();
     data["repeat_last_n"] = predict->repeat();
-    data["repeat_penalty"] = predict->penalty();
+    // PredictOptions.Penalty is a bare proto float, so a caller that names no
+    // repetition penalty sends 0 rather than omitting the field. Since
+    // llama.cpp 9de0fcf2b, common_sampler_init() rejects a non-positive
+    // penalty_repeat outright (it would divide logits by zero), which turned
+    // every such request into "Failed to initialize samplers". Treat 0 as
+    // "unset" and leave llama.cpp's own neutral default in place.
+    if (predict->penalty() > 0.0f) {
+        data["repeat_penalty"] = predict->penalty();
+    }
     data["frequency_penalty"] = predict->frequencypenalty();
     data["presence_penalty"] = predict->presencepenalty();
     data["mirostat"] = predict->mirostat();
@@ -1445,6 +1455,26 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
 }
 
 
+#ifndef LOCALAI_LLAMA_CPP_NO_TTS_TASK
+// MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM hands back raw float32 samples, but the
+// WAV header core/backend/tts.go builds around the streamed chunks announces
+// 16-bit samples, so the wire has to carry s16 or the client decodes floats as
+// integers and hears noise. The scaling matches write_wav16() in
+// tools/mtmd/mtmd-helper-gen.cpp, which is what the non-streaming path writes.
+static std::string tts_pcm_f32_to_s16(const std::string & samples) {
+    const size_t n = samples.size() / sizeof(float);
+    std::string out;
+    out.resize(n * sizeof(int16_t));
+    for (size_t i = 0; i < n; i++) {
+        float v = 0.0f;
+        std::memcpy(&v, samples.data() + i * sizeof(float), sizeof(float));
+        const int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);
+        std::memcpy(&out[i * sizeof(int16_t)], &s, sizeof(int16_t));
+    }
+    return out;
+}
+#endif
+
 // GRPC Server start
 class BackendServiceImpl final : public backend::Backend::Service {
 private:
@@ -2089,15 +2119,23 @@ public:
 
                 task.tokens    = std::move(inputs[i]);
 #ifdef LOCALAI_HAS_SERVER_SCHEMA
+                // The schema evaluator no longer takes the per-slot n_ctx: upstream
+                // dropped the parameter and server-schema stopped consulting n_ctx at
+                // all, leaving the context bound to the slot. Forks that predate the
+                // server-schema split still expect it, so only this branch loses it.
                 task.params           = server_schema::eval_llama_cmpl_schema(
+                        ctx_server.impl->vocab,
+                        params_base,
+                        ctx_server.get_meta().logit_bias_eog,
+                        data);
 #else
                 task.params           = server_task::params_from_json_cmpl(
-#endif
                         ctx_server.impl->vocab,
                         params_base,
                         ctx_server.get_meta().slot_n_ctx,
                         ctx_server.get_meta().logit_bias_eog,
                         data);
+#endif
                 task.id_slot = json_value(data, "id_slot", -1);
 
                 // OAI-compat: enable autoparser (PEG-based chat parsing) so that
@@ -2659,15 +2697,23 @@ public:
 
                 task.tokens    = std::move(inputs[i]);
 #ifdef LOCALAI_HAS_SERVER_SCHEMA
+                // The schema evaluator no longer takes the per-slot n_ctx: upstream
+                // dropped the parameter and server-schema stopped consulting n_ctx at
+                // all, leaving the context bound to the slot. Forks that predate the
+                // server-schema split still expect it, so only this branch loses it.
                 task.params           = server_schema::eval_llama_cmpl_schema(
+                        ctx_server.impl->vocab,
+                        params_base,
+                        ctx_server.get_meta().logit_bias_eog,
+                        data);
 #else
                 task.params           = server_task::params_from_json_cmpl(
-#endif
                         ctx_server.impl->vocab,
                         params_base,
                         ctx_server.get_meta().slot_n_ctx,
                         ctx_server.get_meta().logit_bias_eog,
                         data);
+#endif
                 task.id_slot = json_value(data, "id_slot", -1);
 
                 // OAI-compat: enable autoparser (PEG-based chat parsing) so that
@@ -2994,6 +3040,229 @@ public:
         return grpc::Status::OK;
     }
 
+#ifndef LOCALAI_LLAMA_CPP_NO_TTS_TASK
+    // Builds the shared TTS task from a request. Returns a non-OK status and
+    // leaves `task` untouched when the request is malformed or the loaded model
+    // cannot synthesise audio.
+    grpc::Status prepareTTSTask(const backend::TTSRequest* request, bool stream, server_task & task) {
+        if (!ctx_server.get_meta().has_cap_tts) {
+            return grpc::Status(grpc::StatusCode::UNIMPLEMENTED,
+                "the loaded model does not support audio generation (no gen-audio mmproj)");
+        }
+
+        std::map<std::string, std::string> params(request->params().begin(), request->params().end());
+        const auto opts = llama_grpc::parse_tts_request_options(
+            request->text(),
+            request->voice(),
+            request->has_language() ? request->language() : std::string(),
+            params);
+        if (!opts.ok) {
+            return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, opts.error);
+        }
+
+        auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false);
+        if (!wrapper.bitmap) {
+            return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
+                "failed to read speaker reference audio: " + opts.voice_path);
+        }
+
+        task.tts_inp.set_prompt(opts.text);
+        // core/backend/tts.go always sets TTSRequest.language, so has_language()
+        // is true even when the caller named no language and the string is empty.
+        // gen_audio::inp::get() already maps a stored blank to nullptr, so this
+        // guard is behavior-preserving rather than behavior-fixing. It is kept
+        // so the "unset" intent is visible at the call site instead of resting
+        // on a detail of the helper.
+        if (!opts.language.empty()) {
+            task.tts_inp.set_lang(opts.language);
+        }
+        task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+        task.tts_inp.data.top_k    = opts.top_k;
+        task.tts_inp.data.top_p    = opts.top_p;
+        task.tts_inp.data.stream   = stream;
+        task.tts_inp.data.out_type = stream
+            ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM   // Go prepends its own WAV header, see core/backend/tts.go
+            : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+
+        task.params.stream    = stream;
+        // -1 keeps upstream's 512-frame default. The model does not always emit
+        // its codec EOS, so a short input can otherwise generate the full cap.
+        task.params.n_predict = opts.max_frames > 0 ? opts.max_frames : -1;
+        task.params.sampling  = params_base.sampling;
+        // Both values mirror upstream's draft POST /tts handler. Note that the
+        // pair is INERT at this pin: llama_sampler_init_penalties() clamps
+        // penalty_last_n with std::max(penalty_last_n, 0), so -1 means "off",
+        // not "the whole generation", and the penalty sampler is then built
+        // disabled. No repetition penalty is actually applied.
+        //
+        // That is deliberate. Dropping the second line lets the sampling
+        // default of 64 apply and genuinely engages the 1.05 penalty, which was
+        // measured here against the model's habit of never emitting its codec
+        // EOS and running to the frame cap: 0 of 15 short requests ran away
+        // with the penalty inert, 1 of 15 with it active over the last 64
+        // tokens. It does not fix the runaway, so the line stays for parity
+        // with the draft. Use max_frames to bound the output instead.
+        task.params.sampling.penalty_repeat = 1.05f;
+        task.params.sampling.penalty_last_n = -1;
+        if (opts.top_k > 0) {
+            task.params.sampling.top_k = opts.top_k;
+        }
+        if (opts.top_p > 0) {
+            task.params.sampling.top_p = opts.top_p;
+        }
+
+        return grpc::Status::OK;
+    }
+
+    grpc::Status TTS(ServerContext* context, const backend::TTSRequest* request, backend::Result* result) override {
+        auto auth = checkAuth(context);
+        if (!auth.ok()) return auth;
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
+        if (params_base.model.path.empty()) {
+            return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
+        }
+        if (request->dst().empty()) {
+            return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "dst must name an output file path");
+        }
+
+        server_task task(SERVER_TASK_TYPE_TTS);
+        auto prepared = prepareTTSTask(request, /* stream= */ false, task);
+        if (!prepared.ok()) return prepared;
+
+        auto rd = ctx_server.get_response_reader();
+        task.id = rd.get_new_id();
+        rd.post_task(std::move(task));
+
+        auto should_stop = [context]() { return context->IsCancelled(); };
+
+        std::string audio;
+        while (true) {
+            auto res = rd.next(should_stop);
+            if (!res) {
+                return grpc::Status(grpc::StatusCode::CANCELLED, "TTS request cancelled");
+            }
+            if (res->is_error()) {
+                return grpc::Status(grpc::StatusCode::INTERNAL, res->to_json().dump());
+            }
+            auto * tts_res = dynamic_cast<server_task_result_tts *>(res.get());
+            if (tts_res == nullptr) {
+                return grpc::Status(grpc::StatusCode::INTERNAL, "unexpected result type for a TTS task");
+            }
+            audio.append(tts_res->audio);
+            if (tts_res->final) {
+                break;
+            }
+        }
+
+        std::ofstream out(request->dst(), std::ios::binary | std::ios::trunc);
+        if (!out) {
+            return grpc::Status(grpc::StatusCode::INTERNAL, "failed to open output file: " + request->dst());
+        }
+        out.write(audio.data(), (std::streamsize) audio.size());
+        if (!out) {
+            return grpc::Status(grpc::StatusCode::INTERNAL, "failed to write output file: " + request->dst());
+        }
+        // Buffered data is flushed here, so a full disk or a failing device can
+        // surface for the first time on close. Reporting success then would
+        // leave a truncated file behind under the name the caller will read.
+        out.close();
+        if (!out) {
+            return grpc::Status(grpc::StatusCode::INTERNAL, "failed to close output file: " + request->dst());
+        }
+
+        result->set_success(true);
+        result->set_message("TTS audio generated");
+        return grpc::Status::OK;
+    }
+
+    grpc::Status TTSStream(ServerContext* context, const backend::TTSRequest* request, grpc::ServerWriter<backend::Reply>* writer) override {
+        auto auth = checkAuth(context);
+        if (!auth.ok()) return auth;
+        auto identity = checkModelIdentity(request);
+        if (!identity.ok()) return identity;
+        if (params_base.model.path.empty()) {
+            return grpc::Status(grpc::StatusCode::FAILED_PRECONDITION, "Model not loaded");
+        }
+
+        server_task task(SERVER_TASK_TYPE_TTS);
+        auto prepared = prepareTTSTask(request, /* stream= */ true, task);
+        if (!prepared.ok()) return prepared;
+
+        auto rd = ctx_server.get_response_reader();
+        task.id = rd.get_new_id();
+        rd.post_task(std::move(task));
+
+        auto should_stop = [context]() { return context->IsCancelled(); };
+
+        // core/backend/tts.go:ModelTTSStream builds the WAV header itself from
+        // the sample rate in the first reply's Message, then concatenates every
+        // Reply.Audio verbatim. So the rate goes out once, up front, and the
+        // chunks stay raw PCM.
+        //
+        // Send it before draining rather than off the first audio result: a
+        // chunk needs a whole 72-frame window, about 5.8 s of audio and far
+        // longer in wall time on CPU, and the Go side cannot emit the WAV
+        // header until this reply lands. Waiting would hold the client at zero
+        // bytes for that entire stretch. The rate is a property of the loaded
+        // model, available synchronously, so there is nothing to wait for.
+        {
+            backend::Reply header;
+            const json info = { {"sample_rate", mtmd_gen_audio_get_info(ctx_server.impl->mctx).sample_rate} };
+            header.set_message(info.dump());
+            if (!writer->Write(header)) {
+                return grpc::Status(grpc::StatusCode::CANCELLED, "client closed the TTS stream");
+            }
+        }
+
+        while (true) {
+            auto res = rd.next(should_stop);
+            if (!res) {
+                return grpc::Status(grpc::StatusCode::CANCELLED, "TTS request cancelled");
+            }
+            if (res->is_error()) {
+                return grpc::Status(grpc::StatusCode::INTERNAL, res->to_json().dump());
+            }
+            auto * tts_res = dynamic_cast<server_task_result_tts *>(res.get());
+            if (tts_res == nullptr) {
+                return grpc::Status(grpc::StatusCode::INTERNAL, "unexpected result type for a TTS task");
+            }
+
+            if (!tts_res->audio.empty()) {
+                backend::Reply chunk;
+                chunk.set_audio(tts_pcm_f32_to_s16(tts_res->audio));
+                if (!writer->Write(chunk)) {
+                    return grpc::Status(grpc::StatusCode::CANCELLED, "client closed the TTS stream");
+                }
+            }
+
+            if (tts_res->final) {
+                break;
+            }
+        }
+
+        return grpc::Status::OK;
+    }
+#else
+    grpc::Status TTS(ServerContext* context, const backend::TTSRequest* request, backend::Result* result) override {
+        auto auth = checkAuth(context);
+        if (!auth.ok()) return auth;
+        (void) request;
+        (void) result;
+        return grpc::Status(grpc::StatusCode::UNIMPLEMENTED,
+            "TTS is unavailable in this llama.cpp fork backend");
+    }
+
+    grpc::Status TTSStream(ServerContext* context, const backend::TTSRequest* request, grpc::ServerWriter<backend::Reply>* writer) override {
+        auto auth = checkAuth(context);
+        if (!auth.ok()) return auth;
+        (void) request;
+        (void) writer;
+        return grpc::Status(grpc::StatusCode::UNIMPLEMENTED,
+            "TTSStream is unavailable in this llama.cpp fork backend");
+    }
+#endif
+
     // Score returns the model's joint log-probability of each candidate
     // continuation given a shared prompt.
     //

backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch

@@ -0,0 +1,895 @@
+# Carries the server-side half of ggml-org/llama.cpp#26603 (POST /tts), which
+# adds SERVER_TASK_TYPE_TTS to the shared server_context. LocalAI's gRPC
+# adapter rides on that same server_context and cannot drive the mtmd gen-audio
+# loop directly: server_context owns the llama_context and schedules slots on
+# its own thread, so a handler calling llama_decode would corrupt state.
+#
+# The upstream HTTP route (tools/server/server.cpp) and the README/CLI hunks are
+# intentionally excluded - LocalAI does not serve llama.cpp's HTTP surface.
+#
+# Rebased against LLAMA_VERSION 9de0fcf2b with 0001-add-server-task-type-score.patch
+# already applied: the SERVER_TASK_TYPE_TTS case in the tokenize switch lands
+# after the SCORE case that patch adds, so this patch must stay second in ls order.
+#
+# Three fixes on top of the draft, all ours, all candidates to send back to #26603:
+#
+#  1. Its lone SRV_WRN call passes only a format string, but the macro expands
+#     __VA_ARGS__ without the GNU comma-elision extension, so the expansion ends
+#     in a trailing comma and does not compile. The "%s" wrapper added here is the
+#     same idiom upstream already uses for its other argument-less SRV_WRN calls;
+#     drop it if the draft fixes the call before it merges.
+#
+#  2. The slot.prompt_clear() added to the SERVER_TASK_TYPE_TTS branch of
+#     launch_slot_with_task. Without it only the FIRST TTS request in a server
+#     process succeeds and every later one fails instantly in step_prompt. TTS
+#     slots never enter the shared batch (pre_decode() returns early for them and
+#     process_tts_slots() drives them), so they skip the prompt-cache bookkeeping
+#     that would otherwise clear the sequence between requests; meanwhile the
+#     gen-audio pipeline always decodes from position 0 and its reset() only
+#     clears host-side buffers, never the KV cache. The result is that request 2
+#     decodes over request 1's tokens. This one is a genuine defect in the draft,
+#     not a LocalAI integration artifact, and should be reported upstream.
+#
+#  3. The is_supported() guard in server_slot::tts_ctx::reset(). The draft calls
+#     mtmd_helper_gen_audio_reset() unconditionally, but it only ever init()s the
+#     gen-audio pipeline for models that carry a gen-audio mmproj, so the handle
+#     stays null for every ordinary model. Upstream's implementation
+#     (tools/mtmd/mtmd-helper-gen.cpp) reads ctx->pipeline before it null-checks
+#     anything, so the slot-init call to server_slot::reset() segfaults the
+#     process on any non-TTS model - which is every chat model LocalAI loads
+#     through this backend. The missing null check on mtmd_helper_gen_audio_*
+#     is upstream's bug, so carry this guard until they add one, even if the
+#     draft's own reset() call is fixed.
+#
+# REMOVE THIS PATCH once #26603 merges upstream and LLAMA_VERSION is bumped past
+# the merge commit. It exists only because that PR is still a draft. If it merges
+# without fixes 2 and 3 above, those hunks still need carrying.
+diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
+index b52dc8e..fd9d6ca 100644
+--- a/tools/mtmd/mtmd-helper-gen.cpp
++++ b/tools/mtmd/mtmd-helper-gen.cpp
+@@ -48,29 +48,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
+     return LLAMA_TOKEN_NULL;
+ }
+ 
++static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
++    const char * c = (const char *) p;
++    buf.insert(buf.end(), c, c + n);
++}
++
++// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
++static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
++    const uint32_t riff_sz   = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
++    const uint32_t fmt_sz    = 16, byte_rate = (uint32_t) rate * 2;
++    const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
++    const uint32_t rate32    = (uint32_t) rate;
++    put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
++    put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
++    put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
++    put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
++    put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
++}
++
++static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
++    for (size_t i = 0; i < n; i++) {
++        int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
++        put_bytes(buf, &s, 2);
++    }
++}
++
+ static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
+     // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
+     if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
+         return false;
+     }
+-    const uint32_t data_sz   = (uint32_t) (pcm.size() * 2);
+-    const uint32_t riff_sz   = 36 + data_sz;
+-    const uint32_t fmt_sz    = 16, byte_rate = (uint32_t) rate * 2;
+-    const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
+-    const uint32_t rate32    = (uint32_t) rate;
+-    auto put = [&](const void * p, size_t n) {
+-        const char * c = (const char *) p;
+-        buf.insert(buf.end(), c, c + n);
+-    };
+-    put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4);
+-    put("fmt ", 4); put(&fmt_sz, 4);
+-    put(&fmt, 2); put(&ch, 2); put(&rate32, 4);
+-    put(&byte_rate, 4); put(&align, 2); put(&bits, 2);
+-    put("data", 4); put(&data_sz, 4);
+-    for (float v : pcm) {
+-        int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);
+-        put(&s, 2);
+-    }
++    write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
++    append_wav16_pcm(buf, pcm.data(), pcm.size());
+     return true;
+ }
+ 
+@@ -89,6 +98,8 @@ public:
+     // those read what they need from h_state_in instead
+     virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0;
+     virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
++    // forces any buffered codes through code2wav now, regardless of window_frames
++    virtual int32_t flush() = 0;
+ 
+ protected:
+     llama_context * lctx;
+@@ -119,6 +130,9 @@ public:
+         prompt_batch.reset();
+         n_prompt = 0;
+         prompt_pos = 0;
++        stream = false;
++        pcm_sent = 0;
++        wav_header_sent = false;
+     }
+ 
+     int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
+@@ -204,6 +218,7 @@ public:
+         top_k = inp->top_k > 0 ? inp->top_k : 50;
+         top_p = inp->top_p > 0 ? inp->top_p : 1.0f;
+         out_type = inp->out_type;
++        stream = inp->stream;
+ 
+         // the text stream keeps flowing during generation: after frame k, the input adds
+         // trailing text row k on top of the codes embedding, then tts_eos, then tts_pad
+@@ -289,31 +304,60 @@ public:
+     }
+ 
+     int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
+-        if (!flush_gen_wav()) {
+-            return 1;
++        *out_sample_rate = info.sample_rate;
++
++        if (!stream) {
++            // one-shot call: force out whatever's left, regardless of window_frames
++            if (!flush_gen_wav()) {
++                return 1;
++            }
++            if (out_n_samples) {
++                *out_n_samples = (int64_t) audio_pcm.size();
++            }
++            if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
++                *out_data     = (const char *) audio_pcm.data();
++                *out_data_len = audio_pcm.size() * sizeof(float);
++                return 0;
++            }
++            out_buf.clear();
++            if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
++                LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
++                return 1;
++            }
++            *out_data     = out_buf.data();
++            *out_data_len = out_buf.size();
++            return 0;
+         }
+ 
+-        *out_sample_rate = info.sample_rate;
++        // streaming: only return audio produced since the previous call
++        const size_t n_new = audio_pcm.size() - pcm_sent;
+         if (out_n_samples) {
+-            *out_n_samples = (int64_t) audio_pcm.size();
++            *out_n_samples = (int64_t) n_new;
+         }
+ 
+         if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+-            *out_data     = (const char *) audio_pcm.data();
+-            *out_data_len = audio_pcm.size() * sizeof(float);
++            *out_data     = (const char *) (audio_pcm.data() + pcm_sent);
++            *out_data_len = n_new * sizeof(float);
++            pcm_sent = audio_pcm.size();
+             return 0;
+         }
+ 
+         out_buf.clear();
+-        if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+-            LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+-            return 1;
++        if (!wav_header_sent) {
++            write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
++            wav_header_sent = true;
+         }
++        append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
++        pcm_sent = audio_pcm.size();
+         *out_data     = out_buf.data();
+         *out_data_len = out_buf.size();
+         return 0;
+     }
+ 
++    int32_t flush() override {
++        return flush_gen_wav() ? 0 : 1;
++    }
++
+ private:
+     bool ensure_cache() {
+         if (specials_ok) {
+@@ -357,7 +401,7 @@ private:
+             LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
+             return false;
+         }
+-        const std::string  marker = mtmd_default_marker();
++        const std::string  marker = mtmd_get_marker(mctx);
+         mtmd_input_text     text{ marker.c_str(), marker.size(), false, true };
+         mtmd_input_chunks * chunks = mtmd_input_chunks_init();
+         const mtmd_bitmap * bptr = bitmap;
+@@ -442,6 +486,9 @@ private:
+     std::vector<float> h_state_buf;
+     mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+     std::vector<char> out_buf;
++    bool   stream          = false;
++    size_t pcm_sent        = 0; // samples already returned by get_output()
++    bool   wav_header_sent = false;
+ };
+ 
+ static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
+@@ -473,6 +520,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
+     }
+ }
+ 
++struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
++    mtmd_helper_gen_audio_inp inp{};
++    inp.top_k    = 50;
++    inp.top_p    = 1.0f;
++    inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
++    return inp;
++}
++
+ int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
+     if (!ctx->pipeline) {
+         LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
+@@ -503,3 +558,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
+     }
+     return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
+ }
++
++int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
++    if (!ctx->pipeline) {
++        return 1;
++    }
++    return ctx->pipeline->flush();
++}
+diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
+index 7e5cf9b..1f3ec01 100644
+--- a/tools/mtmd/mtmd-helper.h
++++ b/tools/mtmd/mtmd-helper.h
+@@ -175,6 +175,7 @@ enum mtmd_helper_gen_audio_outtype {
+     MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
+ };
+ struct mtmd_helper_gen_audio_inp {
++    bool stream; // if true, output() must be called after each step_gen()
+     llama_seq_id seq_id;
+ 
+     const char * prompt;
+@@ -189,6 +190,8 @@ struct mtmd_helper_gen_audio_inp {
+     enum mtmd_helper_gen_audio_outtype out_type;
+ };
+ 
++MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
++
+ MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
+                                     struct llama_context * lctx,
+                                     struct mtmd_context * mctx);
+@@ -217,6 +220,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
+ 
+ // out_data valid until next get_output() or reset() call
+ // out_n_samples (optional, can be NULL) receives the number of generated PCM samples
++// if inp->stream is true: returns only audio produced since the previous call, and
++// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
+ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
+                         mtmd_helper_gen_audio * ctx,
+                         int32_t * out_sample_rate,
+@@ -224,6 +229,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
+                         size_t * out_data_len,
+                         int64_t * out_n_samples);
+ 
++// forces any buffered codes through code2wav now, regardless of window_frames;
++// call once when generation has ended, before the last get_output() in stream mode
++MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
++
+ #ifdef __cplusplus
+ } // extern "C"
+ #endif
+@@ -250,8 +259,41 @@ struct mtmd_helper_gen_audio_deleter {
+ };
+ using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;
+ struct gen_audio {
++
++    // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp
++    struct inp {
++        mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
++        std::string prompt_str;
++        std::string lang_str;
++        mtmd::bitmap_ptr speaker_ref_ptr;
++
++        inp()                             = default;
++        inp(inp &&)                       = default;
++        inp & operator=(inp &&)           = default;
++        inp(const inp &)                  = delete;
++        inp & operator=(const inp &)      = delete;
++
++        void set_prompt     (std::string p)        { prompt_str = std::move(p); }
++        void set_lang       (std::string l)        { lang_str   = std::move(l); }
++        void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
++
++        // pointers are only valid as long as *this is alive
++        const mtmd_helper_gen_audio_inp * get() {
++            data.prompt      = prompt_str.c_str();
++            data.prompt_len  = prompt_str.size();
++            data.lang        = lang_str.empty() ? nullptr : lang_str.c_str();
++            data.speaker_ref = speaker_ref_ptr.get();
++            return &data;
++        }
++    };
++
+     gen_audio_ptr ctx;
+-    gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {}
++    void init(struct llama_context * lctx, struct mtmd_context * mctx) {
++        ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));
++    }
++    bool valid() const {
++        return ctx.get() != nullptr;
++    }
+     void reset() {
+         mtmd_helper_gen_audio_reset(ctx.get());
+     }
+@@ -267,6 +309,9 @@ struct gen_audio {
+     int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
+         return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
+     }
++    int32_t flush() {
++        return mtmd_helper_gen_audio_flush(ctx.get());
++    }
+ };
+ 
+ } // namespace mtmd_helper
+diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
+index 543dc43..4f01b1e 100644
+--- a/tools/server/server-context.cpp
++++ b/tools/server/server-context.cpp
+@@ -16,6 +16,7 @@
+ #include "speculative.h"
+ #include "mtmd.h"
+ #include "mtmd-helper.h"
++#include "base64.hpp"
+ 
+ #include <algorithm>
+ #include <cstddef>
+@@ -43,7 +44,8 @@ static uint32_t server_n_outputs_max(const common_params & params) {
+     const uint32_t n_batch  = params.n_batch;
+ 
+     if (params.embedding ||
+-            (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
++            (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE) ||
++            !params.mmproj.path.empty()) { // gen-audio (TTS) capability isn't known until the mmproj loads, size generously
+         return n_batch;
+     }
+ 
+@@ -214,6 +216,30 @@ struct server_slot {
+     mtmd_context * mctx = nullptr;
+     mtmd::batch_ptr mbatch = nullptr;
+ 
++    struct tts_ctx {
++        mtmd_helper::gen_audio ctx;
++        const float * h_state;
++        llama_token   sampled;
++        int32_t       n_decoded;
++        bool is_supported() const {
++            return ctx.valid();
++        }
++        void reset() {
++            // mtmd_helper_gen_audio_reset() dereferences its argument before it
++            // null-checks the pipeline, and the pipeline is only allocated for
++            // models that actually carry a gen-audio mmproj. server_slot::reset()
++            // runs for every slot of every model, so without this guard any
++            // non-TTS model segfaults during slot initialization.
++            if (is_supported()) {
++                ctx.reset();
++            }
++            h_state   = nullptr;
++            sampled   = LLAMA_TOKEN_NULL;
++            n_decoded = 0;
++        }
++    };
++    tts_ctx tts;
++
+     // speculative decoding
+     common_speculative * spec;
+ 
+@@ -403,6 +429,8 @@ struct server_slot {
+ 
+         // clear multimodal state
+         mbatch.reset();
++
++        tts.reset();
+     }
+ 
+     void init_sampler() const {
+@@ -949,6 +977,14 @@ public:
+     mtmd_context * mctx = nullptr;
+     const llama_vocab * vocab = nullptr;
+ 
++    bool has_cap_tts() const {
++        return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
++    }
++
++    bool has_cap_chat() const {
++        return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
++    }
++
+     server_queue    queue_tasks;
+     server_response queue_results;
+ 
+@@ -1400,6 +1436,10 @@ private:
+             slot.mctx                   = mctx;
+             slot.prompt.tokens.has_mtmd = mctx != nullptr;
+ 
++            if (has_cap_tts()) {
++                slot.tts.ctx.init(ctx_tgt, mctx);
++            }
++
+             SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
+ 
+             slot.callback_on_release = [this](int id_slot) {
+@@ -1853,6 +1893,28 @@ private:
+ 
+         SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
+ 
++        if (task.type == SERVER_TASK_TYPE_TTS) {
++            GGML_ASSERT(has_cap_tts()); // should already checked in route handler
++            if (!slot.tts.is_supported()) {
++                slot.tts.ctx.init(ctx_tgt, slot.mctx);
++            }
++
++            // TTS slots never enter the shared batch: pre_decode() returns early for
++            // them and process_tts_slots() drives them instead, so they skip the
++            // prompt-cache bookkeeping that clears this sequence between requests.
++            // The gen-audio pipeline always decodes from position 0, and its own
++            // reset() only clears host-side buffers, so without this the second and
++            // later tasks on a slot decode over the previous request's tokens and
++            // step_prompt() fails immediately.
++            slot.prompt_clear();
++
++            task.tts_inp.data.seq_id = slot.id;
++            if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {
++                send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);
++                return false;
++            }
++        }
++
+         // initialize samplers
+         if (task.need_sampling()) {
+             try {
+@@ -1875,6 +1937,9 @@ private:
+             // TODO: getting pre sampling logits is not yet supported with backend sampling
+             backend_sampling &= !need_pre_sample_logits;
+ 
++            // TODO: check verify if this actually works with TTS
++            backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
++
+             // TODO: tmp until backend sampling is fully implemented
+             if (backend_sampling) {
+                 llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
+@@ -1890,9 +1955,13 @@ private:
+ 
+         slot.task = std::make_unique<const server_task>(std::move(task));
+ 
+-        slot.state = slot.task->is_child()
+-            ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
+-            : SLOT_STATE_STARTED;
++        if (slot.task->type == SERVER_TASK_TYPE_TTS) {
++            slot.state = SLOT_STATE_PROCESSING_PROMPT;
++        } else {
++            slot.state = slot.task->is_child()
++                ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
++                : SLOT_STATE_STARTED;
++        }
+ 
+         // reset server kill-switch counter
+         n_empty_consecutive = 0;
+@@ -2169,6 +2238,18 @@ private:
+         queue_results.send(std::move(res));
+     }
+ 
++    void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {
++        auto res = std::make_unique<server_task_result_tts>();
++
++        res->id          = slot.task->id;
++        res->index       = slot.task->index;
++        res->sample_rate = sample_rate;
++        res->audio.assign(data, data_len);
++        res->final       = final;
++
++        queue_results.send(std::move(res));
++    }
++
+     void send_final_response(server_slot & slot) {
+         auto res = std::make_unique<server_task_result_cmpl_final>();
+ 
+@@ -2668,6 +2749,7 @@ private:
+             case SERVER_TASK_TYPE_EMBEDDING:
+             case SERVER_TASK_TYPE_RERANK:
+             case SERVER_TASK_TYPE_SCORE:
++            case SERVER_TASK_TYPE_TTS:
+                 {
+                     // special case: if input is provided via CLI, tokenize it first
+                     // otherwise, no need to tokenize as it's already done inside the HTTP thread
+@@ -3103,6 +3185,14 @@ private:
+             abort_all_slots("pre_decode() failed: " + std::string(e.what()));
+         }
+ 
++        // note: TTS slots bypass the shared batch entirely
++        try {
++            process_tts_slots();
++        } catch (const std::exception & e) {
++            SRV_ERR("process_tts_slots() failed: %s\n", e.what());
++            abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
++        }
++
+         GGML_ASSERT(batch.slot_batched || batch.size() == 0);
+ 
+         if (batch.slot_batched) {
+@@ -3173,10 +3263,77 @@ private:
+         }
+     }
+ 
++    void process_tts_slots() {
++        iterate(slots, [&](server_slot & slot) {
++            if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
++                return;
++            }
++
++            llama_set_embeddings(ctx_tgt, true);
++
++            if (slot.state == SLOT_STATE_PROCESSING_PROMPT) {
++                const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt));
++                if (ret < 0) {
++                    send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER);
++                    slot.release();
++                } else if (ret == 0) {
++                    slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
++                    common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
++                    slot.tts.h_state = llama_get_embeddings_ith(ctx_tgt, -1);
++                    slot.state = SLOT_STATE_GENERATING;
++                }
++                return;
++            }
++
++            const int32_t n_predict = slot.task->params.n_predict > 0 ? slot.task->params.n_predict : 512;
++            if (slot.tts.n_decoded >= n_predict || llama_vocab_is_eog(vocab, slot.tts.sampled)) {
++                int32_t      sample_rate = 0;
++                const char * data        = nullptr;
++                size_t       data_len    = 0;
++                // generation truly ends here: force out any sub-window remainder still buffered
++                if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
++                    send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER);
++                } else {
++                    send_tts_result(slot, sample_rate, data, data_len, true);
++                }
++                slot.release();
++                return;
++            }
++
++            const float * h_state_next = nullptr;
++            if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next) != 0) {
++                send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER);
++                slot.release();
++                return;
++            }
++            slot.tts.h_state = h_state_next;
++            slot.tts.n_decoded++;
++
++            slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
++            common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
++
++            if (slot.task->params.stream) {
++                int32_t      sample_rate = 0;
++                const char * data        = nullptr;
++                size_t       data_len    = 0;
++                if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
++                    send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER);
++                    slot.release();
++                } else if (data_len > 0) {
++                    send_tts_result(slot, sample_rate, data, data_len, false);
++                }
++            }
++        });
++    }
++
+     void pre_decode() {
+         // apply context-shift if needed
+         // TODO: simplify and improve
+         iterate(slots, [&](server_slot & slot) {
++            if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
++                // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
++                return;
++            }
+             if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
+                 if (!params_base.ctx_shift) {
+                     // this check is redundant (for good)
+@@ -3249,7 +3406,7 @@ private:
+ 
+         // determine which slots are generating and drafting
+         iterate(slots, [&](server_slot & slot) {
+-            if (slot.state != SLOT_STATE_GENERATING) {
++            if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
+                 return;
+             }
+ 
+@@ -3381,7 +3538,7 @@ private:
+                     return; // batch is full, skip remaining slots
+                 }
+ 
+-                if (!slot.is_processing()) {
++                if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
+                     return;
+                 }
+ 
+@@ -4390,6 +4547,8 @@ server_context_meta server_context::get_meta() const {
+         /* has_inp_image          */ impl->chat_params.allow_image,
+         /* has_inp_audio          */ impl->chat_params.allow_audio,
+         /* has_inp_video          */ impl->chat_params.allow_video,
++        /* has_cap_chat           */ impl->has_cap_chat(),
++        /* has_cap_tts            */ impl->has_cap_tts(),
+         /* json_ui_settings       */ impl->json_ui_settings,
+         /* slot_n_ctx             */ impl->get_slot_n_ctx(),
+         /* pooling_type           */ llama_pooling_type(impl->ctx_tgt),
+@@ -4469,6 +4628,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
+ 
+     res->set_req(&req); // will also set spipe if needed
+ 
++    if (!ctx_server.has_cap_chat()) {
++        res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));
++        return res;
++    }
++
+     int32_t sse_ping_interval = params.sse_ping_interval;
+ 
+     try {
+@@ -5446,6 +5610,150 @@ void server_routes::init_routes() {
+         return res;
+     };
+ 
++    this->post_tts = [this](const server_http_req & req) {
++        auto res = create_response();
++        res->set_req(&req); // will also set spipe if needed
++
++        if (!ctx_server.has_cap_tts()) {
++            res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
++            return res;
++        }
++
++        const json body = json::parse(req.body);
++
++        std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
++        if (prompt.empty()) {
++            res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
++            return res;
++        }
++
++        const std::string response_format = json_value(body, "response_format", std::string("wav"));
++        const bool stream = json_value(body, "stream", false);
++
++        server_task task(SERVER_TASK_TYPE_TTS);
++        task.tts_inp.set_prompt(prompt);
++        task.tts_inp.set_lang(json_value(body, "lang", std::string()));
++        task.tts_inp.data.top_k     = json_value(body, "top_k", 0);
++        task.tts_inp.data.top_p     = json_value(body, "top_p", 0.0f);
++        task.tts_inp.data.stream    = stream;
++        task.tts_inp.data.out_type  = response_format == "pcm"
++            ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
++            : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
++        task.params.stream    = stream;
++        task.params.n_predict = json_value(body, "n_predict", -1);
++        task.params.sampling  = params.sampling; // baseline defaults, then apply overrides below
++        task.params.sampling.penalty_repeat  = json_value(body, "repeat_penalty", 1.05f);
++        task.params.sampling.penalty_last_n  = -1;
++        if (task.tts_inp.data.top_k > 0) {
++            task.params.sampling.top_k = task.tts_inp.data.top_k;
++        }
++        if (task.tts_inp.data.top_p > 0) {
++            task.params.sampling.top_p = task.tts_inp.data.top_p;
++        }
++
++        // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
++        const unsigned char * speaker_ref_data = nullptr;
++        size_t speaker_ref_len  = 0;
++        std::string speaker_ref_b64_decoded;
++
++        auto speaker_ref_file = req.files.find("speaker_ref");
++        if (speaker_ref_file != req.files.end()) {
++            speaker_ref_data = speaker_ref_file->second.data.data();
++            speaker_ref_len  = speaker_ref_file->second.data.size();
++        } else {
++            std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
++            if (!speaker_ref_b64.empty()) {
++                speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
++                speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
++                speaker_ref_len  = speaker_ref_b64_decoded.size();
++            }
++        }
++
++        if (speaker_ref_len > 0) {
++            auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false);
++            if (!wrapper.bitmap) {
++                res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
++                return res;
++            }
++            task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
++        } else {
++            // SRV_WRN expands __VA_ARGS__ without the GNU comma-elision extension,
++            // so a bare format string leaves a trailing comma and will not compile
++            SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
++        }
++
++        auto & rd = res->rd;
++        task.id = rd.get_new_id();
++        rd.post_task(std::move(task));
++
++        const std::string content_type = response_format == "pcm" ? "audio/L16" : "audio/wav";
++
++        if (!stream) {
++            auto result = rd.next(req.should_stop);
++            if (!result) {
++                GGML_ASSERT(req.should_stop());
++                return res; // connection is closed
++            }
++            if (result->is_error()) {
++                res->error(result->to_json());
++                return res;
++            }
++            auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
++            GGML_ASSERT(tts_res != nullptr);
++            res->status       = 200;
++            res->content_type = content_type;
++            res->data         = std::move(tts_res->audio);
++            return res;
++        } else {
++            auto first_result = rd.next(req.should_stop);
++            if (!first_result) {
++                GGML_ASSERT(req.should_stop());
++                return res; // connection is closed
++            }
++            if (first_result->is_error()) {
++                res->error(first_result->to_json());
++                return res;
++            }
++            auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
++            GGML_ASSERT(first_tts_res != nullptr);
++
++            res->status       = 200;
++            res->content_type = content_type;
++            res->data          = std::move(first_tts_res->audio);
++            bool is_done       = first_tts_res->final;
++
++            res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
++                if (is_done) {
++                    return false;
++                }
++                if (res_this->should_stop()) {
++                    return false;
++                }
++                if (!res_this->data.empty()) {
++                    output = std::move(res_this->data);
++                    res_this->data.clear();
++                    return true;
++                }
++
++                server_response_reader & rd = res_this->rd;
++                if (!rd.has_next()) {
++                    return false;
++                }
++                auto result = rd.next([&res_this]() { return res_this->should_stop(); });
++                if (!result || result->is_error()) {
++                    return false;
++                }
++                auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
++                GGML_ASSERT(tts_res != nullptr);
++                output  = std::move(tts_res->audio);
++                is_done = tts_res->final;
++                return true;
++            });
++        }
++
++        return res;
++    };
++
+     this->get_lora_adapters = [this](const server_http_req & req) {
+         auto res = create_response();
+ 
+diff --git a/tools/server/server-context.h b/tools/server/server-context.h
+index f9ab113..6105126 100644
+--- a/tools/server/server-context.h
++++ b/tools/server/server-context.h
+@@ -22,6 +22,8 @@ struct server_context_meta {
+     bool has_inp_image;
+     bool has_inp_audio;
+     bool has_inp_video;
++    bool has_cap_chat;
++    bool has_cap_tts;
+     json json_ui_settings;
+     int slot_n_ctx;
+     enum llama_pooling_type pooling_type;
+@@ -151,6 +153,7 @@ struct server_routes {
+     server_http_context::handler_t post_embeddings;
+     server_http_context::handler_t post_embeddings_oai;
+     server_http_context::handler_t post_rerank;
++    server_http_context::handler_t post_tts;
+     server_http_context::handler_t get_lora_adapters;
+     server_http_context::handler_t post_lora_adapters;
+ 
+diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
+index 1ee6775..939630b 100644
+--- a/tools/server/server-task.cpp
++++ b/tools/server/server-task.cpp
+@@ -1523,6 +1523,17 @@ json server_task_result_rerank::to_json() {
+     };
+ }
+ 
++//
++// server_task_result_tts
++//
++json server_task_result_tts::to_json() {
++    return json {
++        {"sample_rate", sample_rate},
++        {"n_bytes",     audio.size()},
++        {"final",       final},
++    };
++}
++
+ //
+ // server_task_result_error
+ //
+diff --git a/tools/server/server-task.h b/tools/server/server-task.h
+index 5bedf19..e6ca67a 100644
+--- a/tools/server/server-task.h
++++ b/tools/server/server-task.h
+@@ -10,6 +10,7 @@
+ 
+ // TODO: prevent including the whole server-common.h as we only use server_tokens
+ #include "server-common.h"
++#include "mtmd-helper.h"
+ 
+ using json = nlohmann::ordered_json;
+ 
+@@ -42,6 +43,7 @@ enum server_task_type {
+     SERVER_TASK_TYPE_SLOT_ERASE,
+     SERVER_TASK_TYPE_GET_LORA,
+     SERVER_TASK_TYPE_SET_LORA,
++    SERVER_TASK_TYPE_TTS,
+ };
+ 
+ // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
+@@ -202,6 +204,9 @@ struct server_task {
+     // used by SERVER_TASK_TYPE_SET_LORA
+     std::map<int, float> set_lora; // mapping adapter ID -> scale
+ 
++    // used by SERVER_TASK_TYPE_TTS
++    mtmd_helper::gen_audio::inp tts_inp;
++
+     server_task() = default;
+ 
+     server_task(server_task_type type) : type(type) {}
+@@ -235,6 +240,7 @@ struct server_task {
+         switch (type) {
+             case SERVER_TASK_TYPE_COMPLETION:
+             case SERVER_TASK_TYPE_INFILL:
++            case SERVER_TASK_TYPE_TTS:
+                 return true;
+             default:
+                 return false;
+@@ -514,6 +520,16 @@ struct server_task_result_embd : server_task_result {
+     json to_json_oaicompat();
+ };
+ 
++struct server_task_result_tts : server_task_result {
++    std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
++    int32_t     sample_rate = 0;
++    bool        final = false; // true for the last chunk of a request
++
++    virtual bool is_stop() override { return final; }
++
++    virtual json to_json() override;
++};
++
+ struct server_task_result_rerank : server_task_result {
+     float score = -1e6;
+ 

backend/cpp/llama-cpp/prepare.sh

@@ -28,6 +28,10 @@ cp -r message_content_test.cpp llama.cpp/tools/grpc-server/
 # Generic passthrough parser staging and its standalone regression test.
 cp -r passthrough_options.h llama.cpp/tools/grpc-server/
 cp -r passthrough_options_test.cpp llama.cpp/tools/grpc-server/
+# TTS request validation (included by grpc-server.cpp) and its standalone
+# regression test.
+cp -r tts_request_options.h llama.cpp/tools/grpc-server/
+cp -r tts_request_options_test.cpp llama.cpp/tools/grpc-server/
 # Parent-death watcher (included by grpc-server.cpp) and its standalone unit
 # test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest).
 cp -r parent_watch.h llama.cpp/tools/grpc-server/

backend/cpp/llama-cpp/tts_request_options.h

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: MIT
+
+#pragma once
+
+#include <cstdint>
+#include <exception>
+#include <map>
+#include <string>
+
+namespace llama_grpc {
+
+// Validated, parsed form of a backend::TTSRequest, kept free of llama.cpp,
+// mtmd and gRPC headers so backend/cpp/run-unit-tests.sh can compile it as a
+// standalone translation unit. grpc-server.cpp turns this into a
+// mtmd_helper::gen_audio::inp.
+struct tts_request_options {
+    bool        ok = false;
+    std::string error;
+
+    std::string text;
+    std::string voice_path;
+    std::string language;
+
+    // 0 / 0.0f mean "unset": upstream only overrides the sampler defaults when
+    // the value is strictly positive.
+    int32_t top_k = 0;
+    float   top_p = 0.0f;
+
+    // Upper bound on generated audio frames, exposed because the model does not
+    // always emit its codec EOS and will otherwise run to the 512-frame default,
+    // which is roughly 41 s at the 12.5 Hz frame rate. 0 means unset, leaving
+    // that default in place.
+    int32_t max_frames = 0;
+};
+
+namespace detail {
+
+// Strict whole-string numeric parsing. std::stoi/stof accept trailing garbage
+// ("40abc" -> 40), which would silently honour a typo'd request.
+inline bool parse_whole_int32(const std::string & value, int32_t & out) {
+    if (value.empty()) {
+        return false;
+    }
+    try {
+        size_t     consumed = 0;
+        const long parsed   = std::stol(value, &consumed);
+        if (consumed != value.size()) {
+            return false;
+        }
+        if (parsed < INT32_MIN || parsed > INT32_MAX) {
+            return false;
+        }
+        out = static_cast<int32_t>(parsed);
+        return true;
+    } catch (const std::exception &) {
+        return false;
+    }
+}
+
+inline bool parse_whole_float(const std::string & value, float & out) {
+    if (value.empty()) {
+        return false;
+    }
+    try {
+        size_t      consumed = 0;
+        const float parsed   = std::stof(value, &consumed);
+        if (consumed != value.size()) {
+            return false;
+        }
+        out = parsed;
+        return true;
+    } catch (const std::exception &) {
+        return false;
+    }
+}
+
+inline tts_request_options reject(const std::string & message) {
+    tts_request_options opts;
+    opts.ok    = false;
+    opts.error = message;
+    return opts;
+}
+
+} // namespace detail
+
+inline tts_request_options parse_tts_request_options(
+        const std::string & text,
+        const std::string & voice,
+        const std::string & language,
+        const std::map<std::string, std::string> & params) {
+    if (text.empty()) {
+        return detail::reject("text must be a non-empty string");
+    }
+
+    // The Qwen3-TTS Base checkpoints have no built-in speaker. Without a
+    // reference clip the model picks an arbitrary voice, so an unset voice is
+    // a request error rather than a defaulted one.
+    if (voice.empty()) {
+        return detail::reject("voice must name a speaker reference audio file");
+    }
+
+    tts_request_options opts;
+    opts.text       = text;
+    opts.voice_path = voice;
+    opts.language   = language;
+
+    // Both values are range-checked here rather than left to the caller: the
+    // consumer copies them straight into mtmd_helper::gen_audio::inp, and only
+    // its separate sampler assignment is guarded by "> 0". An out-of-range or
+    // non-finite value would slip past that guard and reach llama.cpp.
+    const auto top_k_it = params.find("top_k");
+    if (top_k_it != params.end()) {
+        if (!detail::parse_whole_int32(top_k_it->second, opts.top_k)) {
+            return detail::reject("top_k must be an integer, got \"" + top_k_it->second + "\"");
+        }
+        if (opts.top_k < 0) {
+            return detail::reject("top_k must be >= 0, got \"" + top_k_it->second + "\"");
+        }
+    }
+
+    const auto top_p_it = params.find("top_p");
+    if (top_p_it != params.end()) {
+        if (!detail::parse_whole_float(top_p_it->second, opts.top_p)) {
+            return detail::reject("top_p must be a number, got \"" + top_p_it->second + "\"");
+        }
+        // Phrased as a negated in-range test, not "p < 0.0f || p > 1.0f",
+        // because every comparison against NaN is false: the obvious form
+        // would accept NaN, and NaN then defeats the consumer's "> 0" guard
+        // too, since that comparison is false as well.
+        if (!(opts.top_p >= 0.0f && opts.top_p <= 1.0f)) {
+            return detail::reject("top_p must be between 0.0 and 1.0, got \"" + top_p_it->second + "\"");
+        }
+    }
+
+    const auto max_frames_it = params.find("max_frames");
+    if (max_frames_it != params.end()) {
+        if (!detail::parse_whole_int32(max_frames_it->second, opts.max_frames)) {
+            return detail::reject("max_frames must be an integer, got \"" + max_frames_it->second + "\"");
+        }
+        if (opts.max_frames < 0) {
+            return detail::reject("max_frames must be >= 0, got \"" + max_frames_it->second + "\"");
+        }
+    }
+
+    opts.ok = true;
+    return opts;
+}
+
+} // namespace llama_grpc

backend/cpp/llama-cpp/tts_request_options_test.cpp

@@ -0,0 +1,209 @@
+// SPDX-License-Identifier: MIT
+
+#include <cstdio>
+#include <map>
+#include <string>
+
+#include "tts_request_options.h"
+
+static int failures = 0;
+
+static void check(bool ok, const char * name) {
+    if (!ok) {
+        ++failures;
+        std::fprintf(stderr, "FAIL: %s\n", name);
+    }
+}
+
+static void test_accepts_a_minimal_valid_request() {
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "en", {});
+
+    check(opts.ok, "minimal request is accepted");
+    check(opts.error.empty(), "minimal request has no error");
+    check(opts.text == "Hello world", "text passes through");
+    check(opts.voice_path == "/models/voices/ref.wav", "voice path passes through");
+    check(opts.language == "en", "language passes through");
+    check(opts.top_k == 0, "top_k defaults to the unset sentinel");
+    check(opts.top_p == 0.0f, "top_p defaults to the unset sentinel");
+    check(opts.max_frames == 0, "max_frames defaults to the unset sentinel");
+}
+
+static void test_rejects_empty_text() {
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "", "/models/voices/ref.wav", "en", {});
+
+    check(!opts.ok, "empty text is rejected");
+    check(opts.error.find("text") != std::string::npos, "empty-text error names the field");
+}
+
+static void test_rejects_missing_speaker_reference() {
+    // Qwen3-TTS Base has no built-in speaker; without a reference it produces
+    // an arbitrary voice, so this must be a hard error rather than a surprise.
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "Hello world", "", "en", {});
+
+    check(!opts.ok, "missing voice is rejected");
+    check(opts.error.find("voice") != std::string::npos, "missing-voice error names the field");
+}
+
+static void test_parses_sampling_params() {
+    const std::map<std::string, std::string> params{
+        {"top_k", "40"},
+        {"top_p", "0.85"},
+    };
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", params);
+
+    check(opts.ok, "sampling params are accepted");
+    check(opts.top_k == 40, "top_k is parsed");
+    check(opts.top_p > 0.849f && opts.top_p < 0.851f, "top_p is parsed");
+    check(opts.language.empty(), "absent language stays empty");
+}
+
+static void test_rejects_malformed_sampling_params() {
+    const auto bad_top_k = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "forty"}});
+    check(!bad_top_k.ok, "non-numeric top_k is rejected");
+    check(bad_top_k.error.find("top_k") != std::string::npos, "top_k error names the field");
+
+    const auto bad_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", ""}});
+    check(!bad_top_p.ok, "empty top_p is rejected");
+
+    const auto trailing = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "40abc"}});
+    check(!trailing.ok, "top_k with trailing garbage is rejected");
+
+    const auto trailing_float = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "0.8abc"}});
+    check(!trailing_float.ok, "top_p with trailing garbage is rejected");
+
+    // std::stol returns a long, which is wider than int32_t on 64-bit hosts, so
+    // an in-range-for-long value still has to be caught before the narrowing.
+    const auto overflow_top_k = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "99999999999"}});
+    check(!overflow_top_k.ok, "top_k beyond int32 range is rejected");
+    check(overflow_top_k.error.find("top_k") != std::string::npos,
+          "top_k overflow error names the field");
+}
+
+static void test_rejects_out_of_range_sampling_params() {
+    // These reach mtmd_helper::gen_audio::inp unconditionally downstream, where
+    // the "> 0" sampler guard does not screen them, so they must die here.
+    const auto negative_top_k = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "-5"}});
+    check(!negative_top_k.ok, "negative top_k is rejected");
+    check(negative_top_k.error.find("top_k") != std::string::npos,
+          "negative top_k error names the field");
+
+    const auto negative_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "-0.1"}});
+    check(!negative_top_p.ok, "negative top_p is rejected");
+    check(negative_top_p.error.find("top_p") != std::string::npos,
+          "negative top_p error names the field");
+
+    const auto large_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "1.5"}});
+    check(!large_top_p.ok, "top_p above 1.0 is rejected");
+
+    // NaN survives a naive "p < 0.0f || p > 1.0f" range test because every
+    // comparison against NaN is false. This case pins the correct form.
+    const auto nan_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "nan"}});
+    check(!nan_top_p.ok, "NaN top_p is rejected");
+
+    const auto inf_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "inf"}});
+    check(!inf_top_p.ok, "infinite top_p is rejected");
+}
+
+static void test_accepts_sampling_param_boundaries() {
+    const auto zero_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "0.0"}});
+    check(zero_top_p.ok, "top_p of 0.0 is accepted");
+    check(zero_top_p.top_p == 0.0f, "top_p of 0.0 round-trips");
+
+    const auto one_top_p = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_p", "1.0"}});
+    check(one_top_p.ok, "top_p of 1.0 is accepted");
+    check(one_top_p.top_p == 1.0f, "top_p of 1.0 round-trips");
+
+    const auto zero_top_k = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "0"}});
+    check(zero_top_k.ok, "top_k of 0 is accepted");
+}
+
+static void test_parses_max_frames() {
+    // The consumer maps a positive value onto n_predict and leaves upstream's
+    // 512-frame default in place when it is unset, so the sentinel matters as
+    // much as the parsed value.
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "120"}});
+
+    check(opts.ok, "max_frames is accepted");
+    check(opts.max_frames == 120, "max_frames is parsed");
+
+    const auto absent = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"top_k", "40"}});
+    check(absent.ok, "a request without max_frames is accepted");
+    check(absent.max_frames == 0, "absent max_frames leaves the unset sentinel");
+
+    const auto zero = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "0"}});
+    check(zero.ok, "max_frames of 0 is accepted");
+    check(zero.max_frames == 0, "max_frames of 0 means unset");
+}
+
+static void test_rejects_malformed_max_frames() {
+    const auto negative = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "-1"}});
+    check(!negative.ok, "negative max_frames is rejected");
+    check(negative.error.find("max_frames") != std::string::npos,
+          "negative max_frames error names the field");
+
+    const auto non_numeric = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "many"}});
+    check(!non_numeric.ok, "non-numeric max_frames is rejected");
+    check(non_numeric.error.find("max_frames") != std::string::npos,
+          "non-numeric max_frames error names the field");
+
+    const auto trailing = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "120abc"}});
+    check(!trailing.ok, "max_frames with trailing garbage is rejected");
+
+    const auto empty = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", ""}});
+    check(!empty.ok, "empty max_frames is rejected");
+
+    const auto overflow = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"max_frames", "99999999999"}});
+    check(!overflow.ok, "max_frames beyond int32 range is rejected");
+}
+
+static void test_ignores_unknown_params() {
+    // Unknown keys are backend-specific knobs meant for other TTS engines. A
+    // request routed here must not fail just because it carries them.
+    const auto opts = llama_grpc::parse_tts_request_options(
+        "Hello world", "/models/voices/ref.wav", "", {{"exaggeration", "0.7"}});
+
+    check(opts.ok, "unknown params are ignored, not rejected");
+}
+
+int main() {
+    test_accepts_a_minimal_valid_request();
+    test_rejects_empty_text();
+    test_rejects_missing_speaker_reference();
+    test_parses_sampling_params();
+    test_rejects_malformed_sampling_params();
+    test_rejects_out_of_range_sampling_params();
+    test_accepts_sampling_param_boundaries();
+    test_parses_max_frames();
+    test_rejects_malformed_max_frames();
+    test_ignores_unknown_params();
+
+    if (failures == 0) {
+        std::printf("tts_request_options_test: all checks passed\n");
+    }
+    return failures;
+}

backend/cpp/turboquant/Makefile

@@ -48,6 +48,7 @@ define turboquant-build
 	# stays compiling against vanilla upstream.
 	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-$(1)-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-$(1)-build/grpc-server.cpp
+	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-$(1)-build/grpc-server.cpp
 	$(info $(GREEN)I turboquant build info:$(1)$(RESET))
 	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(TURBOQUANT_VERSION) \
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../turboquant-$(1)-build llama.cpp
@@ -86,6 +87,7 @@ turboquant-cpu-all:
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../turboquant-cpu-all-build purge
 	bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-cpu-all-build/grpc-server.cpp
 	bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-cpu-all-build/grpc-server.cpp
+	bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../turboquant-cpu-all-build/grpc-server.cpp
 	$(info $(GREEN)I turboquant build info:cpu-all-variants$(RESET))
 	LLAMA_REPO=$(LLAMA_REPO) LLAMA_VERSION=$(TURBOQUANT_VERSION) \
 	$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../turboquant-cpu-all-build llama.cpp

backend/index.yaml

@@ -11,6 +11,8 @@
     - https://github.com/ggerganov/llama.cpp
   tags:
     - text-to-text
+    - text-to-speech
+    - TTS
     - LLM
     - CPU
     - GPU

core/config/backend_capabilities.go

@@ -271,12 +271,18 @@ func referenceVoiceCloning() *VoiceCloningCapability {
 // Use NormalizeBackendName() for names with dots (e.g., "llama.cpp").
 var BackendCapabilities = map[string]BackendCapability{
 	// --- LLM / text generation backends ---
+	// llama.cpp also serves Qwen3-TTS, so TTS is in the union below. It is NOT
+	// in DefaultUsecases: a bare GGUF served by llama.cpp is a chat model, and
+	// the TTS models declare known_usecases: [tts]. VoiceCloning is likewise
+	// narrowed per model in VoiceCloningForModel, since the vast majority of
+	// llama-cpp models in the gallery are text LLMs that clone nothing.
 	"llama-cpp": {
-		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodEmbedding, MethodTokenizeString, MethodScore},
-		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseEdit, UsecaseEmbeddings, UsecaseTokenize, UsecaseVision, UsecaseScore},
+		GRPCMethods:      []GRPCMethod{MethodPredict, MethodPredictStream, MethodEmbedding, MethodTokenizeString, MethodScore, MethodTTS, MethodTTSStream},
+		PossibleUsecases: []string{UsecaseChat, UsecaseCompletion, UsecaseEdit, UsecaseEmbeddings, UsecaseTokenize, UsecaseVision, UsecaseScore, UsecaseTTS},
 		DefaultUsecases:  []string{UsecaseChat},
 		AcceptsImages:    true, // requires mmproj
-		Description:      "llama.cpp GGUF models — LLM inference with optional vision via mmproj",
+		VoiceCloning:     referenceVoiceCloning(),
+		Description:      "llama.cpp GGUF models: LLM inference with optional vision via mmproj, and Qwen3-TTS speech with reference-audio cloning",
 	},
 	// privacy-filter is the standalone GGML engine (backend/cpp/privacy-filter,
 	// wrapping privacy-filter.cpp) for the openai-privacy-filter PII/NER token
@@ -966,6 +972,20 @@ func VoiceCloningForModel(cfg *ModelConfig) *VoiceCloningCapability {
 		supported = strings.Contains(identity, "xtts") || strings.Contains(identity, "your_tts")
 	case "crispasr":
 		supported = strings.Contains(identity, "f5-tts") || strings.Contains(identity, "f5_tts")
+	case "llama-cpp":
+		// llama.cpp is overwhelmingly a text-LLM backend that happens to also
+		// serve Qwen3-TTS, so the permissive default below would advertise
+		// reference-audio cloning on every GGUF chat model in the gallery.
+		// Narrow on the declared usecase rather than the model name: the TTS
+		// checkpoints are the only llama-cpp models that carry
+		// known_usecases: [tts], name matching would have to guess at
+		// third-party GGUF repacks, and "base" (the substring the Qwen and
+		// vLLM cases key on) is a routine word in text-model names.
+		//
+		// Deliberately reads the declared bit instead of HasUsecases, which
+		// falls through to GuessUsecases and would hand the decision to a
+		// heuristic that never had a llama.cpp TTS model in mind.
+		supported = cfg.KnownUsecases != nil && (*cfg.KnownUsecases&FLAG_TTS) == FLAG_TTS
 	default:
 		supported = true
 	}

core/config/backend_capabilities_test.go

@@ -247,6 +247,78 @@ var _ = Describe("VoiceCloningForModel", func() {
 	)
 })
 
+// llama.cpp serves Qwen3-TTS as well as the text LLMs it is known for, so the
+// backend has to advertise TTS. That advertisement is what makes narrowing
+// mandatory: the per-backend switch in VoiceCloningForModel ends in a
+// permissive default, so an unnarrowed llama-cpp entry would offer
+// reference-audio cloning on every GGUF chat model in the gallery.
+var _ = Describe("llama-cpp TTS capabilities", func() {
+	It("advertises the TTS RPCs and usecase", func() {
+		capability := GetBackendCapability("llama-cpp")
+		Expect(capability).NotTo(BeNil())
+		Expect(capability.GRPCMethods).To(ContainElements(MethodTTS, MethodTTSStream))
+		Expect(capability.PossibleUsecases).To(ContainElement(UsecaseTTS))
+	})
+
+	// The gallery filter and the model importer both read DefaultUsecases, and a
+	// bare GGUF served by llama.cpp is a chat model, not a TTS model.
+	It("keeps chat as its only default usecase", func() {
+		Expect(GetBackendCapability("llama-cpp").DefaultUsecases).To(Equal([]string{UsecaseChat}))
+	})
+
+	ttsModel := func(backend string) ModelConfig {
+		cfg := ModelConfig{Name: "qwen3-tts-llamacpp", Backend: backend}
+		cfg.KnownUsecaseStrings = []string{"tts"}
+		cfg.syncKnownUsecasesFromString()
+		return cfg
+	}
+
+	It("resolves voice cloning for a model that declares the TTS usecase", func() {
+		cfg := ttsModel("llama-cpp")
+		cloning := VoiceCloningForModel(&cfg)
+		Expect(cloning).NotTo(BeNil())
+		Expect(cloning.AcceptedAudioFormats).To(ContainElement("audio/wav"))
+	})
+
+	// The spec that constrains the fix. Every one of these is an ordinary
+	// llama.cpp text model, and none of them may be offered in the Voice
+	// Library or accept a localai://voice-profiles/... reference.
+	DescribeTable("never resolves voice cloning for an ordinary llama.cpp model",
+		func(cfg ModelConfig) {
+			Expect(VoiceCloningForModel(&cfg)).To(BeNil())
+		},
+		Entry("plain chat model", ModelConfig{Name: "qwen3-8b", Backend: "llama-cpp"}),
+		Entry("auto-detected GGUF with no backend pinned", ModelConfig{Name: "mistral-7b"}),
+		Entry("a vision model with an mmproj", ModelConfig{Name: "gemma-3-12b", Backend: "llama-cpp", LLMConfig: LLMConfig{MMProj: "mmproj-gemma-3-12b.gguf"}}),
+		Entry("a chat model whose name happens to say base", ModelConfig{Name: "llama-3.1-8b-base", Backend: "llama-cpp"}),
+		Entry("a pinned hardware variant", ModelConfig{Name: "qwen3-8b", Backend: "cuda12-llama-cpp"}),
+	)
+
+	// A declared-TTS model must keep its contract through the pinned gallery
+	// variants an operator can put in `backend:`, the same way vibevoice-cpp
+	// and crispasr do.
+	DescribeTable("resolves through pinned gallery variants",
+		func(backend string) {
+			cfg := ttsModel(backend)
+			Expect(VoiceCloningForModel(&cfg)).NotTo(BeNil())
+		},
+		Entry("cuda12", "cuda12-llama-cpp"),
+		Entry("vulkan", "vulkan-llama-cpp"),
+		Entry("metal darwin arm64", "metal-darwin-arm64-llama-cpp"),
+		Entry("development channel", "llama-cpp-development"),
+	)
+
+	// tts.voice_cloning is the documented escape hatch for a custom build. It
+	// only ever reaches the operator once the backend carries the contract at
+	// all, which is precisely what the unregistered entry prevented.
+	It("still honours an explicit opt-out on a declared-TTS model", func() {
+		cfg := ttsModel("llama-cpp")
+		disabled := false
+		cfg.TTSConfig.VoiceCloning = &disabled
+		Expect(VoiceCloningForModel(&cfg)).To(BeNil())
+	})
+})
+
 var _ = Describe("IsValidUsecaseForBackend", func() {
 	It("accepts a backend's declared usecases", func() {
 		Expect(IsValidUsecaseForBackend("piper", "tts")).To(BeTrue())

core/config/gguf.go

@@ -26,6 +26,27 @@ func reservedNonChatModel(cfg *ModelConfig) bool {
 		(*cfg.KnownUsecases&(FLAG_SCORE|FLAG_TOKEN_CLASSIFY)) != 0
 }
 
+// genAudioEncoderKey is the mmproj metadata flag llama.cpp's mtmd writes for a
+// projector carrying the speech-synthesis pipeline (ggml-org/llama.cpp#26254).
+const genAudioEncoderKey = "clip.has_gen_audio_encoder"
+
+// HasGenAudioProjector reports whether a parsed mmproj GGUF holds a gen-audio
+// pipeline (Qwen3-TTS) rather than a vision tower. This is the same flag
+// mtmd_helper_gen_audio itself checks before building the pipeline, so it is
+// the engine's own answer rather than a filename heuristic: an mmproj is
+// otherwise indistinguishable from a vision projector by name alone, and every
+// TTS repo names it mmproj-*.gguf exactly like a vision one.
+func HasGenAudioProjector(f *gguf.GGUFFile) bool {
+	if f == nil {
+		return false
+	}
+	kv, ok := f.Header.MetadataKV.Get(genAudioEncoderKey)
+	if !ok || kv.ValueType != gguf.GGUFMetadataValueTypeBool {
+		return false
+	}
+	return kv.ValueBool()
+}
+
 func guessGGUFFromFile(cfg *ModelConfig, f *gguf.GGUFFile, defaultCtx int) {
 	// Explicit opt-in: a negative context_size (canonically -1) means "use the
 	// model's full trained context (n_ctx_train) from GGUF metadata". Unlike the

core/config/model_capabilities.go

@@ -54,13 +54,19 @@ func (c *ModelConfig) VisionSupported() bool {
 	if c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_VISION) == FLAG_VISION {
 		return true
 	}
-	if c.MMProj != "" {
+	// A TTS model's mmproj holds a speaker encoder and code predictor, not a
+	// vision tower, and llama.cpp builds an mtmd context (and so reports a media
+	// marker on the first chat probe) for it all the same. Neither signal proves
+	// image input on a declared-TTS model. Callers that genuinely are both
+	// declare FLAG_VISION, checked above.
+	declaredTTS := c.KnownUsecases != nil && (*c.KnownUsecases&FLAG_TTS) == FLAG_TTS
+	if c.MMProj != "" && !declaredTTS {
 		return true
 	}
 	if c.TemplateConfig.Multimodal != "" {
 		return true
 	}
-	if c.MediaMarker != "" {
+	if c.MediaMarker != "" && !declaredTTS {
 		return true
 	}
 	return false

core/config/model_capabilities_test.go

@@ -35,6 +35,31 @@ var _ = Describe("Model capabilities derivation", func() {
 			Expect(cfg.VisionSupported()).To(BeTrue())
 		})
 
+		It("is false for a TTS model whose mmproj is a speaker encoder", func() {
+			// Qwen3-TTS on llama-cpp ships an mmproj that holds the speaker
+			// encoder and code predictor, not a vision tower.
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "llama-cpp"}
+			cfg.MMProj = "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+
+		It("is false for a TTS model whose backend reported a media marker", func() {
+			// llama.cpp builds an mtmd context for the speaker-encoder projector
+			// and reports its marker on the first chat probe, which would
+			// otherwise resurrect vision after the model has been used once.
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS), Backend: "llama-cpp"}
+			cfg.MMProj = "mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"
+			cfg.MediaMarker = "<__media__>"
+			Expect(cfg.VisionSupported()).To(BeFalse())
+		})
+
+		It("is still true for a TTS model that also declares vision", func() {
+			// An omni model can legitimately be both. The explicit bit wins.
+			cfg := &ModelConfig{KnownUsecases: usecaseBits(FLAG_TTS | FLAG_VISION), Backend: "llama-cpp"}
+			cfg.MMProj = "mmproj.gguf"
+			Expect(cfg.VisionSupported()).To(BeTrue())
+		})
+
 		It("does not fall for the GuessUsecases FLAG_VISION false positive", func() {
 			// A chat model with a chat template would make HasUsecases(FLAG_VISION)
 			// return true via the guess heuristic; VisionSupported must not.
@@ -42,6 +67,26 @@ var _ = Describe("Model capabilities derivation", func() {
 			cfg.TemplateConfig.Chat = "{{.Input}}"
 			Expect(cfg.VisionSupported()).To(BeFalse())
 		})
+
+		It("survives the loader re-syncing known_usecases from the rewritten list", func() {
+			// syncKnownUsecasesFromString rewrites KnownUsecaseStrings from
+			// HasUsecases, and the loader calls it more than once per file. If a
+			// guessed "vision" leaks into that list, the next pass parses it back
+			// into KnownUsecases as an explicit bit and the mmproj exemption above
+			// is bypassed. Reproduces the gallery entry qwen3-tts-llamacpp-q4.
+			cfg := &ModelConfig{Backend: "llama-cpp"}
+			cfg.KnownUsecaseStrings = []string{"tts"}
+			cfg.MMProj = "qwen3-tts-llamacpp-q4/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf"
+			cfg.TemplateConfig.UseTokenizerTemplate = true
+
+			cfg.syncKnownUsecasesFromString()
+			cfg.syncKnownUsecasesFromString()
+
+			Expect(cfg.KnownUsecaseStrings).NotTo(ContainElement("FLAG_VISION"))
+			Expect(cfg.VisionSupported()).To(BeFalse())
+			Expect(cfg.Capabilities()).NotTo(ContainElement(UsecaseVision))
+			Expect(cfg.InputModalities()).NotTo(ContainElement(ModalityImage))
+		})
 	})
 
 	Describe("AudioInputSupported / VideoInputSupported", func() {

core/config/model_config.go

@@ -1876,6 +1876,20 @@ func (c *ModelConfig) GuessUsecases(u ModelConfigUsecase) bool {
 		}
 	}
 
+	if (u & FLAG_VISION) == FLAG_VISION {
+		// Without a branch here the function falls through to true, which paints
+		// vision onto every chat model. That is not just a cosmetic wrong answer:
+		// syncKnownUsecasesFromString rewrites KnownUsecaseStrings from
+		// HasUsecases, so a guessed FLAG_VISION is written back and the next sync
+		// parses it into KnownUsecases as if the operator had declared it,
+		// defeating the explicit-signal checks in VisionSupported. Defer to the
+		// same explicit signals here; VisionSupported never calls back into
+		// HasUsecases, so this does not recurse.
+		if !c.VisionSupported() {
+			return false
+		}
+	}
+
 	if (u & FLAG_DETECTION) == FLAG_DETECTION {
 		detectionBackends := []string{"rfdetr", "sam3-cpp", "insightface"}
 		if !slices.Contains(detectionBackends, c.Backend) {

core/config/mtp_test.go

@@ -122,3 +122,52 @@ var _ = Describe("MTP auto-defaults", func() {
 		})
 	})
 })
+
+// The mmproj of a Qwen3-TTS GGUF repo is named exactly like a vision
+// projector, so the gen-audio flag llama.cpp's own mtmd gates its speech
+// pipeline on is the only thing that tells the two apart.
+var _ = Describe("HasGenAudioProjector", func() {
+	mmproj := func(key string, valueType gguf.GGUFMetadataValueType, value any) *gguf.GGUFFile {
+		return &gguf.GGUFFile{
+			Header: gguf.GGUFHeader{
+				MetadataKV: gguf.GGUFMetadataKVs{
+					{Key: "general.architecture", ValueType: gguf.GGUFMetadataValueTypeString, Value: "clip"},
+					{Key: key, ValueType: valueType, Value: value},
+				},
+			},
+		}
+	}
+
+	It("detects the gen-audio projector Qwen3-TTS ships", func() {
+		f := mmproj("clip.has_gen_audio_encoder", gguf.GGUFMetadataValueTypeBool, true)
+		Expect(HasGenAudioProjector(f)).To(BeTrue())
+	})
+
+	It("is false for a vision projector", func() {
+		f := mmproj("clip.has_vision_encoder", gguf.GGUFMetadataValueTypeBool, true)
+		Expect(HasGenAudioProjector(f)).To(BeFalse())
+	})
+
+	// A speaker encoder alone is reference-audio INPUT. The gen-audio decoder
+	// is what makes the model emit speech, and Qwen3-TTS carries both.
+	It("is false for a projector that only encodes speaker audio", func() {
+		f := mmproj("clip.has_audio_encoder", gguf.GGUFMetadataValueTypeBool, true)
+		Expect(HasGenAudioProjector(f)).To(BeFalse())
+	})
+
+	It("is false when the flag is present but off", func() {
+		f := mmproj("clip.has_gen_audio_encoder", gguf.GGUFMetadataValueTypeBool, false)
+		Expect(HasGenAudioProjector(f)).To(BeFalse())
+	})
+
+	// ValueBool panics on a type mismatch, and this runs against arbitrary
+	// user-supplied repos.
+	It("is false, not a panic, when the flag carries the wrong type", func() {
+		f := mmproj("clip.has_gen_audio_encoder", gguf.GGUFMetadataValueTypeString, "true")
+		Expect(HasGenAudioProjector(f)).To(BeFalse())
+	})
+
+	It("is false for a nil file", func() {
+		Expect(HasGenAudioProjector(nil)).To(BeFalse())
+	})
+})

core/gallery/importers/llama-cpp.go

@@ -300,6 +300,12 @@ func (i *LlamaCPPImporter) Import(details Details) (gallery.ModelConfig, error)
 	// it after the first start.
 	maybeApplyMTPDefaults(&modelConfig, details, &cfg)
 
+	// llama.cpp serves Qwen3-TTS through the same GGUF + mmproj shape as a
+	// vision model, so without this the TTS repos import as chat models with a
+	// vision projector: wrong usecase, wrong modality, and no route to the
+	// Voice Library.
+	maybeApplyTTSUsecase(&modelConfig, &cfg)
+
 	data, err := yaml.Marshal(modelConfig)
 	if err != nil {
 		return gallery.ModelConfig{}, err
@@ -414,6 +420,61 @@ func maybeApplyMTPDefaults(modelConfig *config.ModelConfig, details Details, cfg
 	config.ApplyMTPDefaults(modelConfig, n)
 }
 
+// maybeApplyTTSUsecase probes the selected mmproj GGUF header and, when it
+// carries llama.cpp's gen-audio pipeline instead of a vision tower, imports
+// the model as TTS rather than chat.
+//
+// A probe is the only honest signal here. A Qwen3-TTS repo has exactly the
+// shape of a vision repo, one backbone GGUF plus one mmproj-*.gguf, so neither
+// the filename nor the repo name distinguishes them; clip.has_gen_audio_encoder
+// is the key llama.cpp's own mtmd_helper_gen_audio gates the pipeline on.
+//
+// Failures are non-fatal, as in maybeApplyMTPDefaults: a network blip leaves
+// the chat default in place rather than breaking the import.
+func maybeApplyTTSUsecase(modelConfig *config.ModelConfig, cfg *gallery.ModelConfig) {
+	probeURL := pickMMProjProbeURL(modelConfig.MMProj, cfg)
+	if probeURL == "" {
+		return
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	defer func() {
+		if r := recover(); r != nil {
+			xlog.Debug("[tts-importer] panic while probing mmproj GGUF header", "uri", probeURL, "recover", r)
+		}
+	}()
+
+	f, err := gguf.ParseGGUFFileRemote(ctx, probeURL)
+	if err != nil {
+		xlog.Debug("[tts-importer] failed to read remote mmproj header for gen-audio detection", "uri", probeURL, "error", err)
+		return
+	}
+
+	if !config.HasGenAudioProjector(f) {
+		return
+	}
+
+	modelConfig.KnownUsecaseStrings = []string{config.UsecaseTTS}
+	xlog.Info("[tts-importer] gen-audio projector detected; importing as a TTS model", "name", modelConfig.Name)
+}
+
+// pickMMProjProbeURL returns an HTTP(S) URL for the mmproj the import selected,
+// or "" when none was selected or its URI cannot be range-fetched (local path,
+// OCI/Ollama artifact).
+func pickMMProjProbeURL(mmproj string, cfg *gallery.ModelConfig) string {
+	if mmproj == "" || cfg == nil {
+		return ""
+	}
+	for _, f := range cfg.Files {
+		if f.Filename == mmproj {
+			return resolveHTTPProbe(f.URI)
+		}
+	}
+	return ""
+}
+
 // pickMTPProbeURL returns an HTTP(S) URL pointing at the main (non-mmproj)
 // GGUF shard that should be inspected for an MTP head, or "" when no
 // suitable URL is available. Custom URI schemes (`huggingface://`,

core/gallery/importers/llama-cpp_internal_test.go

@@ -0,0 +1,56 @@
+package importers
+
+import (
+	"github.com/mudler/LocalAI/core/gallery"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+// The gen-audio probe has to find the mmproj the import actually selected.
+// modelConfig.MMProj holds the TARGET filename the file will be written to,
+// which is not the URI it is fetched from, so the two have to be joined
+// through cfg.Files or the probe silently never runs and every Qwen3-TTS
+// import stays labelled as chat.
+var _ = Describe("pickMMProjProbeURL", func() {
+	cfgWith := func(files ...gallery.File) *gallery.ModelConfig {
+		return &gallery.ModelConfig{Files: files}
+	}
+
+	It("resolves the URI of the selected mmproj", func() {
+		cfg := cfgWith(
+			gallery.File{Filename: "llama-cpp/models/tts/model.gguf", URI: "https://example.invalid/model.gguf"},
+			gallery.File{Filename: "llama-cpp/mmproj/tts/mmproj.gguf", URI: "https://example.invalid/mmproj.gguf"},
+		)
+		Expect(pickMMProjProbeURL("llama-cpp/mmproj/tts/mmproj.gguf", cfg)).To(Equal("https://example.invalid/mmproj.gguf"))
+	})
+
+	It("resolves a huggingface:// mmproj URI to a fetchable URL", func() {
+		cfg := cfgWith(gallery.File{
+			Filename: "mmproj.gguf",
+			URI:      "huggingface://ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf",
+		})
+		Expect(pickMMProjProbeURL("mmproj.gguf", cfg)).To(HavePrefix("https://"))
+	})
+
+	It("returns nothing when the import selected no mmproj", func() {
+		cfg := cfgWith(gallery.File{Filename: "model.gguf", URI: "https://example.invalid/model.gguf"})
+		Expect(pickMMProjProbeURL("", cfg)).To(BeEmpty())
+	})
+
+	It("returns nothing when the mmproj is not among the files", func() {
+		cfg := cfgWith(gallery.File{Filename: "model.gguf", URI: "https://example.invalid/model.gguf"})
+		Expect(pickMMProjProbeURL("mmproj.gguf", cfg)).To(BeEmpty())
+	})
+
+	// OCI/Ollama artifacts are not range-fetchable as a GGUF byte stream, the
+	// same reason the MTP probe skips them.
+	It("returns nothing for an OCI artifact", func() {
+		cfg := cfgWith(gallery.File{Filename: "mmproj.gguf", URI: "oci://quay.io/example/model:latest"})
+		Expect(pickMMProjProbeURL("mmproj.gguf", cfg)).To(BeEmpty())
+	})
+
+	It("returns nothing for a nil config", func() {
+		Expect(pickMMProjProbeURL("mmproj.gguf", nil)).To(BeEmpty())
+	})
+})

core/http/routes/ui_api_internal_test.go

@@ -11,3 +11,21 @@ func TestUsecaseFiltersIncludes3D(t *testing.T) {
 	g := gomega.NewWithT(t)
 	g.Expect(usecaseFilters[config.Usecase3D]).To(gomega.Equal(config.FLAG_3D))
 }
+
+// GET /api/backends/usecases projects each backend's PossibleUsecases through
+// usecaseFilters, and the gallery greys out any filter a selected backend does
+// not report. llama-cpp serves Qwen3-TTS, so the TTS filter has to survive that
+// projection or the gallery hides the very entries the backend can run.
+func TestBackendUsecasesReportsTTSForLlamaCpp(t *testing.T) {
+	g := gomega.NewWithT(t)
+
+	var keys []string
+	for _, uc := range config.BackendCapabilities["llama-cpp"].PossibleUsecases {
+		if _, ok := usecaseFilters[uc]; ok {
+			keys = append(keys, uc)
+		}
+	}
+
+	g.Expect(keys).To(gomega.ContainElement(config.UsecaseTTS))
+	g.Expect(keys).To(gomega.ContainElement(config.UsecaseChat))
+}

docs/content/features/text-to-audio.md

@@ -130,6 +130,7 @@ When a saved profile is selected, LocalAI supplies both its private WAV and exac
 | --- | --- |
 | `chatterbox`, `faster-qwen3-tts`, `fish-speech`, `moss-tts-cpp`, `neutts`, `omnivoice-cpp`, `pocket-tts`, `voxcpm` | Reference-audio cloning models served by these dedicated backends. |
 | `qwen-tts`, `qwen3-tts-cpp`, `vllm-omni` | Base or VoiceClone variants. CustomVoice and VoiceDesign variants are not raw reference-audio models. |
+| `llama-cpp` | Models that declare `known_usecases: [tts]`, which the Qwen3-TTS gallery entries (`qwen3-tts-llamacpp`, `qwen3-tts-llamacpp-q4`) do. A reference clip is required, since the Base checkpoints have no built-in speaker. Ordinary GGUF chat and vision models served by this backend are excluded. |
 | `vibevoice-cpp` | 1.5B reference-WAV variants. The realtime 0.5B preset-prompt model is excluded. |
 | `coqui` | XTTS and YourTTS variants. |
 | `crispasr` | F5-TTS variants. ASR, Piper, Orpheus, and other CrispASR model families are excluded. |
@@ -170,7 +171,7 @@ curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
 }' > output.wav
 ```
 
-Note: Streaming TTS is currently supported by the `voxcpm` backend. Other backends will fall back to non-streaming mode if streaming is not supported.
+Note: Streaming TTS is implemented by the `audio-cpp`, `crispasr`, `llama-cpp`, `magpie-tts-cpp`, `moss-tts-cpp`, `omnivoice-cpp`, `qwen3-tts-cpp`, `sherpa-onnx`, `supertonic`, `vibevoice-cpp` and `voxcpm` backends. Other backends will fall back to non-streaming mode if streaming is not supported.
 
 ## Backends
 
@@ -535,6 +536,59 @@ tts:
   audio_path: voices/default-reference.wav  # optional fallback
 ```
 
+#### llama.cpp gallery variants
+
+llama.cpp gained native Qwen3-TTS support in [ggml-org/llama.cpp#26254](https://github.com/ggml-org/llama.cpp/pull/26254), so the `llama-cpp` backend can serve it on the same accelerator matrix it already uses for text generation: CUDA, ROCm, SYCL, Vulkan and Metal.
+
+Install `qwen3-tts-llamacpp` (Q8_0 backbone) or `qwen3-tts-llamacpp-q4` (Q4_K_M backbone) from the Model gallery, or run `local-ai models install qwen3-tts-llamacpp-q4`.
+
+These models load two files: the backbone GGUF and a multimodal projector holding the speaker encoder and code predictor. A hand-written configuration must point at both:
+
+```yaml
+name: qwen3-tts-llamacpp
+backend: llama-cpp
+known_usecases:
+  - tts
+mmproj: qwen3-tts-llamacpp/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+parameters:
+  model: qwen3-tts-llamacpp/Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+```
+
+`known_usecases: [tts]` is not optional here. It is how LocalAI tells a Qwen3-TTS checkpoint apart from the text and vision GGUFs the same backend serves: without it the model is treated as a chat model, its projector is read as a vision tower, and Voice Library profiles are refused.
+
+Importing such a repo through the Models page or `local-ai models import` writes that declaration for you. The importer reads the projector's header and recognises the speech-synthesis pipeline, so `ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF` imports as a TTS model rather than as a chat model with a vision projector.
+
+The upstream checkpoints are Base variants with no built-in speaker, so `voice` is **required** on every request. Pass either a path to a reference clip or a saved Voice Library profile. A request without one is rejected rather than served in an arbitrary voice:
+
+```bash
+curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
+  "model": "qwen3-tts-llamacpp",
+  "input": "Hello world, this is a test.",
+  "voice": "voices/my-reference.wav"
+}' > output.wav
+```
+
+Output is always 24 kHz mono 16-bit WAV. Streaming works on this backend, so `"stream": true` returns audio chunk by chunk as it is generated.
+
+Set `language` to an ISO 639-1 code to pin the output language. The supported codes are `zh`, `en`, `de`, `it`, `pt`, `es`, `ja`, `ko`, `fr` and `ru`.
+
+Three optional knobs travel in `params`: `top_k` and `top_p` adjust sampling, and `max_frames` caps how much audio a single request may generate. The model runs at 12.5 frames per second, so one frame is 0.08 seconds and the maximum duration in seconds is `max_frames / 12.5`. Leave it unset for the engine default of 512 frames, which is 40.96 seconds.
+
+`max_frames` exists because generation occasionally fails to stop on its own. The model normally ends an utterance by emitting its end-of-speech token, but once in a while it does not, and the request then runs to the cap and returns far more audio than the text called for. It is uncommon, and it happens more on short inputs than on long ones. If you are synthesising predictable text and want a hard bound, allow roughly 8 frames per word: about 100 frames (8 seconds) for a short sentence, about 300 frames (24 seconds) for a paragraph.
+
+```bash
+curl http://localhost:8080/tts -H "Content-Type: application/json" -d '{
+  "model": "qwen3-tts-llamacpp",
+  "input": "Hello world, this is a test.",
+  "voice": "voices/my-reference.wav",
+  "params": {"max_frames": "100"}
+}' > output.wav
+```
+
+This backend accepts but ignores `instructions`, `speed` and `sample_rate`. The Base checkpoints have no expressive-style or rate control, and the output rate is fixed at 24 kHz.
+
+Note that `qwen3-tts-cpp` (qwentts.cpp) remains available and is unaffected. It is a separate, independently maintained path to the same family of weights, not something this replaces.
+
 #### Usage
 
 Use the tts endpoint by specifying the qwen-tts backend:

gallery/index.yaml

@@ -5085,6 +5085,12 @@
     known_usecases:
       - chat
       - completion
+      - vision
+    known_input_modalities:
+      - text
+      - image
+      - video
+      - audio
     parameters:
       min_p: 0.01
       model: mudler/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-APEX-GGUF
@@ -8880,6 +8886,69 @@
     - filename: qwen3-tts-cpp-1.7b-voicedesign-q4/qwen-tokenizer-12hz-Q4_K_M.gguf
       sha256: cf3788b4d50aaa665fb6e57c170396aae03a3555fea52d2b5d0cda902d658039
       uri: huggingface://Serveurperso/Qwen3-TTS-GGUF/qwen-tokenizer-12hz-Q4_K_M.gguf
+- &qwen3ttsllamacpp_gallery
+  name: qwen3-tts-llamacpp
+  variants:
+    - model: qwen3-tts-llamacpp-q4
+  url: github:mudler/LocalAI/gallery/virtual.yaml@master
+  urls:
+    - https://huggingface.co/ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF
+    - https://github.com/ggml-org/llama.cpp/pull/26254
+  description: |
+    Qwen3-TTS 1.7B Base served by the llama.cpp backend, using upstream's own
+    GGUF conversion. Runs on the full llama-cpp accelerator matrix (CUDA, ROCm,
+    SYCL, Vulkan, Metal). Streaming output and zero-shot voice cloning: set
+    `voice` to a reference clip or a saved Voice Library profile, which is
+    required since the Base checkpoint has no built-in speaker. 24kHz mono,
+    10 languages. Q8_0 backbone (~1.8 GB) plus a Q8_0 projector.
+  license: apache-2.0
+  icon: https://avatars.githubusercontent.com/u/12608286?s=200&v=4
+  tags:
+    - tts
+    - text-to-speech
+    - voice-cloning
+    - streaming
+    - qwen3-tts
+    - llama-cpp
+    - gguf
+  last_checked: "2026-08-05"
+  overrides:
+    backend: llama-cpp
+    known_usecases:
+      - tts
+    name: qwen3-tts-llamacpp
+    mmproj: qwen3-tts-llamacpp/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+    parameters:
+      model: qwen3-tts-llamacpp/Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+  files:
+    - filename: qwen3-tts-llamacpp/Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+      sha256: ac7931aeb2e7aad1a6ed6602d353a5679c9d096b18ce8204ac730a8408d572e1
+      uri: huggingface://ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+    - filename: qwen3-tts-llamacpp/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+      sha256: 6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2
+      uri: huggingface://ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+- !!merge <<: *qwen3ttsllamacpp_gallery
+  name: qwen3-tts-llamacpp-q4
+  variants: []
+  description: |
+    Qwen3-TTS 1.7B Base served by the llama.cpp backend, Q4_K_M backbone
+    (~1.1 GB) plus a Q8_0 projector. Streaming and voice cloning, 24kHz mono,
+    10 languages. A `voice` reference clip is required.
+  overrides:
+    backend: llama-cpp
+    known_usecases:
+      - tts
+    name: qwen3-tts-llamacpp-q4
+    mmproj: qwen3-tts-llamacpp-q4/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+    parameters:
+      model: qwen3-tts-llamacpp-q4/Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf
+  files:
+    - filename: qwen3-tts-llamacpp-q4/Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf
+      sha256: 8d18c94acb2addd042f97da63c98be144eafa76d0d9495177eab65130cf85129
+      uri: huggingface://ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/Qwen3-TTS-12Hz-1.7B-Base-Q4_K_M.gguf
+    - filename: qwen3-tts-llamacpp-q4/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
+      sha256: 6fd65188839bcd6ecc91b277ad471e22a0edfada4699a0fe82f1165c18cfcce2
+      uri: huggingface://ggml-org/Qwen3-TTS-12Hz-1.7B-Base-GGUF/mmproj-Qwen3-TTS-12Hz-1.7B-Base-Q8_0.gguf
 - &mossttscpp_gallery
   name: moss-tts-cpp-v1_5-q8_0
   variants:

website/content/blog/what-landed-in-localai-4-8.md

@@ -3,12 +3,12 @@ title: "What landed in LocalAI 4.8"
 date: 2026-08-04
 author: "Ettore Di Giacinto"
 category: "Release"
-tags: ["release", "vllm.cpp", "audio.cpp", "3d", "gallery", "distributed", "performance"]
-summary: "A new inference engine, 3D generation, one backend that serves six audio endpoints, and a web interface 3.48x lighter. 374 pull requests in twenty-one days."
+tags: ["release", "vllm.cpp", "audio.cpp", "3d", "agent", "gallery", "distributed", "performance"]
+summary: "A new inference engine, a terminal agent in the CLI, 3D generation, and a web interface 3.48x lighter. 386 pull requests in twenty-two days."
 extracss: ["blog.css"]
 ---
 
-LocalAI 4.8.0 is out, after twenty-one days and 374 merged pull requests. There are three new things LocalAI can do, and a lot of repair work on things it already did.
+LocalAI 4.8.0 is out, after twenty-two days and 386 merged pull requests. There are four new things LocalAI can do, and a lot of repair work on things it already did.
 
 The full notes list everything. This post covers the parts that change what you do day to day, with the pull request numbers so you can read the diffs.
 
@@ -144,6 +144,20 @@ The first engine behind it is `trellis2cpp`, an image-to-3D backend over TRELLIS
 <figcaption>trellis2-4b, 2,502,928 vertices and 5,012,118 triangles, turning in the browser. The remesh slider below it is the print path.</figcaption>
 </figure>
 
+## `local-ai chat` stopped being a REPL
+
+`local-ai chat` used to be a chat prompt in a terminal. It is now an agent, and it is the [nib](https://github.com/mudler/nib) harness compiled straight into the binary: tool use behind an approval gate, sub-agents, MCP servers, plugins and skills, auto-configured against your own instance. Nothing extra to install.
+
+```bash
+local-ai chat                      # the agent, pointed at your models
+echo "what is 2+2" | local-ai chat --cli
+local-ai chat --init zsh           # Ctrl+Space from any shell prompt
+```
+
+That last one prints a shell integration script (zsh, bash or fish), so you can pull the agent up from wherever you already are instead of opening something else.
+
+It runs shell commands now, so every tool call goes through an approval prompt you control, and read-only ones like `ls` and `cat` run without asking. If you had habits around the old REPL, a few things moved: `/clear` is gone and `/compact` is the closest thing, `/models` and `/model <name>` mean what they always meant, and switching model keeps the conversation instead of starting over ([#11291](https://github.com/mudler/LocalAI/pull/11291)).
+
 ## One backend, six audio endpoints
 
 The usual shape for audio is one backend per model family, which means a process per capability and a config file for each. `audio-cpp` wraps [audio.cpp](https://github.com/0xShug0/audio.cpp), a multi-family ggml audio engine. One backend process serves several unrelated families through a single runtime vocabulary, and works out which family a checkpoint belongs to from the GGUF's own `audiocpp.model_spec.family` metadata key. There is nothing backend-specific to write in the model config.

website/layouts/index.html

@@ -19,7 +19,7 @@
           <div><b class="tnum" data-count="{{ .Site.Data.stats.stars }}">0</b><span>GitHub stars</span></div>
           <div><b class="tnum" data-count="73">0</b><span>Backends</span></div>
           <div><b class="tnum" data-count="{{ len .Site.Data.engines.engines }}">0</b><span>Engines we wrote</span></div>
-          <div><b class="tnum" data-count="1585">0</b><span>Models, one click</span></div>
+          <div><b class="tnum" data-count="1255">0</b><span>Models, one click</span></div>
         </div>
       </div>
       <div class="fd">
@@ -58,7 +58,7 @@
       </div>
       <div class="duo__m rv">
         <figure class="screen" style="margin:0">
-          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,585 models</b></figcaption>
+          <figcaption class="screen__bar"><i></i> localai · model gallery <b>1,255 models</b></figcaption>
           <video src="/media/gallery.mp4" muted loop playsinline preload="none" data-lazy aria-label="Installing a model from the LocalAI gallery"></video>
         </figure>
       </div>
@@ -328,7 +328,7 @@
   <div class="shell">
     <div class="bars rv" aria-hidden="true"><i></i><i></i><i></i><i></i></div>
     <p class="kicker rv">The gallery</p>
-    <h2 class="rv mt1" style="max-width:20ch">1,585 models. No notebook, no conversion script.</h2>
+    <h2 class="rv mt1" style="max-width:20ch">1,255 models. No notebook, no conversion script.</h2>
     <div class="cards">
       <a class="cd rv" href="/docs/getting-started/models/"><p class="cd__k">Quantizations</p><h3>201 APEX builds</h3>
         <p>Every tier of every model we quantize, ranked against the hardware you actually have and installed with one click.</p><span class="cd__go">Browse the gallery →</span></a>