From 7cfccdc2bf179cd0a77c9c9daa9f9ea938613d78 Mon Sep 17 00:00:00 2001 From: "mudler's LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Tue, 11 Aug 2026 09:52:24 +0200 Subject: [PATCH] chore: :arrow_up: Update ggml-org/llama.cpp to `030ebb558a5820b444a8f836ed5cdd46c9b4bd7a` (#11454) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): rebase server patches Adapt score output limits and TTS backend sampling to the updated llama.cpp server APIs. Assisted-by: Codex:gpt-5.4 --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- backend/cpp/llama-cpp/Makefile | 2 +- .../0001-add-server-task-type-score.patch | 85 +++--- .../0002-add-server-task-type-tts.patch | 246 ++++++++---------- 3 files changed, 158 insertions(+), 175 deletions(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 71858d3e0..e37f3d544 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=9de0fcf2b3e587a43f293d9a2b6ec0a32991f768 +LLAMA_VERSION?=030ebb558a5820b444a8f836ed5cdd46c9b4bd7a LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index b89388823..905248ecb 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -1,8 +1,21 @@ +From 75220a0d74892e3315f4042274b1efa6195868d8 Mon Sep 17 00:00:00 2001 +From: Codex +Date: Mon, 10 Aug 2026 23:05:52 +0000 +Subject: [PATCH 1/2] score-patch + +--- + common/common.cpp | 6 +- + common/common.h | 3 + + tools/CMakeLists.txt | 1 + + tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++- + tools/server/server-task.h | 47 +++++ + 5 files changed, 406 insertions(+), 9 deletions(-) + diff --git a/common/common.cpp b/common/common.cpp -index 8f13217..fc584e1 100644 +index 2e3f14c..0cec0dc 100644 --- a/common/common.cpp +++ b/common/common.cpp -@@ -1591,8 +1591,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & +@@ -1636,8 +1636,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & auto cparams = llama_context_default_params(); cparams.n_ctx = params.n_ctx; @@ -13,13 +26,13 @@ index 8f13217..fc584e1 100644 + cparams.n_seq_max = params.n_parallel + params.n_seq_score_forks; + cparams.n_rs_seq = std::max(params.speculative.need_n_rs_seq(), (uint32_t) std::max(0, params.n_rs_seq)); cparams.n_outputs_max = std::max(params.n_outputs_max, 0); + cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0); cparams.n_batch = params.n_batch; - cparams.n_ubatch = params.n_ubatch; diff --git a/common/common.h b/common/common.h -index bffc176..e313bd6 100644 +index 878534d..4001df2 100644 --- a/common/common.h +++ b/common/common.h -@@ -455,6 +455,9 @@ struct common_params { +@@ -445,6 +445,9 @@ struct common_params { int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) int32_t n_parallel = 1; // number of parallel sequences to decode @@ -28,7 +41,7 @@ index bffc176..e313bd6 100644 + bool score_enabled = false; // reserve server resources for the Score task type int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) - int32_t grp_attn_n = 1; // group-attention factor + int32_t n_outputs_max_per_seq = 1; // max outputs per sequence diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt index 780df32..1d2fe8f 100644 --- a/tools/CMakeLists.txt @@ -39,28 +52,24 @@ index 780df32..1d2fe8f 100644 endif() +add_subdirectory(grpc-server) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 715477e..de5bed8 100644 +index 3b5f6a1..d0e18e6 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp -@@ -49,7 +49,16 @@ static uint32_t server_n_outputs_max(const common_params & params) { +@@ -48,6 +48,13 @@ static common_speculative_output_limits server_output_limits(const common_params + auto result = common_speculative_get_output_limits( + params.n_batch, params.n_parallel, common_speculative_n_max(¶ms.speculative)); - const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(¶ms.speculative); - -- const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq; -+ // score tasks (SERVER_TASK_TYPE_SCORE) output logits for every candidate -+ // token, so reserve room for a bounded candidate tail per parallel slot -+ if (!params.score_enabled) { -+ return std::max(1, std::min(n_batch, -+ (uint64_t) params.n_parallel * n_outputs_per_seq)); ++ // Score tasks output logits for every candidate token, so reserve room ++ // for a bounded candidate tail per parallel slot. ++ if (params.score_enabled) { ++ result.per_seq = std::max(result.per_seq, 1 + SERVER_SCORE_MAX_CAND_TOKENS); ++ result.total = std::min(params.n_batch, params.n_parallel * result.per_seq); + } + -+ const uint32_t n_outputs_score_seq = 1 + SERVER_SCORE_MAX_CAND_TOKENS; -+ -+ const uint64_t n_outputs = (uint64_t) params.n_parallel * std::max(n_outputs_per_seq, n_outputs_score_seq); - - return std::max(1, std::min(n_batch, n_outputs)); - } -@@ -202,6 +211,26 @@ struct server_slot { + result.total = std::max(1, result.total); + result.per_seq = std::max(1, result.per_seq); + return result; +@@ -239,6 +246,26 @@ struct server_slot { std::vector generated_token_probs; @@ -87,7 +96,7 @@ index 715477e..de5bed8 100644 bool has_next_token = true; bool has_new_line = false; bool truncated = false; -@@ -311,6 +340,10 @@ struct server_slot { +@@ -351,6 +378,10 @@ struct server_slot { } generated_tokens.clear(); generated_token_probs.clear(); @@ -98,7 +107,7 @@ index 715477e..de5bed8 100644 json_schema = json(); // clear speculative decoding stats -@@ -2205,6 +2238,229 @@ private: +@@ -2271,6 +2302,229 @@ private: queue_results.send(std::move(res)); } @@ -328,7 +337,7 @@ index 715477e..de5bed8 100644 // // Functions to process the task // -@@ -2341,6 +2597,7 @@ private: +@@ -2407,6 +2661,7 @@ private: case SERVER_TASK_TYPE_INFILL: case SERVER_TASK_TYPE_EMBEDDING: case SERVER_TASK_TYPE_RERANK: @@ -336,7 +345,7 @@ index 715477e..de5bed8 100644 { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -2832,6 +3089,13 @@ private: +@@ -2903,6 +3158,13 @@ private: break; // stop any further processing } } @@ -350,7 +359,7 @@ index 715477e..de5bed8 100644 } void pre_decode() { -@@ -3154,6 +3418,16 @@ private: +@@ -3222,6 +3484,16 @@ private: n_past = std::min(n_past, slot.alora_invocation_start - 1); } @@ -367,7 +376,7 @@ index 715477e..de5bed8 100644 const auto n_cache_reuse = slot.task->params.n_cache_reuse; const bool can_cache_reuse = -@@ -3395,8 +3669,12 @@ private: +@@ -3455,8 +3727,12 @@ private: bool do_checkpoint = params_base.n_ctx_checkpoints > 0; @@ -382,7 +391,7 @@ index 715477e..de5bed8 100644 // make a checkpoint of the parts of the memory that cannot be rolled back. // checkpoints are created only if: -@@ -3463,10 +3741,17 @@ private: +@@ -3523,10 +3799,17 @@ private: // embedding requires all tokens in the batch to be output; // MTP also wants logits at every prompt position so the // streaming hook can mirror t_h_nextn into ctx_dft. @@ -401,7 +410,7 @@ index 715477e..de5bed8 100644 slot.prompt.tokens.push_back(cur_tok); slot.n_prompt_tokens_processed++; -@@ -3481,6 +3766,32 @@ private: +@@ -3541,6 +3824,32 @@ private: } } @@ -434,7 +443,7 @@ index 715477e..de5bed8 100644 // process the last few tokens of the prompt separately in order to allow for a checkpoint to be created. // create checkpoints that many tokens before the end of the prompt: // - 4 + n_ubatch -@@ -3513,6 +3824,15 @@ private: +@@ -3573,6 +3882,15 @@ private: const bool is_user_start = spans.is_user_start(n_tokens_start); const bool is_last_user_message = n_tokens_start == last_user_pos; @@ -450,7 +459,7 @@ index 715477e..de5bed8 100644 // entire prompt has been processed if (slot.prompt.n_tokens() == slot.task->n_tokens()) { slot.state = SLOT_STATE_DONE_PROMPT; -@@ -3528,8 +3848,8 @@ private: +@@ -3588,8 +3906,8 @@ private: slot.init_sampler(); } else { // skip ordinary mid-prompt checkpoints, unless the batch starts a user @@ -461,7 +470,7 @@ index 715477e..de5bed8 100644 do_checkpoint = false; } } -@@ -3546,10 +3866,10 @@ private: +@@ -3606,10 +3924,10 @@ private: // do not checkpoint after mtmd chunks do_checkpoint = do_checkpoint && !has_mtmd; @@ -474,7 +483,7 @@ index 715477e..de5bed8 100644 n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); -@@ -3703,6 +4023,13 @@ private: +@@ -3772,6 +4090,13 @@ private: } } @@ -488,7 +497,7 @@ index 715477e..de5bed8 100644 if (!is_inside_view(slot.i_batch)) { // the required token not in this sub-batch, skip return; -@@ -3724,6 +4051,25 @@ private: +@@ -3793,6 +4118,25 @@ private: return; } @@ -515,7 +524,7 @@ index 715477e..de5bed8 100644 // prompt evaluated for next-token prediction diff --git a/tools/server/server-task.h b/tools/server/server-task.h -index c3eea2e..fb3c178 100644 +index 6275ec7..5bedf19 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -13,10 +13,25 @@ @@ -597,3 +606,5 @@ index c3eea2e..fb3c178 100644 struct server_task_result_error : server_task_result { error_type err_type = ERROR_TYPE_SERVER; std::string err_msg; +-- +2.39.5 diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 2a9f7fb02..4e05d9f79 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -1,57 +1,25 @@ -# Carries the server-side half of ggml-org/llama.cpp#26603 (POST /tts), which -# adds SERVER_TASK_TYPE_TTS to the shared server_context. LocalAI's gRPC -# adapter rides on that same server_context and cannot drive the mtmd gen-audio -# loop directly: server_context owns the llama_context and schedules slots on -# its own thread, so a handler calling llama_decode would corrupt state. -# -# The upstream HTTP route (tools/server/server.cpp) and the README/CLI hunks are -# intentionally excluded - LocalAI does not serve llama.cpp's HTTP surface. -# -# Rebased against LLAMA_VERSION 9de0fcf2b with 0001-add-server-task-type-score.patch -# already applied: the SERVER_TASK_TYPE_TTS case in the tokenize switch lands -# after the SCORE case that patch adds, so this patch must stay second in ls order. -# -# Three fixes on top of the draft, all ours, all candidates to send back to #26603: -# -# 1. Its lone SRV_WRN call passes only a format string, but the macro expands -# __VA_ARGS__ without the GNU comma-elision extension, so the expansion ends -# in a trailing comma and does not compile. The "%s" wrapper added here is the -# same idiom upstream already uses for its other argument-less SRV_WRN calls; -# drop it if the draft fixes the call before it merges. -# -# 2. The slot.prompt_clear() added to the SERVER_TASK_TYPE_TTS branch of -# launch_slot_with_task. Without it only the FIRST TTS request in a server -# process succeeds and every later one fails instantly in step_prompt. TTS -# slots never enter the shared batch (pre_decode() returns early for them and -# process_tts_slots() drives them), so they skip the prompt-cache bookkeeping -# that would otherwise clear the sequence between requests; meanwhile the -# gen-audio pipeline always decodes from position 0 and its reset() only -# clears host-side buffers, never the KV cache. The result is that request 2 -# decodes over request 1's tokens. This one is a genuine defect in the draft, -# not a LocalAI integration artifact, and should be reported upstream. -# -# 3. The is_supported() guard in server_slot::tts_ctx::reset(). The draft calls -# mtmd_helper_gen_audio_reset() unconditionally, but it only ever init()s the -# gen-audio pipeline for models that carry a gen-audio mmproj, so the handle -# stays null for every ordinary model. Upstream's implementation -# (tools/mtmd/mtmd-helper-gen.cpp) reads ctx->pipeline before it null-checks -# anything, so the slot-init call to server_slot::reset() segfaults the -# process on any non-TTS model - which is every chat model LocalAI loads -# through this backend. The missing null check on mtmd_helper_gen_audio_* -# is upstream's bug, so carry this guard until they add one, even if the -# draft's own reset() call is fixed. -# -# REMOVE THIS PATCH once #26603 merges upstream and LLAMA_VERSION is bumped past -# the merge commit. It exists only because that PR is still a draft. If it merges -# without fixes 2 and 3 above, those hunks still need carrying. +From 861fb06531e770fcea65e86296f80eba830dac30 Mon Sep 17 00:00:00 2001 +From: Codex +Date: Mon, 10 Aug 2026 23:05:53 +0000 +Subject: [PATCH 2/2] tts-patch + +--- + tools/mtmd/mtmd-helper-gen.cpp | 118 +++++++++--- + tools/mtmd/mtmd-helper.h | 47 ++++- + tools/server/server-context.cpp | 322 +++++++++++++++++++++++++++++++- + tools/server/server-context.h | 3 + + tools/server/server-task.cpp | 11 ++ + tools/server/server-task.h | 16 ++ + 6 files changed, 481 insertions(+), 36 deletions(-) + diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp -index b52dc8e..fd9d6ca 100644 +index 85671d1..66ccf0a 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp @@ -48,29 +48,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri return LLAMA_TOKEN_NULL; } - + +static void put_bytes(std::vector & buf, const void * p, size_t n) { + const char * c = (const char *) p; + buf.insert(buf.end(), c, c + n); @@ -104,17 +72,17 @@ index b52dc8e..fd9d6ca 100644 + append_wav16_pcm(buf, pcm.data(), pcm.size()); return true; } - + @@ -89,6 +98,8 @@ public: // those read what they need from h_state_in instead virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0; virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; + // forces any buffered codes through code2wav now, regardless of window_frames + virtual int32_t flush() = 0; - + protected: llama_context * lctx; -@@ -119,6 +130,9 @@ public: +@@ -118,6 +129,9 @@ public: prompt_batch.reset(); n_prompt = 0; prompt_pos = 0; @@ -122,19 +90,19 @@ index b52dc8e..fd9d6ca 100644 + pcm_sent = 0; + wav_header_sent = false; } - + int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { -@@ -204,6 +218,7 @@ public: +@@ -203,6 +217,7 @@ public: top_k = inp->top_k > 0 ? inp->top_k : 50; top_p = inp->top_p > 0 ? inp->top_p : 1.0f; out_type = inp->out_type; + stream = inp->stream; - - // the text stream keeps flowing during generation: after frame k, the input adds - // trailing text row k on top of the codes embedding, then tts_eos, then tts_pad -@@ -289,31 +304,60 @@ public: + + // the prompt above holds the whole text stream up to tts_eos, so every generated + // frame adds tts_pad on top of the codes embedding +@@ -284,31 +299,60 @@ public: } - + int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { - if (!flush_gen_wav()) { - return 1; @@ -162,7 +130,7 @@ index b52dc8e..fd9d6ca 100644 + *out_data_len = out_buf.size(); + return 0; } - + - *out_sample_rate = info.sample_rate; + // streaming: only return audio produced since the previous call + const size_t n_new = audio_pcm.size() - pcm_sent; @@ -170,7 +138,7 @@ index b52dc8e..fd9d6ca 100644 - *out_n_samples = (int64_t) audio_pcm.size(); + *out_n_samples = (int64_t) n_new; } - + if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { - *out_data = (const char *) audio_pcm.data(); - *out_data_len = audio_pcm.size() * sizeof(float); @@ -179,7 +147,7 @@ index b52dc8e..fd9d6ca 100644 + pcm_sent = audio_pcm.size(); return 0; } - + out_buf.clear(); - if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { - LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); @@ -194,7 +162,7 @@ index b52dc8e..fd9d6ca 100644 *out_data_len = out_buf.size(); return 0; } - + + int32_t flush() override { + return flush_gen_wav() ? 0 : 1; + } @@ -202,7 +170,7 @@ index b52dc8e..fd9d6ca 100644 private: bool ensure_cache() { if (specials_ok) { -@@ -357,7 +401,7 @@ private: +@@ -352,7 +396,7 @@ private: LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } @@ -211,7 +179,7 @@ index b52dc8e..fd9d6ca 100644 mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; -@@ -442,6 +486,9 @@ private: +@@ -436,6 +480,9 @@ private: std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; @@ -219,12 +187,12 @@ index b52dc8e..fd9d6ca 100644 + size_t pcm_sent = 0; // samples already returned by get_output() + bool wav_header_sent = false; }; - + static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { -@@ -473,6 +520,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { +@@ -467,6 +514,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { } } - + +struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { + mtmd_helper_gen_audio_inp inp{}; + inp.top_k = 50; @@ -236,7 +204,7 @@ index b52dc8e..fd9d6ca 100644 int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { if (!ctx->pipeline) { LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); -@@ -503,3 +558,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * +@@ -497,3 +552,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * } return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); } @@ -257,19 +225,19 @@ index 7e5cf9b..1f3ec01 100644 struct mtmd_helper_gen_audio_inp { + bool stream; // if true, output() must be called after each step_gen() llama_seq_id seq_id; - + const char * prompt; @@ -189,6 +190,8 @@ struct mtmd_helper_gen_audio_inp { enum mtmd_helper_gen_audio_outtype out_type; }; - + +MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void); + MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( struct llama_context * lctx, struct mtmd_context * mctx); @@ -217,6 +220,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( - + // out_data valid until next get_output() or reset() call // out_n_samples (optional, can be NULL) receives the number of generated PCM samples +// if inp->stream is true: returns only audio produced since the previous call, and @@ -280,7 +248,7 @@ index 7e5cf9b..1f3ec01 100644 @@ -224,6 +229,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output( size_t * out_data_len, int64_t * out_n_samples); - + +// forces any buffered codes through code2wav now, regardless of window_frames; +// call once when generation has ended, before the last get_output() in stream mode +MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); @@ -339,10 +307,10 @@ index 7e5cf9b..1f3ec01 100644 + return mtmd_helper_gen_audio_flush(ctx.get()); + } }; - + } // namespace mtmd_helper diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 543dc43..4f01b1e 100644 +index d0e18e6..0765065 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -16,6 +16,7 @@ @@ -350,23 +318,25 @@ index 543dc43..4f01b1e 100644 #include "mtmd.h" #include "mtmd-helper.h" +#include "base64.hpp" - + #include #include -@@ -43,7 +44,8 @@ static uint32_t server_n_outputs_max(const common_params & params) { - const uint32_t n_batch = params.n_batch; - +@@ -41,8 +42,9 @@ constexpr int HTTP_POLLING_SECONDS = 1; + + static common_speculative_output_limits server_output_limits(const common_params & params) { if (params.embedding || - (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) { +- return { params.n_batch, 1 }; + (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE) || + !params.mmproj.path.empty()) { // gen-audio (TTS) capability isn't known until the mmproj loads, size generously - return n_batch; ++ return { params.n_batch, params.n_batch }; } - -@@ -214,6 +216,30 @@ struct server_slot { + + auto result = common_speculative_get_output_limits( +@@ -211,6 +213,30 @@ struct server_slot { mtmd_context * mctx = nullptr; mtmd::batch_ptr mbatch = nullptr; - + + struct tts_ctx { + mtmd_helper::gen_audio ctx; + const float * h_state; @@ -393,20 +363,20 @@ index 543dc43..4f01b1e 100644 + // speculative decoding common_speculative * spec; - -@@ -403,6 +429,8 @@ struct server_slot { - + +@@ -400,6 +426,8 @@ struct server_slot { + // clear multimodal state mbatch.reset(); + + tts.reset(); } - + void init_sampler() const { -@@ -949,6 +977,14 @@ public: +@@ -946,6 +974,14 @@ public: mtmd_context * mctx = nullptr; const llama_vocab * vocab = nullptr; - + + bool has_cap_tts() const { + return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE; + } @@ -417,22 +387,22 @@ index 543dc43..4f01b1e 100644 + server_queue queue_tasks; server_response queue_results; - -@@ -1400,6 +1436,10 @@ private: + +@@ -1399,6 +1435,10 @@ private: slot.mctx = mctx; slot.prompt.tokens.has_mtmd = mctx != nullptr; - + + if (has_cap_tts()) { + slot.tts.ctx.init(ctx_tgt, mctx); + } + SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx); - + slot.callback_on_release = [this](int id_slot) { -@@ -1853,6 +1893,28 @@ private: - +@@ -1852,6 +1892,28 @@ private: + SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); - + + if (task.type == SERVER_TASK_TYPE_TTS) { + GGML_ASSERT(has_cap_tts()); // should already checked in route handler + if (!slot.tts.is_supported()) { @@ -458,20 +428,20 @@ index 543dc43..4f01b1e 100644 // initialize samplers if (task.need_sampling()) { try { -@@ -1875,6 +1937,9 @@ private: +@@ -1869,6 +1931,9 @@ private: // TODO: getting pre sampling logits is not yet supported with backend sampling - backend_sampling &= !need_pre_sample_logits; - + use_backend_sampling &= !need_pre_sample_logits; + + // TODO: check verify if this actually works with TTS -+ backend_sampling &= task.type != SERVER_TASK_TYPE_TTS; ++ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS; + // TODO: tmp until backend sampling is fully implemented - if (backend_sampling) { + if (use_backend_sampling) { llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get())); -@@ -1890,9 +1955,13 @@ private: - +@@ -1884,9 +1949,13 @@ private: + slot.task = std::make_unique(std::move(task)); - + - slot.state = slot.task->is_child() - ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt - : SLOT_STATE_STARTED; @@ -482,13 +452,13 @@ index 543dc43..4f01b1e 100644 + ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt + : SLOT_STATE_STARTED; + } - + // reset server kill-switch counter n_empty_consecutive = 0; -@@ -2169,6 +2238,18 @@ private: +@@ -2163,6 +2232,18 @@ private: queue_results.send(std::move(res)); } - + + void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) { + auto res = std::make_unique(); + @@ -503,8 +473,8 @@ index 543dc43..4f01b1e 100644 + void send_final_response(server_slot & slot) { auto res = std::make_unique(); - -@@ -2668,6 +2749,7 @@ private: + +@@ -2662,6 +2743,7 @@ private: case SERVER_TASK_TYPE_EMBEDDING: case SERVER_TASK_TYPE_RERANK: case SERVER_TASK_TYPE_SCORE: @@ -512,10 +482,10 @@ index 543dc43..4f01b1e 100644 { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -3103,6 +3185,14 @@ private: +@@ -3097,6 +3179,14 @@ private: abort_all_slots("pre_decode() failed: " + std::string(e.what())); } - + + // note: TTS slots bypass the shared batch entirely + try { + process_tts_slots(); @@ -525,12 +495,12 @@ index 543dc43..4f01b1e 100644 + } + GGML_ASSERT(batch.slot_batched || batch.size() == 0); - + if (batch.slot_batched) { -@@ -3173,10 +3263,77 @@ private: +@@ -3167,10 +3257,77 @@ private: } } - + + void process_tts_slots() { + iterate(slots, [&](server_slot & slot) { + if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) { @@ -605,25 +575,25 @@ index 543dc43..4f01b1e 100644 if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { if (!params_base.ctx_shift) { // this check is redundant (for good) -@@ -3249,7 +3406,7 @@ private: - +@@ -3243,7 +3400,7 @@ private: + // determine which slots are generating and drafting iterate(slots, [&](server_slot & slot) { - if (slot.state != SLOT_STATE_GENERATING) { + if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) { return; } - -@@ -3381,7 +3538,7 @@ private: + +@@ -3375,7 +3532,7 @@ private: return; // batch is full, skip remaining slots } - + - if (!slot.is_processing()) { + if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) { return; } - -@@ -4390,6 +4547,8 @@ server_context_meta server_context::get_meta() const { + +@@ -4384,6 +4541,8 @@ server_context_meta server_context::get_meta() const { /* has_inp_image */ impl->chat_params.allow_image, /* has_inp_audio */ impl->chat_params.allow_audio, /* has_inp_video */ impl->chat_params.allow_video, @@ -632,22 +602,22 @@ index 543dc43..4f01b1e 100644 /* json_ui_settings */ impl->json_ui_settings, /* slot_n_ctx */ impl->get_slot_n_ctx(), /* pooling_type */ llama_pooling_type(impl->ctx_tgt), -@@ -4469,6 +4628,11 @@ std::unique_ptr server_routes::handle_completions_impl( - +@@ -4463,6 +4622,11 @@ std::unique_ptr server_routes::handle_completions_impl( + res->set_req(&req); // will also set spipe if needed - + + if (!ctx_server.has_cap_chat()) { + res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED)); + return res; + } + int32_t sse_ping_interval = params.sse_ping_interval; - + try { -@@ -5446,6 +5610,150 @@ void server_routes::init_routes() { +@@ -5440,6 +5604,150 @@ void server_routes::init_routes() { return res; }; - + + this->post_tts = [this](const server_http_req & req) { + auto res = create_response(); + res->set_req(&req); // will also set spipe if needed @@ -794,7 +764,7 @@ index 543dc43..4f01b1e 100644 + this->get_lora_adapters = [this](const server_http_req & req) { auto res = create_response(); - + diff --git a/tools/server/server-context.h b/tools/server/server-context.h index f9ab113..6105126 100644 --- a/tools/server/server-context.h @@ -815,7 +785,7 @@ index f9ab113..6105126 100644 + server_http_context::handler_t post_tts; server_http_context::handler_t get_lora_adapters; server_http_context::handler_t post_lora_adapters; - + diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp index 1ee6775..939630b 100644 --- a/tools/server/server-task.cpp @@ -823,7 +793,7 @@ index 1ee6775..939630b 100644 @@ -1523,6 +1523,17 @@ json server_task_result_rerank::to_json() { }; } - + +// +// server_task_result_tts +// @@ -843,30 +813,30 @@ index 5bedf19..e6ca67a 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -10,6 +10,7 @@ - + // TODO: prevent including the whole server-common.h as we only use server_tokens #include "server-common.h" +#include "mtmd-helper.h" - + using json = nlohmann::ordered_json; - + @@ -42,6 +43,7 @@ enum server_task_type { SERVER_TASK_TYPE_SLOT_ERASE, SERVER_TASK_TYPE_GET_LORA, SERVER_TASK_TYPE_SET_LORA, + SERVER_TASK_TYPE_TTS, }; - + // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common @@ -202,6 +204,9 @@ struct server_task { // used by SERVER_TASK_TYPE_SET_LORA std::map set_lora; // mapping adapter ID -> scale - + + // used by SERVER_TASK_TYPE_TTS + mtmd_helper::gen_audio::inp tts_inp; + server_task() = default; - + server_task(server_task_type type) : type(type) {} @@ -235,6 +240,7 @@ struct server_task { switch (type) { @@ -879,7 +849,7 @@ index 5bedf19..e6ca67a 100644 @@ -514,6 +520,16 @@ struct server_task_result_embd : server_task_result { json to_json_oaicompat(); }; - + +struct server_task_result_tts : server_task_result { + std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type) + int32_t sample_rate = 0; @@ -892,4 +862,6 @@ index 5bedf19..e6ca67a 100644 + struct server_task_result_rerank : server_task_result { float score = -1e6; - + +-- +2.39.5