mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-09-12 22:33:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

chore: ⬆️ Update ggml-org/llama.cpp to 030ebb558a5820b444a8f836ed5cdd46c9b4bd7a (#11454)

Browse Source 
* ⬆️ Update ggml-org/llama.cpp

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>

* fix(llama-cpp): rebase server patches

Adapt score output limits and TTS backend sampling to the updated llama.cpp server APIs.

Assisted-by: Codex:gpt-5.4

---------

Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: mudler <2420543+mudler@users.noreply.github.com>
Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
mudler's LocalAI [bot]mudlerlocalai-org-maint-bot
 authored and GitHub committed 2026-08-11 09:52:24 +02:00
1 parent 129f834039
commit 7cfccdc2bf
3 files changed
+158 -175
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/cpp/llama-cpp/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
LLAMA_VERSION?=9de0fcf2b3e587a43f293d9a2b6ec0a32991f768
LLAMA_VERSION?=030ebb558a5820b444a8f836ed5cdd46c9b4bd7a
LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
CMAKE_ARGS?=
backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch
+48 -37
View File
@@ -1,8 +1,21 @@
From 75220a0d74892e3315f4042274b1efa6195868d8 Mon Sep 17 00:00:00 2001
From: Codex <codex@local>
Date: Mon, 10 Aug 2026 23:05:52 +0000
Subject: [PATCH 1/2] score-patch
---
common/common.cpp | 6 +-
common/common.h | 3 +
tools/CMakeLists.txt | 1 +
tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++-
tools/server/server-task.h | 47 +++++
5 files changed, 406 insertions(+), 9 deletions(-)
diff --git a/common/common.cpp b/common/common.cpp
index 8f13217..fc584e1 100644
index 2e3f14c..0cec0dc 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1591,8 +1591,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
@@ -1636,8 +1636,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
auto cparams = llama_context_default_params();
cparams.n_ctx = params.n_ctx;
@@ -13,13 +26,13 @@ index 8f13217..fc584e1 100644
+ cparams.n_seq_max = params.n_parallel + params.n_seq_score_forks;
+ cparams.n_rs_seq = std::max(params.speculative.need_n_rs_seq(), (uint32_t) std::max(0, params.n_rs_seq));
cparams.n_outputs_max = std::max(params.n_outputs_max, 0);
cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0);
cparams.n_batch = params.n_batch;
cparams.n_ubatch = params.n_ubatch;
diff --git a/common/common.h b/common/common.h
index bffc176..e313bd6 100644
index 878534d..4001df2 100644
--- a/common/common.h
+++ b/common/common.h
@@ -455,6 +455,9 @@ struct common_params {
@@ -445,6 +445,9 @@ struct common_params {
int32_t n_keep = 0; // number of tokens to keep from initial prompt
int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
int32_t n_parallel = 1; // number of parallel sequences to decode
@@ -28,7 +41,7 @@ index bffc176..e313bd6 100644
+ bool score_enabled = false; // reserve server resources for the Score task type
int32_t n_sequences = 1; // number of sequences to decode
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
int32_t grp_attn_n = 1; // group-attention factor
int32_t n_outputs_max_per_seq = 1; // max outputs per sequence
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 780df32..1d2fe8f 100644
--- a/tools/CMakeLists.txt
@@ -39,28 +52,24 @@ index 780df32..1d2fe8f 100644
endif()
+add_subdirectory(grpc-server)
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 715477e..de5bed8 100644
index 3b5f6a1..d0e18e6 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -49,7 +49,16 @@ static uint32_t server_n_outputs_max(const common_params & params) {
@@ -48,6 +48,13 @@ static common_speculative_output_limits server_output_limits(const common_params
auto result = common_speculative_get_output_limits(
params.n_batch, params.n_parallel, common_speculative_n_max(&params.speculative));
const uint32_t n_outputs_per_seq = 1 + common_speculative_n_max(&params.speculative);
- const uint64_t n_outputs = (uint64_t) params.n_parallel * n_outputs_per_seq;
+ // score tasks (SERVER_TASK_TYPE_SCORE) output logits for every candidate
+ // token, so reserve room for a bounded candidate tail per parallel slot
+ if (!params.score_enabled) {
+ return std::max<uint32_t>(1, std::min<uint64_t>(n_batch,
+ (uint64_t) params.n_parallel * n_outputs_per_seq));
+ // Score tasks output logits for every candidate token, so reserve room
+ // for a bounded candidate tail per parallel slot.
+ if (params.score_enabled) {
+ result.per_seq = std::max<int32_t>(result.per_seq, 1 + SERVER_SCORE_MAX_CAND_TOKENS);
+ result.total = std::min<int32_t>(params.n_batch, params.n_parallel * result.per_seq);
+ }
+
+ const uint32_t n_outputs_score_seq = 1 + SERVER_SCORE_MAX_CAND_TOKENS;
+
+ const uint64_t n_outputs = (uint64_t) params.n_parallel * std::max(n_outputs_per_seq, n_outputs_score_seq);
return std::max<uint32_t>(1, std::min<uint64_t>(n_batch, n_outputs));
}
@@ -202,6 +211,26 @@ struct server_slot {
result.total = std::max<int32_t>(1, result.total);
result.per_seq = std::max<int32_t>(1, result.per_seq);
return result;
@@ -239,6 +246,26 @@ struct server_slot {
std::vector<completion_token_output> generated_token_probs;
@@ -87,7 +96,7 @@ index 715477e..de5bed8 100644
bool has_next_token = true;
bool has_new_line = false;
bool truncated = false;
@@ -311,6 +340,10 @@ struct server_slot {
@@ -351,6 +378,10 @@ struct server_slot {
}
generated_tokens.clear();
generated_token_probs.clear();
@@ -98,7 +107,7 @@ index 715477e..de5bed8 100644
json_schema = json();
// clear speculative decoding stats
@@ -2205,6 +2238,229 @@ private:
@@ -2271,6 +2302,229 @@ private:
queue_results.send(std::move(res));
}
@@ -328,7 +337,7 @@ index 715477e..de5bed8 100644
//
// Functions to process the task
//
@@ -2341,6 +2597,7 @@ private:
@@ -2407,6 +2661,7 @@ private:
case SERVER_TASK_TYPE_INFILL:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
@@ -336,7 +345,7 @@ index 715477e..de5bed8 100644
{
// special case: if input is provided via CLI, tokenize it first
// otherwise, no need to tokenize as it's already done inside the HTTP thread
@@ -2832,6 +3089,13 @@ private:
@@ -2903,6 +3158,13 @@ private:
break; // stop any further processing
}
}
@@ -350,7 +359,7 @@ index 715477e..de5bed8 100644
}
void pre_decode() {
@@ -3154,6 +3418,16 @@ private:
@@ -3222,6 +3484,16 @@ private:
n_past = std::min(n_past, slot.alora_invocation_start - 1);
}
@@ -367,7 +376,7 @@ index 715477e..de5bed8 100644
const auto n_cache_reuse = slot.task->params.n_cache_reuse;
const bool can_cache_reuse =
@@ -3395,8 +3669,12 @@ private:
@@ -3455,8 +3727,12 @@ private:
bool do_checkpoint = params_base.n_ctx_checkpoints > 0;
@@ -382,7 +391,7 @@ index 715477e..de5bed8 100644
// make a checkpoint of the parts of the memory that cannot be rolled back.
// checkpoints are created only if:
@@ -3463,10 +3741,17 @@ private:
@@ -3523,10 +3799,17 @@ private:
// embedding requires all tokens in the batch to be output;
// MTP also wants logits at every prompt position so the
// streaming hook can mirror t_h_nextn into ctx_dft.
@@ -401,7 +410,7 @@ index 715477e..de5bed8 100644
slot.prompt.tokens.push_back(cur_tok);
slot.n_prompt_tokens_processed++;
@@ -3481,6 +3766,32 @@ private:
@@ -3541,6 +3824,32 @@ private:
}
}
@@ -434,7 +443,7 @@ index 715477e..de5bed8 100644
// process the last few tokens of the prompt separately in order to allow for a checkpoint to be created.
// create checkpoints that many tokens before the end of the prompt:
// - 4 + n_ubatch
@@ -3513,6 +3824,15 @@ private:
@@ -3573,6 +3882,15 @@ private:
const bool is_user_start = spans.is_user_start(n_tokens_start);
const bool is_last_user_message = n_tokens_start == last_user_pos;
@@ -450,7 +459,7 @@ index 715477e..de5bed8 100644
// entire prompt has been processed
if (slot.prompt.n_tokens() == slot.task->n_tokens()) {
slot.state = SLOT_STATE_DONE_PROMPT;
@@ -3528,8 +3848,8 @@ private:
@@ -3588,8 +3906,8 @@ private:
slot.init_sampler();
} else {
// skip ordinary mid-prompt checkpoints, unless the batch starts a user
@@ -461,7 +470,7 @@ index 715477e..de5bed8 100644
do_checkpoint = false;
}
}
@@ -3546,10 +3866,10 @@ private:
@@ -3606,10 +3924,10 @@ private:
// do not checkpoint after mtmd chunks
do_checkpoint = do_checkpoint && !has_mtmd;
@@ -474,7 +483,7 @@ index 715477e..de5bed8 100644
n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
@@ -3703,6 +4023,13 @@ private:
@@ -3772,6 +4090,13 @@ private:
}
}
@@ -488,7 +497,7 @@ index 715477e..de5bed8 100644
if (!is_inside_view(slot.i_batch)) {
// the required token not in this sub-batch, skip
return;
@@ -3724,6 +4051,25 @@ private:
@@ -3793,6 +4118,25 @@ private:
return;
}
@@ -515,7 +524,7 @@ index 715477e..de5bed8 100644
// prompt evaluated for next-token prediction
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index c3eea2e..fb3c178 100644
index 6275ec7..5bedf19 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -13,10 +13,25 @@
@@ -597,3 +606,5 @@ index c3eea2e..fb3c178 100644
struct server_task_result_error : server_task_result {
error_type err_type = ERROR_TYPE_SERVER;
std::string err_msg;
--
2.39.5
backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch
+109 -137
View File
@@ -1,57 +1,25 @@
# Carries the server-side half of ggml-org/llama.cpp#26603 (POST /tts), which
# adds SERVER_TASK_TYPE_TTS to the shared server_context. LocalAI's gRPC
# adapter rides on that same server_context and cannot drive the mtmd gen-audio
# loop directly: server_context owns the llama_context and schedules slots on
# its own thread, so a handler calling llama_decode would corrupt state.
#
# The upstream HTTP route (tools/server/server.cpp) and the README/CLI hunks are
# intentionally excluded - LocalAI does not serve llama.cpp's HTTP surface.
#
# Rebased against LLAMA_VERSION 9de0fcf2b with 0001-add-server-task-type-score.patch
# already applied: the SERVER_TASK_TYPE_TTS case in the tokenize switch lands
# after the SCORE case that patch adds, so this patch must stay second in ls order.
#
# Three fixes on top of the draft, all ours, all candidates to send back to #26603:
#
# 1. Its lone SRV_WRN call passes only a format string, but the macro expands
# __VA_ARGS__ without the GNU comma-elision extension, so the expansion ends
# in a trailing comma and does not compile. The "%s" wrapper added here is the
# same idiom upstream already uses for its other argument-less SRV_WRN calls;
# drop it if the draft fixes the call before it merges.
#
# 2. The slot.prompt_clear() added to the SERVER_TASK_TYPE_TTS branch of
# launch_slot_with_task. Without it only the FIRST TTS request in a server
# process succeeds and every later one fails instantly in step_prompt. TTS
# slots never enter the shared batch (pre_decode() returns early for them and
# process_tts_slots() drives them), so they skip the prompt-cache bookkeeping
# that would otherwise clear the sequence between requests; meanwhile the
# gen-audio pipeline always decodes from position 0 and its reset() only
# clears host-side buffers, never the KV cache. The result is that request 2
# decodes over request 1's tokens. This one is a genuine defect in the draft,
# not a LocalAI integration artifact, and should be reported upstream.
#
# 3. The is_supported() guard in server_slot::tts_ctx::reset(). The draft calls
# mtmd_helper_gen_audio_reset() unconditionally, but it only ever init()s the
# gen-audio pipeline for models that carry a gen-audio mmproj, so the handle
# stays null for every ordinary model. Upstream's implementation
# (tools/mtmd/mtmd-helper-gen.cpp) reads ctx->pipeline before it null-checks
# anything, so the slot-init call to server_slot::reset() segfaults the
# process on any non-TTS model - which is every chat model LocalAI loads
# through this backend. The missing null check on mtmd_helper_gen_audio_*
# is upstream's bug, so carry this guard until they add one, even if the
# draft's own reset() call is fixed.
#
# REMOVE THIS PATCH once #26603 merges upstream and LLAMA_VERSION is bumped past
# the merge commit. It exists only because that PR is still a draft. If it merges
# without fixes 2 and 3 above, those hunks still need carrying.
From 861fb06531e770fcea65e86296f80eba830dac30 Mon Sep 17 00:00:00 2001
From: Codex <codex@local>
Date: Mon, 10 Aug 2026 23:05:53 +0000
Subject: [PATCH 2/2] tts-patch
---
tools/mtmd/mtmd-helper-gen.cpp | 118 +++++++++---
tools/mtmd/mtmd-helper.h | 47 ++++-
tools/server/server-context.cpp | 322 +++++++++++++++++++++++++++++++-
tools/server/server-context.h | 3 +
tools/server/server-task.cpp | 11 ++
tools/server/server-task.h | 16 ++
6 files changed, 481 insertions(+), 36 deletions(-)
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index b52dc8e..fd9d6ca 100644
index 85671d1..66ccf0a 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -48,29 +48,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
return LLAMA_TOKEN_NULL;
}
+static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
+ const char * c = (const char *) p;
+ buf.insert(buf.end(), c, c + n);
@@ -104,17 +72,17 @@ index b52dc8e..fd9d6ca 100644
+ append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
@@ -89,6 +98,8 @@ public:
// those read what they need from h_state_in instead
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
+ // forces any buffered codes through code2wav now, regardless of window_frames
+ virtual int32_t flush() = 0;
protected:
llama_context * lctx;
@@ -119,6 +130,9 @@ public:
@@ -118,6 +129,9 @@ public:
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
@@ -122,19 +90,19 @@ index b52dc8e..fd9d6ca 100644
+ pcm_sent = 0;
+ wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -204,6 +218,7 @@ public:
@@ -203,6 +217,7 @@ public:
top_k = inp->top_k > 0 ? inp->top_k : 50;
top_p = inp->top_p > 0 ? inp->top_p : 1.0f;
out_type = inp->out_type;
+ stream = inp->stream;
// the text stream keeps flowing during generation: after frame k, the input adds
// trailing text row k on top of the codes embedding, then tts_eos, then tts_pad
@@ -289,31 +304,60 @@ public:
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -284,31 +299,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
- if (!flush_gen_wav()) {
- return 1;
@@ -162,7 +130,7 @@ index b52dc8e..fd9d6ca 100644
+ *out_data_len = out_buf.size();
+ return 0;
}
- *out_sample_rate = info.sample_rate;
+ // streaming: only return audio produced since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
@@ -170,7 +138,7 @@ index b52dc8e..fd9d6ca 100644
- *out_n_samples = (int64_t) audio_pcm.size();
+ *out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
- *out_data = (const char *) audio_pcm.data();
- *out_data_len = audio_pcm.size() * sizeof(float);
@@ -179,7 +147,7 @@ index b52dc8e..fd9d6ca 100644
+ pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
@@ -194,7 +162,7 @@ index b52dc8e..fd9d6ca 100644
*out_data_len = out_buf.size();
return 0;
}
+ int32_t flush() override {
+ return flush_gen_wav() ? 0 : 1;
+ }
@@ -202,7 +170,7 @@ index b52dc8e..fd9d6ca 100644
private:
bool ensure_cache() {
if (specials_ok) {
@@ -357,7 +401,7 @@ private:
@@ -352,7 +396,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
@@ -211,7 +179,7 @@ index b52dc8e..fd9d6ca 100644
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -442,6 +486,9 @@ private:
@@ -436,6 +480,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
@@ -219,12 +187,12 @@ index b52dc8e..fd9d6ca 100644
+ size_t pcm_sent = 0; // samples already returned by get_output()
+ bool wav_header_sent = false;
};
static std::unique_ptr<mtmd_gen_audio_pipeline> make_pipeline(llama_context * lctx, mtmd_context * mctx) {
@@ -473,6 +520,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
@@ -467,6 +514,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
}
}
+struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
+ mtmd_helper_gen_audio_inp inp{};
+ inp.top_k = 50;
@@ -236,7 +204,7 @@ index b52dc8e..fd9d6ca 100644
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
@@ -503,3 +558,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
@@ -497,3 +552,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
@@ -257,19 +225,19 @@ index 7e5cf9b..1f3ec01 100644
struct mtmd_helper_gen_audio_inp {
+ bool stream; // if true, output() must be called after each step_gen()
llama_seq_id seq_id;
const char * prompt;
@@ -189,6 +190,8 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_outtype out_type;
};
+MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
+
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
struct llama_context * lctx,
struct mtmd_context * mctx);
@@ -217,6 +220,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
// out_data valid until next get_output() or reset() call
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
+// if inp->stream is true: returns only audio produced since the previous call, and
@@ -280,7 +248,7 @@ index 7e5cf9b..1f3ec01 100644
@@ -224,6 +229,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
size_t * out_data_len,
int64_t * out_n_samples);
+// forces any buffered codes through code2wav now, regardless of window_frames;
+// call once when generation has ended, before the last get_output() in stream mode
+MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
@@ -339,10 +307,10 @@ index 7e5cf9b..1f3ec01 100644
+ return mtmd_helper_gen_audio_flush(ctx.get());
+ }
};
} // namespace mtmd_helper
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 543dc43..4f01b1e 100644
index d0e18e6..0765065 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -16,6 +16,7 @@
@@ -350,23 +318,25 @@ index 543dc43..4f01b1e 100644
#include "mtmd.h"
#include "mtmd-helper.h"
+#include "base64.hpp"
#include <algorithm>
#include <cstddef>
@@ -43,7 +44,8 @@ static uint32_t server_n_outputs_max(const common_params & params) {
const uint32_t n_batch = params.n_batch;
@@ -41,8 +42,9 @@ constexpr int HTTP_POLLING_SECONDS = 1;
static common_speculative_output_limits server_output_limits(const common_params & params) {
if (params.embedding ||
- (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
- return { params.n_batch, 1 };
+ (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE) ||
+ !params.mmproj.path.empty()) { // gen-audio (TTS) capability isn't known until the mmproj loads, size generously
return n_batch;
+ return { params.n_batch, params.n_batch };
}
@@ -214,6 +216,30 @@ struct server_slot {
auto result = common_speculative_get_output_limits(
@@ -211,6 +213,30 @@ struct server_slot {
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
+ struct tts_ctx {
+ mtmd_helper::gen_audio ctx;
+ const float * h_state;
@@ -393,20 +363,20 @@ index 543dc43..4f01b1e 100644
+
// speculative decoding
common_speculative * spec;
@@ -403,6 +429,8 @@ struct server_slot {
@@ -400,6 +426,8 @@ struct server_slot {
// clear multimodal state
mbatch.reset();
+
+ tts.reset();
}
void init_sampler() const {
@@ -949,6 +977,14 @@ public:
@@ -946,6 +974,14 @@ public:
mtmd_context * mctx = nullptr;
const llama_vocab * vocab = nullptr;
+ bool has_cap_tts() const {
+ return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
+ }
@@ -417,22 +387,22 @@ index 543dc43..4f01b1e 100644
+
server_queue queue_tasks;
server_response queue_results;
@@ -1400,6 +1436,10 @@ private:
@@ -1399,6 +1435,10 @@ private:
slot.mctx = mctx;
slot.prompt.tokens.has_mtmd = mctx != nullptr;
+ if (has_cap_tts()) {
+ slot.tts.ctx.init(ctx_tgt, mctx);
+ }
+
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
slot.callback_on_release = [this](int id_slot) {
@@ -1853,6 +1893,28 @@ private:
@@ -1852,6 +1892,28 @@ private:
SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
+ if (task.type == SERVER_TASK_TYPE_TTS) {
+ GGML_ASSERT(has_cap_tts()); // should already checked in route handler
+ if (!slot.tts.is_supported()) {
@@ -458,20 +428,20 @@ index 543dc43..4f01b1e 100644
// initialize samplers
if (task.need_sampling()) {
try {
@@ -1875,6 +1937,9 @@ private:
@@ -1869,6 +1931,9 @@ private:
// TODO: getting pre sampling logits is not yet supported with backend sampling
backend_sampling &= !need_pre_sample_logits;
use_backend_sampling &= !need_pre_sample_logits;
+ // TODO: check verify if this actually works with TTS
+ backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
+ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
+
// TODO: tmp until backend sampling is fully implemented
if (backend_sampling) {
if (use_backend_sampling) {
llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
@@ -1890,9 +1955,13 @@ private:
@@ -1884,9 +1949,13 @@ private:
slot.task = std::make_unique<const server_task>(std::move(task));
- slot.state = slot.task->is_child()
- ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
- : SLOT_STATE_STARTED;
@@ -482,13 +452,13 @@ index 543dc43..4f01b1e 100644
+ ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
+ : SLOT_STATE_STARTED;
+ }
// reset server kill-switch counter
n_empty_consecutive = 0;
@@ -2169,6 +2238,18 @@ private:
@@ -2163,6 +2232,18 @@ private:
queue_results.send(std::move(res));
}
+ void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {
+ auto res = std::make_unique<server_task_result_tts>();
+
@@ -503,8 +473,8 @@ index 543dc43..4f01b1e 100644
+
void send_final_response(server_slot & slot) {
auto res = std::make_unique<server_task_result_cmpl_final>();
@@ -2668,6 +2749,7 @@ private:
@@ -2662,6 +2743,7 @@ private:
case SERVER_TASK_TYPE_EMBEDDING:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_SCORE:
@@ -512,10 +482,10 @@ index 543dc43..4f01b1e 100644
{
// special case: if input is provided via CLI, tokenize it first
// otherwise, no need to tokenize as it's already done inside the HTTP thread
@@ -3103,6 +3185,14 @@ private:
@@ -3097,6 +3179,14 @@ private:
abort_all_slots("pre_decode() failed: " + std::string(e.what()));
}
+ // note: TTS slots bypass the shared batch entirely
+ try {
+ process_tts_slots();
@@ -525,12 +495,12 @@ index 543dc43..4f01b1e 100644
+ }
+
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
if (batch.slot_batched) {
@@ -3173,10 +3263,77 @@ private:
@@ -3167,10 +3257,77 @@ private:
}
}
+ void process_tts_slots() {
+ iterate(slots, [&](server_slot & slot) {
+ if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
@@ -605,25 +575,25 @@ index 543dc43..4f01b1e 100644
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
if (!params_base.ctx_shift) {
// this check is redundant (for good)
@@ -3249,7 +3406,7 @@ private:
@@ -3243,7 +3400,7 @@ private:
// determine which slots are generating and drafting
iterate(slots, [&](server_slot & slot) {
- if (slot.state != SLOT_STATE_GENERATING) {
+ if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -3381,7 +3538,7 @@ private:
@@ -3375,7 +3532,7 @@ private:
return; // batch is full, skip remaining slots
}
- if (!slot.is_processing()) {
+ if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -4390,6 +4547,8 @@ server_context_meta server_context::get_meta() const {
@@ -4384,6 +4541,8 @@ server_context_meta server_context::get_meta() const {
/* has_inp_image */ impl->chat_params.allow_image,
/* has_inp_audio */ impl->chat_params.allow_audio,
/* has_inp_video */ impl->chat_params.allow_video,
@@ -632,22 +602,22 @@ index 543dc43..4f01b1e 100644
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->get_slot_n_ctx(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
@@ -4469,6 +4628,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
@@ -4463,6 +4622,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
res->set_req(&req); // will also set spipe if needed
+ if (!ctx_server.has_cap_chat()) {
+ res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
int32_t sse_ping_interval = params.sse_ping_interval;
try {
@@ -5446,6 +5610,150 @@ void server_routes::init_routes() {
@@ -5440,6 +5604,150 @@ void server_routes::init_routes() {
return res;
};
+ this->post_tts = [this](const server_http_req & req) {
+ auto res = create_response();
+ res->set_req(&req); // will also set spipe if needed
@@ -794,7 +764,7 @@ index 543dc43..4f01b1e 100644
+
this->get_lora_adapters = [this](const server_http_req & req) {
auto res = create_response();
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index f9ab113..6105126 100644
--- a/tools/server/server-context.h
@@ -815,7 +785,7 @@ index f9ab113..6105126 100644
+ server_http_context::handler_t post_tts;
server_http_context::handler_t get_lora_adapters;
server_http_context::handler_t post_lora_adapters;
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index 1ee6775..939630b 100644
--- a/tools/server/server-task.cpp
@@ -823,7 +793,7 @@ index 1ee6775..939630b 100644
@@ -1523,6 +1523,17 @@ json server_task_result_rerank::to_json() {
};
}
+//
+// server_task_result_tts
+//
@@ -843,30 +813,30 @@ index 5bedf19..e6ca67a 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -10,6 +10,7 @@
// TODO: prevent including the whole server-common.h as we only use server_tokens
#include "server-common.h"
+#include "mtmd-helper.h"
using json = nlohmann::ordered_json;
@@ -42,6 +43,7 @@ enum server_task_type {
SERVER_TASK_TYPE_SLOT_ERASE,
SERVER_TASK_TYPE_GET_LORA,
SERVER_TASK_TYPE_SET_LORA,
+ SERVER_TASK_TYPE_TTS,
};
// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -202,6 +204,9 @@ struct server_task {
// used by SERVER_TASK_TYPE_SET_LORA
std::map<int, float> set_lora; // mapping adapter ID -> scale
+ // used by SERVER_TASK_TYPE_TTS
+ mtmd_helper::gen_audio::inp tts_inp;
+
server_task() = default;
server_task(server_task_type type) : type(type) {}
@@ -235,6 +240,7 @@ struct server_task {
switch (type) {
@@ -879,7 +849,7 @@ index 5bedf19..e6ca67a 100644
@@ -514,6 +520,16 @@ struct server_task_result_embd : server_task_result {
json to_json_oaicompat();
};
+struct server_task_result_tts : server_task_result {
+ std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
+ int32_t sample_rate = 0;
@@ -892,4 +862,6 @@ index 5bedf19..e6ca67a 100644
+
struct server_task_result_rerank : server_task_result {
float score = -1e6;
--
2.39.5