mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-10-28 09:08:54 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(gallery): add flint and bump llama.cpp to a37cc461 (#12854)

Browse Source 
* chore(llama-cpp): bump to a37cc461 and drop the TTS patch

Move the llama.cpp pin to a37cc461, the merge of the Flint
architecture (ggml-org/llama.cpp#30928).

The TTS task patch is gone: upstream now carries
SERVER_TASK_TYPE_TTS, server_task::tts_inp, server_task_result_tts,
server_context_meta::has_cap_tts and the mtmd gen-audio helpers. The
patch is detected as already applied on the new pin. The gRPC server
uses the same symbols. The bonsai and turboquant forks still lack
them and keep using disable-tts-task.sh, so they are unchanged.

The score task patch is still needed: upstream has no
SERVER_TASK_TYPE_SCORE. It is regenerated against the new pin and
applies with no offset or fuzz. Upstream added
SERVER_TASK_TYPE_TRANSCRIBE_RT to server_task::need_logits(), which
broke one hunk, so SCORE is added next to it by hand.

The C++ backend was not built for this change.

Assisted-by: Claude:claude-sonnet-5-5

* feat(gallery): add flint

Add Recursive Flint, a 39.8B mixture-of-experts coding model with
about 2.5B active parameters, from the Unsloth GGUF repository. It
runs on llama.cpp with the Flint architecture support added upstream
in ggml-org/llama.cpp#30928.

Three quantizations are listed, with the default declaring the others
as variants: UD-Q4_K_XL (25 GB), UD-Q2_K_XL (13.6 GB) for small hosts
and UD-Q6_K_XL (36 GB) for larger ones. Sampling defaults follow the
model card (temperature 1, top_p 0.95, top_k 20). The library file
gallery/models/index.yaml is edited and gallery/index.yaml is
regenerated from it.

Assisted-by: Claude:claude-sonnet-5-5

---------

Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
mudler-agentandEttore Di Giacinto authored and GitHub committed 2026-10-23 15:58:25 +02:00
1 parent 08de1cb919
commit 86994fac46
5 files changed
+313 -905
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch
+54 -54
View File
@@ -1,10 +1,10 @@
diff --git a/common/common.cpp b/common/common.cpp
index aca1949..428b120 100644
index 6ef3b78..037a00b 100644
--- a/common/common.cpp
+++ b/common/common.cpp
@@ -1662,8 +1662,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
@@ -1716,8 +1716,10 @@ struct llama_context_params common_context_params_to_llama(const common_params &
auto cparams = llama_context_default_params();
cparams.n_ctx = params.n_ctx;
- cparams.n_seq_max = params.n_parallel;
- cparams.n_rs_seq = params.speculative.need_n_rs_seq();
@@ -16,10 +16,10 @@ index aca1949..428b120 100644
cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0);
cparams.n_batch = params.n_batch;
diff --git a/common/common.h b/common/common.h
index 04ffbcd..5daa023 100644
index 70d6a67..93e2269 100644
--- a/common/common.h
+++ b/common/common.h
@@ -455,6 +455,9 @@ struct common_params {
@@ -460,6 +460,9 @@ struct common_params {
int32_t n_keep = 0; // number of tokens to keep from initial prompt
int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited)
int32_t n_parallel = 1; // number of parallel sequences to decode
@@ -30,13 +30,13 @@ index 04ffbcd..5daa023 100644
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
int32_t n_outputs_max_per_seq = 1; // max outputs per sequence
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index edb8e2d..9c88feb 100644
index 296893f..ef38f90 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -48,6 +48,13 @@ static common_speculative_output_limits server_output_limits(const common_params
@@ -60,6 +60,13 @@ static common_speculative_output_limits server_output_limits(const common_params
auto result = common_speculative_get_output_limits(
params.n_batch, params.n_parallel, common_speculative_n_max(&params.speculative));
+ // Score tasks output logits for every candidate token, so reserve room
+ // for a bounded candidate tail per parallel slot.
+ if (params.score_enabled) {
@@ -47,10 +47,10 @@ index edb8e2d..9c88feb 100644
result.total = std::max<int32_t>(1, result.total);
result.per_seq = std::max<int32_t>(1, result.per_seq);
return result;
@@ -243,6 +250,26 @@ struct server_slot {
@@ -325,6 +332,26 @@ struct server_slot {
std::vector<completion_token_output> generated_token_probs;
+ // SERVER_TASK_TYPE_SCORE: shared-prefix token logprobs harvested
+ // incrementally across batch views (NaN = not yet produced)
+ std::vector<float> score_logprobs;
@@ -74,7 +74,7 @@ index edb8e2d..9c88feb 100644
bool has_next_token = true;
bool has_new_line = false;
bool truncated = false;
@@ -346,6 +373,10 @@ struct server_slot {
@@ -433,6 +460,10 @@ struct server_slot {
}
generated_tokens.clear();
generated_token_probs.clear();
@@ -83,12 +83,12 @@ index edb8e2d..9c88feb 100644
+ score_suffix_pending = false;
+ score_divergence = -1;
json_schema = json();
task_prev = std::move(task);
@@ -2308,6 +2339,227 @@ private:
@@ -2571,6 +2602,227 @@ private:
queue_results.send(std::move(res));
}
+ // log(sum(exp(logits))) with max-subtraction for stability — the
+ // log_softmax denominator shared by every token read from one output
+ static double score_log_denom(const float * logits, int32_t n_vocab) {
@@ -313,15 +313,15 @@ index edb8e2d..9c88feb 100644
//
// Functions to process the task
//
@@ -2468,6 +2720,7 @@ private:
case SERVER_TASK_TYPE_EMBEDDING:
@@ -2957,6 +3209,7 @@ private:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_TTS:
case SERVER_TASK_TYPE_DECISION:
+ case SERVER_TASK_TYPE_SCORE:
{
// special case: if input is provided via CLI, tokenize it first
// otherwise, no need to tokenize as it's already done inside the HTTP thread
@@ -2986,6 +3239,13 @@ private:
@@ -3549,6 +3802,13 @@ private:
break; // stop any further processing
}
}
@@ -333,12 +333,12 @@ index edb8e2d..9c88feb 100644
+ abort_all_slots("update_score_suffixes() failed: " + std::string(e.what()));
+ }
}
void pre_decode() {
@@ -3325,6 +3585,16 @@ private:
void process_tts_slots() {
@@ -3987,6 +4247,16 @@ private:
n_past = std::min(n_past, slot.alora_invocation_start - 1);
}
+ // score tasks need the logits that predict the first candidate
+ // token, so the last shared-prompt token must be (re-)decoded
+ // even when the cache already covers it
@@ -350,12 +350,12 @@ index edb8e2d..9c88feb 100644
+ }
+
const auto n_cache_reuse = slot.task->params.n_cache_reuse;
const bool can_cache_reuse =
@@ -3578,8 +3848,12 @@ private:
@@ -4257,8 +4527,12 @@ private:
bool do_checkpoint = params_base.n_ctx_checkpoints > 0;
- // make checkpoints only for completion tasks
- do_checkpoint = do_checkpoint && slot.task->type == SERVER_TASK_TYPE_COMPLETION;
+ // make checkpoints for completion tasks, and for score tasks at the
@@ -364,10 +364,10 @@ index edb8e2d..9c88feb 100644
+ // prompt for every candidate of a scoring call
+ do_checkpoint = do_checkpoint && (slot.task->type == SERVER_TASK_TYPE_COMPLETION ||
+ slot.task->type == SERVER_TASK_TYPE_SCORE);
// make a checkpoint of the parts of the memory that cannot be rolled back.
// checkpoints are created only if:
@@ -3806,16 +4080,49 @@ private:
@@ -4356,16 +4630,49 @@ private:
// embedding requires all tokens in the batch to be output;
// MTP also wants logits at every prompt position so the
// streaming hook can mirror t_h_nextn into ctx_dft.
@@ -388,7 +388,7 @@ index edb8e2d..9c88feb 100644
batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]);
}
slot.prompt.tokens.push_back(cur_tok);
+ // score tasks: break at the shared-prompt boundary so the checkpoint
+ // below lands exactly there — the other candidates of the same
+ // scoring call re-process only their own tokens. Also break at the
@@ -418,10 +418,10 @@ index edb8e2d..9c88feb 100644
// break at the last user message, or at user messages at least min step past the last checkpoint
if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) {
const auto pos = slot.prompt.n_tokens();
@@ -3719,6 +4026,15 @@ private:
@@ -4408,6 +4715,15 @@ private:
const bool is_user_start = spans.is_user_start(n_tokens_start);
const bool is_last_user_message = n_tokens_start == last_user_pos;
+ // a batch starting at the score boundary or divergence point must
+ // always checkpoint — min-step spacing would otherwise suppress it
+ // and every candidate / next scoring call would re-process the prompt
@@ -434,7 +434,7 @@ index edb8e2d..9c88feb 100644
// entire prompt has been processed
if (slot.prompt.n_tokens() == slot.task->n_tokens()) {
slot.state = SLOT_STATE_DONE_PROMPT;
@@ -3734,8 +4050,8 @@ private:
@@ -4423,8 +4739,8 @@ private:
slot.init_sampler();
} else {
// skip ordinary mid-prompt checkpoints, unless the batch starts a user
@@ -445,10 +445,10 @@ index edb8e2d..9c88feb 100644
do_checkpoint = false;
}
}
@@ -3752,10 +4068,10 @@ private:
@@ -4441,10 +4757,10 @@ private:
// do not checkpoint after mtmd chunks
do_checkpoint = do_checkpoint && !has_mtmd;
- // no need to create checkpoints that are too close together, unless it's the last user message
+ // no need to create checkpoints that are too close together, unless it's the last user message or the score boundary
do_checkpoint = do_checkpoint && (
@@ -457,11 +457,11 @@ index edb8e2d..9c88feb 100644
+ is_last_user_message || near_prompt_end || is_score_boundary ||
n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step);
SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max);
@@ -3943,6 +4259,13 @@ private:
@@ -4690,6 +5006,13 @@ private:
}
}
+ // score slots harvest logprobs from every view that contains
+ // their outputs, not just the one holding the final token
+ if (slot.task && slot.task->type == SERVER_TASK_TYPE_SCORE &&
@@ -472,10 +472,10 @@ index edb8e2d..9c88feb 100644
if (!is_inside_view(slot.i_batch)) {
// the required token not in this sub-batch, skip
return;
@@ -3971,6 +4294,25 @@ private:
@@ -4718,6 +5041,25 @@ private:
return;
}
+ if (slot.task->type == SERVER_TASK_TYPE_SCORE) {
+ // shared-prefix logprobs (and every candidate's first
+ // suffix logprob) were accumulated per view above;
@@ -496,16 +496,16 @@ index edb8e2d..9c88feb 100644
+ }
+
GGML_ASSERT(slot.task->need_sampling());
// prompt evaluated for next-token prediction
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 8c5fa9a..4b38805 100644
index 810cefb..3e1ab8d 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -12,11 +12,26 @@
#include "server-common.h"
@@ -13,11 +13,26 @@
#include "mtmd-helper.h"
+// SERVER_TASK_TYPE_SCORE emits one logits output per candidate token (plus
+// the forced last-token output), and the context's output budget
+// (n_outputs_max) is reserved up front — so candidate length must be
@@ -527,12 +527,12 @@ index 8c5fa9a..4b38805 100644
SERVER_TASK_TYPE_DECISION,
+ SERVER_TASK_TYPE_SCORE,
SERVER_TASK_TYPE_INFILL,
SERVER_TASK_TYPE_TRANSCRIBE_RT,
SERVER_TASK_TYPE_CANCEL,
SERVER_TASK_TYPE_CONTROL,
@@ -156,6 +171,18 @@ struct server_task {
@@ -168,6 +183,18 @@ struct server_task {
task_params params;
server_tokens tokens;
+ // used by SERVER_TASK_TYPE_SCORE: `tokens` holds the shared prefix
+ // (prompt + longest common candidate token prefix) and logprobs are
+ // returned for its tokens from n_score_prompt onward. Each candidate's
@@ -548,18 +548,18 @@ index 8c5fa9a..4b38805 100644
// only used by CLI, this allow tokenizing CLI inputs on server side
// we need this because mtmd_context and vocab are not accessible outside of server_context
bool cli = false;
@@ -234,6 +261,7 @@ struct server_task {
switch (type) {
@@ -249,6 +276,7 @@ struct server_task {
case SERVER_TASK_TYPE_COMPLETION:
case SERVER_TASK_TYPE_INFILL:
case SERVER_TASK_TYPE_TRANSCRIBE_RT:
+ case SERVER_TASK_TYPE_SCORE:
return true;
default:
return false;
@@ -509,6 +537,25 @@ struct server_task_result_decision : server_task_result {
case SERVER_TASK_TYPE_DECISION:
return !decision.labels.empty();
@@ -558,6 +586,25 @@ struct server_task_result_decision : server_task_result {
virtual json to_json() override;
};
+struct server_task_result_score : server_task_result {
+ // log P(token | prefix) for the shared-prefix tokens after
+ // n_score_prompt, in order; NaN marks positions the decode never
backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch
-850
View File
@@ -1,850 +0,0 @@
diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp
index 5fb7ea9..b554bf2 100644
--- a/tools/mtmd/mtmd-helper-gen.cpp
+++ b/tools/mtmd/mtmd-helper-gen.cpp
@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri
return LLAMA_TOKEN_NULL;
}
+static void put_bytes(std::vector<char> & buf, const void * p, size_t n) {
+ const char * c = (const char *) p;
+ buf.insert(buf.end(), c, c + n);
+}
+
+// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe
+static void write_wav16_header(std::vector<char> & buf, uint32_t data_sz, int32_t rate) {
+ const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz;
+ const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
+ const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
+ const uint32_t rate32 = (uint32_t) rate;
+ put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4);
+ put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4);
+ put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4);
+ put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2);
+ put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4);
+}
+
+static void append_wav16_pcm(std::vector<char> & buf, const float * pcm, size_t n) {
+ for (size_t i = 0; i < n; i++) {
+ int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f);
+ put_bytes(buf, &s, 2);
+ }
+}
+
static bool write_wav16(std::vector<char> & buf, const std::vector<float> & pcm, int32_t rate) {
// RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header
if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) {
return false;
}
- const uint32_t data_sz = (uint32_t) (pcm.size() * 2);
- const uint32_t riff_sz = 36 + data_sz;
- const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2;
- const uint16_t fmt = 1, ch = 1, align = 2, bits = 16;
- const uint32_t rate32 = (uint32_t) rate;
- auto put = [&](const void * p, size_t n) {
- const char * c = (const char *) p;
- buf.insert(buf.end(), c, c + n);
- };
- put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4);
- put("fmt ", 4); put(&fmt_sz, 4);
- put(&fmt, 2); put(&ch, 2); put(&rate32, 4);
- put(&byte_rate, 4); put(&align, 2); put(&bits, 2);
- put("data", 4); put(&data_sz, 4);
- for (float v : pcm) {
- int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f);
- put(&s, 2);
- }
+ write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate);
+ append_wav16_pcm(buf, pcm.data(), pcm.size());
return true;
}
@@ -92,6 +101,8 @@ public:
// set out_stop on end-of-speech, h_state_out must be null if no frame is generated
virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0;
virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0;
+ // forces any buffered codes through code2wav now, regardless of window_frames
+ virtual int32_t flush() { return 0; }
protected:
llama_context * lctx;
@@ -121,6 +132,9 @@ public:
prompt_batch.reset();
n_prompt = 0;
prompt_pos = 0;
+ stream = false;
+ pcm_sent = 0;
+ wav_header_sent = false;
}
int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override {
@@ -208,6 +222,7 @@ public:
top_p = inp->top_p > 0 ? inp->top_p : def.top_p;
seed = inp->seed;
out_type = inp->out_type;
+ stream = inp->stream;
// the prompt above holds the whole text stream up to tts_eos, so every generated
// frame adds tts_pad on top of the codes embedding
@@ -301,31 +316,60 @@ public:
}
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override {
- if (!flush_gen_wav()) {
- return 1;
+ *out_sample_rate = info.sample_rate;
+
+ if (!stream) {
+ // one-shot call: force out whatever's left, regardless of window_frames
+ if (!flush_gen_wav()) {
+ return 1;
+ }
+ if (out_n_samples) {
+ *out_n_samples = (int64_t) audio_pcm.size();
+ }
+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
+ *out_data = (const char *) audio_pcm.data();
+ *out_data_len = audio_pcm.size() * sizeof(float);
+ return 0;
+ }
+ out_buf.clear();
+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
+ return 1;
+ }
+ *out_data = out_buf.data();
+ *out_data_len = out_buf.size();
+ return 0;
}
- *out_sample_rate = info.sample_rate;
+ // streaming: only return audio produced since the previous call
+ const size_t n_new = audio_pcm.size() - pcm_sent;
if (out_n_samples) {
- *out_n_samples = (int64_t) audio_pcm.size();
+ *out_n_samples = (int64_t) n_new;
}
if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) {
- *out_data = (const char *) audio_pcm.data();
- *out_data_len = audio_pcm.size() * sizeof(float);
+ *out_data = (const char *) (audio_pcm.data() + pcm_sent);
+ *out_data_len = n_new * sizeof(float);
+ pcm_sent = audio_pcm.size();
return 0;
}
out_buf.clear();
- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) {
- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n");
- return 1;
+ if (!wav_header_sent) {
+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate);
+ wav_header_sent = true;
}
+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new);
+ pcm_sent = audio_pcm.size();
*out_data = out_buf.data();
*out_data_len = out_buf.size();
return 0;
}
+ int32_t flush() override {
+ return flush_gen_wav() ? 0 : 1;
+ }
+
private:
bool ensure_cache() {
if (specials_ok) {
@@ -369,7 +413,7 @@ private:
LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n");
return false;
}
- const std::string marker = mtmd_default_marker();
+ const std::string marker = mtmd_get_marker(mctx);
mtmd_input_text text{ marker.c_str(), marker.size(), false, true };
mtmd_input_chunks * chunks = mtmd_input_chunks_init();
const mtmd_bitmap * bptr = bitmap;
@@ -455,6 +499,9 @@ private:
std::vector<float> h_state_buf;
mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
std::vector<char> out_buf;
+ bool stream = false;
+ size_t pcm_sent = 0; // samples already returned by get_output()
+ bool wav_header_sent = false;
};
// settings that only live in the reference's per-pack yaml, not in the checkpoint
@@ -1022,6 +1069,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) {
}
}
+struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) {
+ mtmd_helper_gen_audio_inp inp{};
+ inp.top_k = 50;
+ inp.top_p = 1.0f;
+ inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ return inp;
+}
+
int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) {
if (!ctx->pipeline) {
LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n");
@@ -1058,3 +1113,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t *
}
return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples);
}
+
+int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) {
+ if (!ctx->pipeline) {
+ return 1;
+ }
+ return ctx->pipeline->flush();
+}
diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h
index 7436230..acbfefc 100644
--- a/tools/mtmd/mtmd-helper.h
+++ b/tools/mtmd/mtmd-helper.h
@@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype {
MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono
};
struct mtmd_helper_gen_audio_inp {
+ bool stream; // if true, output() must be called after each step_gen()
llama_seq_id seq_id;
const char * prompt;
@@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp {
enum mtmd_helper_gen_audio_outtype out_type;
};
+MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void);
+
MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init(
struct llama_context * lctx,
struct mtmd_context * mctx);
@@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen(
// out_data valid until next get_output() or reset() call
// out_n_samples (optional, can be NULL) receives the number of generated PCM samples
+// if inp->stream is true: returns only audio produced since the previous call, and
+// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet
MTMD_API int32_t mtmd_helper_gen_audio_get_output(
mtmd_helper_gen_audio * ctx,
int32_t * out_sample_rate,
@@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output(
size_t * out_data_len,
int64_t * out_n_samples);
+// forces any buffered codes through code2wav now, regardless of window_frames;
+// call once when generation has ended, before the last get_output() in stream mode
+MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx);
+
#ifdef __cplusplus
} // extern "C"
#endif
@@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter {
};
using gen_audio_ptr = std::unique_ptr<mtmd_helper_gen_audio, mtmd_helper_gen_audio_deleter>;
struct gen_audio {
+
+ // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp
+ struct inp {
+ mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default();
+ std::string prompt_str;
+ std::string lang_str;
+ mtmd::bitmap_ptr speaker_ref_ptr;
+
+ inp() = default;
+ inp(inp &&) = default;
+ inp & operator=(inp &&) = default;
+ inp(const inp &) = delete;
+ inp & operator=(const inp &) = delete;
+
+ void set_prompt (std::string p) { prompt_str = std::move(p); }
+ void set_lang (std::string l) { lang_str = std::move(l); }
+ void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); }
+
+ // pointers are only valid as long as *this is alive
+ const mtmd_helper_gen_audio_inp * get() {
+ data.prompt = prompt_str.c_str();
+ data.prompt_len = prompt_str.size();
+ data.lang = lang_str.empty() ? nullptr : lang_str.c_str();
+ data.speaker_ref = speaker_ref_ptr.get();
+ return &data;
+ }
+ };
+
gen_audio_ptr ctx;
- gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {}
+ void init(struct llama_context * lctx, struct mtmd_context * mctx) {
+ ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx));
+ }
+ bool valid() const {
+ return ctx.get() != nullptr;
+ }
void reset() {
mtmd_helper_gen_audio_reset(ctx.get());
}
@@ -300,6 +342,9 @@ struct gen_audio {
int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) {
return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples);
}
+ int32_t flush() {
+ return mtmd_helper_gen_audio_flush(ctx.get());
+ }
};
} // namespace mtmd_helper
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 9c88feb..064bb51 100644
--- a/tools/server/server-context.cpp
+++ b/tools/server/server-context.cpp
@@ -17,6 +17,7 @@
#include "speculative.h"
#include "mtmd.h"
#include "mtmd-helper.h"
+#include "base64.hpp"
#include <algorithm>
#include <cstddef>
@@ -41,8 +42,9 @@ constexpr int HTTP_POLLING_SECONDS = 1;
static common_speculative_output_limits server_output_limits(const common_params & params) {
if (params.embedding ||
- (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) {
- return { params.n_batch, 1 };
+ (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE) ||
+ !params.mmproj.path.empty()) { // gen-audio (TTS) capability isn't known until the mmproj loads, size generously
+ return { params.n_batch, params.n_batch };
}
auto result = common_speculative_get_output_limits(
@@ -215,6 +217,30 @@ struct server_slot {
mtmd_context * mctx = nullptr;
mtmd::batch_ptr mbatch = nullptr;
+ struct tts_ctx {
+ mtmd_helper::gen_audio ctx;
+ const float * h_state;
+ llama_token sampled;
+ int32_t n_decoded;
+ bool is_supported() const {
+ return ctx.valid();
+ }
+ void reset() {
+ // mtmd_helper_gen_audio_reset() dereferences its argument before it
+ // null-checks the pipeline, and the pipeline is only allocated for
+ // models that actually carry a gen-audio mmproj. server_slot::reset()
+ // runs for every slot of every model, so without this guard any
+ // non-TTS model segfaults during slot initialization.
+ if (is_supported()) {
+ ctx.reset();
+ }
+ h_state = nullptr;
+ sampled = LLAMA_TOKEN_NULL;
+ n_decoded = 0;
+ }
+ };
+ tts_ctx tts;
+
// speculative decoding
common_speculative * spec;
@@ -395,6 +421,8 @@ struct server_slot {
// clear multimodal state
mbatch.reset();
+
+ tts.reset();
}
void init_sampler() const {
@@ -871,6 +899,14 @@ public:
server_decision_context decision;
+ bool has_cap_tts() const {
+ return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE;
+ }
+
+ bool has_cap_chat() const {
+ return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx);
+ }
+
server_queue queue_tasks;
server_response queue_results;
@@ -1350,6 +1386,10 @@ private:
slot.mctx = mctx;
slot.prompt.tokens.has_mtmd = mctx != nullptr;
+ if (has_cap_tts()) {
+ slot.tts.ctx.init(ctx_tgt, mctx);
+ }
+
SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx);
slot.callback_on_release = [this](int id_slot) {
@@ -1830,6 +1870,28 @@ private:
SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str());
+ if (task.type == SERVER_TASK_TYPE_TTS) {
+ GGML_ASSERT(has_cap_tts()); // should already checked in route handler
+ if (!slot.tts.is_supported()) {
+ slot.tts.ctx.init(ctx_tgt, slot.mctx);
+ }
+
+ // TTS slots never enter the shared batch: pre_decode() returns early for
+ // them and process_tts_slots() drives them instead, so they skip the
+ // prompt-cache bookkeeping that clears this sequence between requests.
+ // The gen-audio pipeline always decodes from position 0, and its own
+ // reset() only clears host-side buffers, so without this the second and
+ // later tasks on a slot decode over the previous request's tokens and
+ // step_prompt() fails immediately.
+ slot.prompt_clear();
+
+ task.tts_inp.data.seq_id = slot.id;
+ if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) {
+ send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER);
+ return false;
+ }
+ }
+
// initialize samplers
if (task.need_sampling()) {
try {
@@ -1847,6 +1909,9 @@ private:
// TODO: getting pre sampling logits is not yet supported with backend sampling
use_backend_sampling &= !need_pre_sample_logits;
+ // TODO: check verify if this actually works with TTS
+ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS;
+
// TODO: tmp until backend sampling is fully implemented
if (use_backend_sampling) {
llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get()));
@@ -1872,9 +1937,13 @@ private:
slot.task = std::make_unique<const server_task>(std::move(task));
- slot.state = slot.task->is_child()
- ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
- : SLOT_STATE_STARTED;
+ if (slot.task->type == SERVER_TASK_TYPE_TTS) {
+ slot.state = SLOT_STATE_PROCESSING_PROMPT;
+ } else {
+ slot.state = slot.task->is_child()
+ ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt
+ : SLOT_STATE_STARTED;
+ }
// reset server kill-switch counter
n_empty_consecutive = 0;
@@ -2139,6 +2208,18 @@ private:
queue_results.send(std::move(res));
}
+ void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) {
+ auto res = std::make_unique<server_task_result_tts>();
+
+ res->id = slot.task->id;
+ res->index = slot.task->index;
+ res->sample_rate = sample_rate;
+ res->audio.assign(data, data_len);
+ res->final = final;
+
+ queue_results.send(std::move(res));
+ }
+
void send_final_response(server_slot & slot) {
auto res = std::make_unique<server_task_result_cmpl_final>();
@@ -2721,6 +2802,7 @@ private:
case SERVER_TASK_TYPE_RERANK:
case SERVER_TASK_TYPE_DECISION:
case SERVER_TASK_TYPE_SCORE:
+ case SERVER_TASK_TYPE_TTS:
{
// special case: if input is provided via CLI, tokenize it first
// otherwise, no need to tokenize as it's already done inside the HTTP thread
@@ -3179,6 +3261,14 @@ private:
return;
}
+ // note: TTS slots bypass the shared batch entirely
+ try {
+ process_tts_slots();
+ } catch (const std::exception & e) {
+ SRV_ERR("process_tts_slots() failed: %s\n", e.what());
+ abort_all_slots("process_tts_slots() failed: " + std::string(e.what()));
+ }
+
GGML_ASSERT(batch.slot_batched || batch.size() == 0);
if (batch.slot_batched) {
@@ -3248,10 +3338,77 @@ private:
}
}
+ void process_tts_slots() {
+ iterate(slots, [&](server_slot & slot) {
+ if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) {
+ return;
+ }
+
+ llama_set_embeddings(ctx_tgt, true);
+
+ if (slot.state == SLOT_STATE_PROCESSING_PROMPT) {
+ const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt));
+ if (ret < 0) {
+ send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER);
+ slot.release();
+ } else if (ret == 0) {
+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
+ slot.tts.h_state = llama_get_embeddings_ith(ctx_tgt, -1);
+ slot.state = SLOT_STATE_GENERATING;
+ }
+ return;
+ }
+
+ const int32_t n_predict = slot.task->params.n_predict > 0 ? slot.task->params.n_predict : 512;
+ if (slot.tts.n_decoded >= n_predict || llama_vocab_is_eog(vocab, slot.tts.sampled)) {
+ int32_t sample_rate = 0;
+ const char * data = nullptr;
+ size_t data_len = 0;
+ // generation truly ends here: force out any sub-window remainder still buffered
+ if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
+ send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER);
+ } else {
+ send_tts_result(slot, sample_rate, data, data_len, true);
+ }
+ slot.release();
+ return;
+ }
+
+ const float * h_state_next = nullptr;
+ if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next) != 0) {
+ send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER);
+ slot.release();
+ return;
+ }
+ slot.tts.h_state = h_state_next;
+ slot.tts.n_decoded++;
+
+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1);
+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true);
+
+ if (slot.task->params.stream) {
+ int32_t sample_rate = 0;
+ const char * data = nullptr;
+ size_t data_len = 0;
+ if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) {
+ send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER);
+ slot.release();
+ } else if (data_len > 0) {
+ send_tts_result(slot, sample_rate, data, data_len, false);
+ }
+ }
+ });
+ }
+
void pre_decode() {
// apply context-shift if needed
// TODO: simplify and improve
iterate(slots, [&](server_slot & slot) {
+ if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) {
+ // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch
+ return;
+ }
if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) {
if (!params_base.ctx_shift) {
// this check is redundant (for good)
@@ -3324,7 +3481,7 @@ private:
// determine which slots are generating and drafting
iterate(slots, [&](server_slot & slot) {
- if (slot.state != SLOT_STATE_GENERATING) {
+ if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -3458,7 +3615,7 @@ private:
return; // batch is full, skip remaining slots
}
- if (!slot.is_processing()) {
+ if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) {
return;
}
@@ -4678,6 +4835,8 @@ server_context_meta server_context::get_meta() const {
/* has_inp_image */ impl->chat_params.allow_image,
/* has_inp_audio */ impl->chat_params.allow_audio,
/* has_inp_video */ impl->chat_params.allow_video,
+ /* has_cap_chat */ impl->has_cap_chat(),
+ /* has_cap_tts */ impl->has_cap_tts(),
/* json_ui_settings */ impl->json_ui_settings,
/* slot_n_ctx */ impl->n_ctx_slot(),
/* pooling_type */ llama_pooling_type(impl->ctx_tgt),
@@ -4751,6 +4910,11 @@ std::unique_ptr<server_res_generator> server_routes::handle_completions_impl(
res->set_req(&req); // will also set spipe if needed
+ if (!ctx_server.has_cap_chat()) {
+ res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
int32_t sse_ping_interval = params.sse_ping_interval;
try {
@@ -5776,6 +5940,150 @@ void server_routes::init_routes() {
return res;
};
+ this->post_tts = [this](const server_http_req & req) {
+ auto res = create_response();
+ res->set_req(&req); // will also set spipe if needed
+
+ if (!ctx_server.has_cap_tts()) {
+ res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED));
+ return res;
+ }
+
+ const json body = json::parse(req.body);
+
+ std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string()));
+ if (prompt.empty()) {
+ res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+
+ const std::string response_format = json_value(body, "response_format", std::string("wav"));
+ const bool stream = json_value(body, "stream", false);
+
+ server_task task(SERVER_TASK_TYPE_TTS);
+ task.tts_inp.set_prompt(prompt);
+ task.tts_inp.set_lang(json_value(body, "lang", std::string()));
+ task.tts_inp.data.top_k = json_value(body, "top_k", 0);
+ task.tts_inp.data.top_p = json_value(body, "top_p", 0.0f);
+ task.tts_inp.data.stream = stream;
+ task.tts_inp.data.out_type = response_format == "pcm"
+ ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM
+ : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV;
+ task.params.stream = stream;
+ task.params.n_predict = json_value(body, "n_predict", -1);
+ task.params.sampling = params.sampling; // baseline defaults, then apply overrides below
+ task.params.sampling.penalty_repeat = json_value(body, "repeat_penalty", 1.05f);
+ task.params.sampling.penalty_last_n = -1;
+ if (task.tts_inp.data.top_k > 0) {
+ task.params.sampling.top_k = task.tts_inp.data.top_k;
+ }
+ if (task.tts_inp.data.top_p > 0) {
+ task.params.sampling.top_p = task.tts_inp.data.top_p;
+ }
+
+ // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64")
+ const unsigned char * speaker_ref_data = nullptr;
+ size_t speaker_ref_len = 0;
+ std::string speaker_ref_b64_decoded;
+
+ auto speaker_ref_file = req.files.find("speaker_ref");
+ if (speaker_ref_file != req.files.end()) {
+ speaker_ref_data = speaker_ref_file->second.data.data();
+ speaker_ref_len = speaker_ref_file->second.data.size();
+ } else {
+ std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string());
+ if (!speaker_ref_b64.empty()) {
+ speaker_ref_b64_decoded = base64::decode(speaker_ref_b64);
+ speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data();
+ speaker_ref_len = speaker_ref_b64_decoded.size();
+ }
+ }
+
+ if (speaker_ref_len > 0) {
+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
+ if (!wrapper.bitmap) {
+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
+ return res;
+ }
+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap));
+ } else {
+ // SRV_WRN expands __VA_ARGS__ without the GNU comma-elision extension,
+ // so a bare format string leaves a trailing comma and will not compile
+ SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n");
+ }
+
+ auto & rd = res->rd;
+ task.id = rd.get_new_id();
+ rd.post_task(std::move(task));
+
+ const std::string content_type = response_format == "pcm" ? "audio/L16" : "audio/wav";
+
+ if (!stream) {
+ auto result = rd.next(req.should_stop);
+ if (!result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (result->is_error()) {
+ res->error(result->to_json());
+ return res;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = std::move(tts_res->audio);
+ return res;
+ } else {
+ auto first_result = rd.next(req.should_stop);
+ if (!first_result) {
+ GGML_ASSERT(req.should_stop());
+ return res; // connection is closed
+ }
+ if (first_result->is_error()) {
+ res->error(first_result->to_json());
+ return res;
+ }
+ auto * first_tts_res = dynamic_cast<server_task_result_tts *>(first_result.get());
+ GGML_ASSERT(first_tts_res != nullptr);
+
+ res->status = 200;
+ res->content_type = content_type;
+ res->data = std::move(first_tts_res->audio);
+ bool is_done = first_tts_res->final;
+
+ res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool {
+ if (is_done) {
+ return false;
+ }
+ if (res_this->should_stop()) {
+ return false;
+ }
+ if (!res_this->data.empty()) {
+ output = std::move(res_this->data);
+ res_this->data.clear();
+ return true;
+ }
+
+ server_response_reader & rd = res_this->rd;
+ if (!rd.has_next()) {
+ return false;
+ }
+ auto result = rd.next([&res_this]() { return res_this->should_stop(); });
+ if (!result || result->is_error()) {
+ return false;
+ }
+ auto * tts_res = dynamic_cast<server_task_result_tts *>(result.get());
+ GGML_ASSERT(tts_res != nullptr);
+ output = std::move(tts_res->audio);
+ is_done = tts_res->final;
+ return true;
+ });
+ }
+
+ return res;
+ };
+
this->get_lora_adapters = [this](const server_http_req & req) {
auto res = create_response();
diff --git a/tools/server/server-context.h b/tools/server/server-context.h
index c554bb9..f025792 100644
--- a/tools/server/server-context.h
+++ b/tools/server/server-context.h
@@ -23,6 +23,8 @@ struct server_context_meta {
bool has_inp_image;
bool has_inp_audio;
bool has_inp_video;
+ bool has_cap_chat;
+ bool has_cap_tts;
json json_ui_settings;
int slot_n_ctx;
enum llama_pooling_type pooling_type;
@@ -152,6 +154,7 @@ struct server_routes {
server_http_context::handler_t post_embeddings;
server_http_context::handler_t post_embeddings_oai;
server_http_context::handler_t post_rerank;
+ server_http_context::handler_t post_tts;
server_http_context::handler_t post_systemone;
server_http_context::handler_t get_lora_adapters;
server_http_context::handler_t post_lora_adapters;
diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp
index a5c33c0..7e59745 100644
--- a/tools/server/server-task.cpp
+++ b/tools/server/server-task.cpp
@@ -1506,6 +1506,17 @@ json server_task_result_decision::to_json() {
};
}
+//
+// server_task_result_tts
+//
+json server_task_result_tts::to_json() {
+ return json {
+ {"sample_rate", sample_rate},
+ {"n_bytes", audio.size()},
+ {"final", final},
+ };
+}
+
//
// server_task_result_error
//
diff --git a/tools/server/server-task.h b/tools/server/server-task.h
index 4b38805..d79ea4f 100644
--- a/tools/server/server-task.h
+++ b/tools/server/server-task.h
@@ -10,6 +10,7 @@
// TODO: prevent including the whole server-common.h as we only use server_tokens
#include "server-common.h"
+#include "mtmd-helper.h"
// SERVER_TASK_TYPE_SCORE emits one logits output per candidate token (plus
@@ -43,6 +44,7 @@ enum server_task_type {
SERVER_TASK_TYPE_SLOT_ERASE,
SERVER_TASK_TYPE_GET_LORA,
SERVER_TASK_TYPE_SET_LORA,
+ SERVER_TASK_TYPE_TTS,
};
// TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common
@@ -225,6 +227,9 @@ struct server_task {
// used by SERVER_TASK_TYPE_SET_LORA
std::map<int, float> set_lora; // mapping adapter ID -> scale
+ // used by SERVER_TASK_TYPE_TTS
+ mtmd_helper::gen_audio::inp tts_inp;
+
server_task() = default;
server_task(server_task_type type) : type(type) {}
@@ -249,6 +254,7 @@ struct server_task {
switch (type) {
case SERVER_TASK_TYPE_COMPLETION:
case SERVER_TASK_TYPE_INFILL:
+ case SERVER_TASK_TYPE_TTS:
return true;
case SERVER_TASK_TYPE_DECISION:
return !decision.labels.empty();
@@ -521,5 +527,15 @@ struct server_task_result_embd : server_task_result {
json to_json_oaicompat();
};
+struct server_task_result_tts : server_task_result {
+ std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type)
+ int32_t sample_rate = 0;
+ bool final = false; // true for the last chunk of a request
+
+ virtual bool is_stop() override { return final; }
+
+ virtual json to_json() override;
+};
+
struct server_task_result_rerank : server_task_result {
float score = -1e6;