diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index e37f3d544..5d238969a 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=030ebb558a5820b444a8f836ed5cdd46c9b4bd7a +LLAMA_VERSION?=84e908c625fb60992b4cdef8180fb12fa9b4c4bf LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 4e05d9f79..566a62cfe 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -1,22 +1,8 @@ -From 861fb06531e770fcea65e86296f80eba830dac30 Mon Sep 17 00:00:00 2001 -From: Codex -Date: Mon, 10 Aug 2026 23:05:53 +0000 -Subject: [PATCH 2/2] tts-patch - ---- - tools/mtmd/mtmd-helper-gen.cpp | 118 +++++++++--- - tools/mtmd/mtmd-helper.h | 47 ++++- - tools/server/server-context.cpp | 322 +++++++++++++++++++++++++++++++- - tools/server/server-context.h | 3 + - tools/server/server-task.cpp | 11 ++ - tools/server/server-task.h | 16 ++ - 6 files changed, 481 insertions(+), 36 deletions(-) - diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp -index 85671d1..66ccf0a 100644 +index 1c58d3ae1..196cbd433 100644 --- a/tools/mtmd/mtmd-helper-gen.cpp +++ b/tools/mtmd/mtmd-helper-gen.cpp -@@ -48,29 +48,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri +@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri return LLAMA_TOKEN_NULL; } @@ -73,16 +59,16 @@ index 85671d1..66ccf0a 100644 return true; } -@@ -89,6 +98,8 @@ public: - // those read what they need from h_state_in instead - virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out) = 0; +@@ -92,6 +101,8 @@ public: + // set out_stop on end-of-speech, h_state_out must be null if no frame is generated + virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0; virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; + // forces any buffered codes through code2wav now, regardless of window_frames -+ virtual int32_t flush() = 0; ++ virtual int32_t flush() { return 0; } protected: llama_context * lctx; -@@ -118,6 +129,9 @@ public: +@@ -121,6 +132,9 @@ public: prompt_batch.reset(); n_prompt = 0; prompt_pos = 0; @@ -92,15 +78,15 @@ index 85671d1..66ccf0a 100644 } int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { -@@ -203,6 +217,7 @@ public: - top_k = inp->top_k > 0 ? inp->top_k : 50; - top_p = inp->top_p > 0 ? inp->top_p : 1.0f; +@@ -208,6 +222,7 @@ public: + top_p = inp->top_p > 0 ? inp->top_p : def.top_p; + seed = inp->seed; out_type = inp->out_type; + stream = inp->stream; // the prompt above holds the whole text stream up to tts_eos, so every generated // frame adds tts_pad on top of the codes embedding -@@ -284,31 +299,60 @@ public: +@@ -302,31 +317,60 @@ public: } int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { @@ -170,7 +156,7 @@ index 85671d1..66ccf0a 100644 private: bool ensure_cache() { if (specials_ok) { -@@ -352,7 +396,7 @@ private: +@@ -370,7 +414,7 @@ private: LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); return false; } @@ -179,7 +165,7 @@ index 85671d1..66ccf0a 100644 mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; mtmd_input_chunks * chunks = mtmd_input_chunks_init(); const mtmd_bitmap * bptr = bitmap; -@@ -436,6 +480,9 @@ private: +@@ -456,6 +500,9 @@ private: std::vector h_state_buf; mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; std::vector out_buf; @@ -188,8 +174,8 @@ index 85671d1..66ccf0a 100644 + bool wav_header_sent = false; }; - static std::unique_ptr make_pipeline(llama_context * lctx, mtmd_context * mctx) { -@@ -467,6 +514,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { + // settings that only live in the reference's per-pack yaml, not in the checkpoint +@@ -1024,6 +1071,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { } } @@ -204,7 +190,7 @@ index 85671d1..66ccf0a 100644 int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { if (!ctx->pipeline) { LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); -@@ -497,3 +552,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * +@@ -1060,3 +1115,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * } return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); } @@ -216,7 +202,7 @@ index 85671d1..66ccf0a 100644 + return ctx->pipeline->flush(); +} diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h -index 7e5cf9b..1f3ec01 100644 +index 832f7171a..3eaa01aab 100644 --- a/tools/mtmd/mtmd-helper.h +++ b/tools/mtmd/mtmd-helper.h @@ -175,6 +175,7 @@ enum mtmd_helper_gen_audio_outtype { @@ -227,7 +213,7 @@ index 7e5cf9b..1f3ec01 100644 llama_seq_id seq_id; const char * prompt; -@@ -189,6 +190,8 @@ struct mtmd_helper_gen_audio_inp { +@@ -190,6 +191,8 @@ struct mtmd_helper_gen_audio_inp { enum mtmd_helper_gen_audio_outtype out_type; }; @@ -236,7 +222,7 @@ index 7e5cf9b..1f3ec01 100644 MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( struct llama_context * lctx, struct mtmd_context * mctx); -@@ -217,6 +220,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( +@@ -221,6 +224,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( // out_data valid until next get_output() or reset() call // out_n_samples (optional, can be NULL) receives the number of generated PCM samples @@ -245,7 +231,7 @@ index 7e5cf9b..1f3ec01 100644 MTMD_API int32_t mtmd_helper_gen_audio_get_output( mtmd_helper_gen_audio * ctx, int32_t * out_sample_rate, -@@ -224,6 +229,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output( +@@ -228,6 +233,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output( size_t * out_data_len, int64_t * out_n_samples); @@ -256,7 +242,7 @@ index 7e5cf9b..1f3ec01 100644 #ifdef __cplusplus } // extern "C" #endif -@@ -250,8 +259,41 @@ struct mtmd_helper_gen_audio_deleter { +@@ -254,8 +263,41 @@ struct mtmd_helper_gen_audio_deleter { }; using gen_audio_ptr = std::unique_ptr; struct gen_audio { @@ -299,7 +285,7 @@ index 7e5cf9b..1f3ec01 100644 void reset() { mtmd_helper_gen_audio_reset(ctx.get()); } -@@ -267,6 +309,9 @@ struct gen_audio { +@@ -271,6 +313,9 @@ struct gen_audio { int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) { return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples); } @@ -310,7 +296,7 @@ index 7e5cf9b..1f3ec01 100644 } // namespace mtmd_helper diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index d0e18e6..0765065 100644 +index 9069463fe..b7fa1e534 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp @@ -16,6 +16,7 @@ @@ -373,7 +359,7 @@ index d0e18e6..0765065 100644 } void init_sampler() const { -@@ -946,6 +974,14 @@ public: +@@ -941,6 +969,14 @@ public: mtmd_context * mctx = nullptr; const llama_vocab * vocab = nullptr; @@ -388,7 +374,7 @@ index d0e18e6..0765065 100644 server_queue queue_tasks; server_response queue_results; -@@ -1399,6 +1435,10 @@ private: +@@ -1394,6 +1430,10 @@ private: slot.mctx = mctx; slot.prompt.tokens.has_mtmd = mctx != nullptr; @@ -399,7 +385,7 @@ index d0e18e6..0765065 100644 SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx); slot.callback_on_release = [this](int id_slot) { -@@ -1852,6 +1892,28 @@ private: +@@ -1847,6 +1887,28 @@ private: SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); @@ -428,7 +414,7 @@ index d0e18e6..0765065 100644 // initialize samplers if (task.need_sampling()) { try { -@@ -1869,6 +1931,9 @@ private: +@@ -1864,6 +1926,9 @@ private: // TODO: getting pre sampling logits is not yet supported with backend sampling use_backend_sampling &= !need_pre_sample_logits; @@ -438,7 +424,7 @@ index d0e18e6..0765065 100644 // TODO: tmp until backend sampling is fully implemented if (use_backend_sampling) { llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get())); -@@ -1884,9 +1949,13 @@ private: +@@ -1879,9 +1944,13 @@ private: slot.task = std::make_unique(std::move(task)); @@ -455,7 +441,7 @@ index d0e18e6..0765065 100644 // reset server kill-switch counter n_empty_consecutive = 0; -@@ -2163,6 +2232,18 @@ private: +@@ -2158,6 +2227,18 @@ private: queue_results.send(std::move(res)); } @@ -474,7 +460,7 @@ index d0e18e6..0765065 100644 void send_final_response(server_slot & slot) { auto res = std::make_unique(); -@@ -2662,6 +2743,7 @@ private: +@@ -2657,6 +2738,7 @@ private: case SERVER_TASK_TYPE_EMBEDDING: case SERVER_TASK_TYPE_RERANK: case SERVER_TASK_TYPE_SCORE: @@ -482,7 +468,7 @@ index d0e18e6..0765065 100644 { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -3097,6 +3179,14 @@ private: +@@ -3092,6 +3174,14 @@ private: abort_all_slots("pre_decode() failed: " + std::string(e.what())); } @@ -497,7 +483,7 @@ index d0e18e6..0765065 100644 GGML_ASSERT(batch.slot_batched || batch.size() == 0); if (batch.slot_batched) { -@@ -3167,10 +3257,77 @@ private: +@@ -3162,10 +3252,77 @@ private: } } @@ -575,7 +561,7 @@ index d0e18e6..0765065 100644 if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { if (!params_base.ctx_shift) { // this check is redundant (for good) -@@ -3243,7 +3400,7 @@ private: +@@ -3238,7 +3395,7 @@ private: // determine which slots are generating and drafting iterate(slots, [&](server_slot & slot) { @@ -584,7 +570,7 @@ index d0e18e6..0765065 100644 return; } -@@ -3375,7 +3532,7 @@ private: +@@ -3370,7 +3527,7 @@ private: return; // batch is full, skip remaining slots } @@ -593,7 +579,7 @@ index d0e18e6..0765065 100644 return; } -@@ -4384,6 +4541,8 @@ server_context_meta server_context::get_meta() const { +@@ -4379,6 +4536,8 @@ server_context_meta server_context::get_meta() const { /* has_inp_image */ impl->chat_params.allow_image, /* has_inp_audio */ impl->chat_params.allow_audio, /* has_inp_video */ impl->chat_params.allow_video, @@ -602,7 +588,7 @@ index d0e18e6..0765065 100644 /* json_ui_settings */ impl->json_ui_settings, /* slot_n_ctx */ impl->get_slot_n_ctx(), /* pooling_type */ llama_pooling_type(impl->ctx_tgt), -@@ -4463,6 +4622,11 @@ std::unique_ptr server_routes::handle_completions_impl( +@@ -4458,6 +4617,11 @@ std::unique_ptr server_routes::handle_completions_impl( res->set_req(&req); // will also set spipe if needed @@ -614,7 +600,7 @@ index d0e18e6..0765065 100644 int32_t sse_ping_interval = params.sse_ping_interval; try { -@@ -5440,6 +5604,150 @@ void server_routes::init_routes() { +@@ -5435,6 +5599,150 @@ void server_routes::init_routes() { return res; }; @@ -766,7 +752,7 @@ index d0e18e6..0765065 100644 auto res = create_response(); diff --git a/tools/server/server-context.h b/tools/server/server-context.h -index f9ab113..6105126 100644 +index f9ab1132b..610512678 100644 --- a/tools/server/server-context.h +++ b/tools/server/server-context.h @@ -22,6 +22,8 @@ struct server_context_meta { @@ -787,7 +773,7 @@ index f9ab113..6105126 100644 server_http_context::handler_t post_lora_adapters; diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp -index 1ee6775..939630b 100644 +index 1ee677553..939630b8b 100644 --- a/tools/server/server-task.cpp +++ b/tools/server/server-task.cpp @@ -1523,6 +1523,17 @@ json server_task_result_rerank::to_json() { @@ -809,7 +795,7 @@ index 1ee6775..939630b 100644 // server_task_result_error // diff --git a/tools/server/server-task.h b/tools/server/server-task.h -index 5bedf19..e6ca67a 100644 +index 5bedf1987..e6ca67a65 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h @@ -10,6 +10,7 @@ @@ -846,7 +832,7 @@ index 5bedf19..e6ca67a 100644 return true; default: return false; -@@ -514,6 +520,16 @@ struct server_task_result_embd : server_task_result { +@@ -514,5 +520,15 @@ struct server_task_result_embd : server_task_result { json to_json_oaicompat(); }; @@ -862,6 +848,3 @@ index 5bedf19..e6ca67a 100644 + struct server_task_result_rerank : server_task_result { float score = -1e6; - --- -2.39.5