diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 3bce117b3..bcd388182 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=c0fd78626405c6d9b934efef0ec7291fcc20e3d6 +LLAMA_VERSION?=a37cc46104eb50900d5ab4a9ba02a0abdac039db LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index f51f70ad7..47f46788f 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -1,10 +1,10 @@ diff --git a/common/common.cpp b/common/common.cpp -index aca1949..428b120 100644 +index 6ef3b78..037a00b 100644 --- a/common/common.cpp +++ b/common/common.cpp -@@ -1662,8 +1662,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & +@@ -1716,8 +1716,10 @@ struct llama_context_params common_context_params_to_llama(const common_params & auto cparams = llama_context_default_params(); - + cparams.n_ctx = params.n_ctx; - cparams.n_seq_max = params.n_parallel; - cparams.n_rs_seq = params.speculative.need_n_rs_seq(); @@ -16,10 +16,10 @@ index aca1949..428b120 100644 cparams.n_outputs_max_per_seq = std::max(params.n_outputs_max_per_seq, 0); cparams.n_batch = params.n_batch; diff --git a/common/common.h b/common/common.h -index 04ffbcd..5daa023 100644 +index 70d6a67..93e2269 100644 --- a/common/common.h +++ b/common/common.h -@@ -455,6 +455,9 @@ struct common_params { +@@ -460,6 +460,9 @@ struct common_params { int32_t n_keep = 0; // number of tokens to keep from initial prompt int32_t n_chunks = -1; // max number of chunks to process (-1 = unlimited) int32_t n_parallel = 1; // number of parallel sequences to decode @@ -30,13 +30,13 @@ index 04ffbcd..5daa023 100644 int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t n_outputs_max_per_seq = 1; // max outputs per sequence diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index edb8e2d..9c88feb 100644 +index 296893f..ef38f90 100644 --- a/tools/server/server-context.cpp +++ b/tools/server/server-context.cpp -@@ -48,6 +48,13 @@ static common_speculative_output_limits server_output_limits(const common_params +@@ -60,6 +60,13 @@ static common_speculative_output_limits server_output_limits(const common_params auto result = common_speculative_get_output_limits( params.n_batch, params.n_parallel, common_speculative_n_max(¶ms.speculative)); - + + // Score tasks output logits for every candidate token, so reserve room + // for a bounded candidate tail per parallel slot. + if (params.score_enabled) { @@ -47,10 +47,10 @@ index edb8e2d..9c88feb 100644 result.total = std::max(1, result.total); result.per_seq = std::max(1, result.per_seq); return result; -@@ -243,6 +250,26 @@ struct server_slot { - +@@ -325,6 +332,26 @@ struct server_slot { + std::vector generated_token_probs; - + + // SERVER_TASK_TYPE_SCORE: shared-prefix token logprobs harvested + // incrementally across batch views (NaN = not yet produced) + std::vector score_logprobs; @@ -74,7 +74,7 @@ index edb8e2d..9c88feb 100644 bool has_next_token = true; bool has_new_line = false; bool truncated = false; -@@ -346,6 +373,10 @@ struct server_slot { +@@ -433,6 +460,10 @@ struct server_slot { } generated_tokens.clear(); generated_token_probs.clear(); @@ -83,12 +83,12 @@ index edb8e2d..9c88feb 100644 + score_suffix_pending = false; + score_divergence = -1; json_schema = json(); - + task_prev = std::move(task); -@@ -2308,6 +2339,227 @@ private: +@@ -2571,6 +2602,227 @@ private: queue_results.send(std::move(res)); } - + + // log(sum(exp(logits))) with max-subtraction for stability — the + // log_softmax denominator shared by every token read from one output + static double score_log_denom(const float * logits, int32_t n_vocab) { @@ -313,15 +313,15 @@ index edb8e2d..9c88feb 100644 // // Functions to process the task // -@@ -2468,6 +2720,7 @@ private: - case SERVER_TASK_TYPE_EMBEDDING: +@@ -2957,6 +3209,7 @@ private: case SERVER_TASK_TYPE_RERANK: + case SERVER_TASK_TYPE_TTS: case SERVER_TASK_TYPE_DECISION: + case SERVER_TASK_TYPE_SCORE: { // special case: if input is provided via CLI, tokenize it first // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -2986,6 +3239,13 @@ private: +@@ -3549,6 +3802,13 @@ private: break; // stop any further processing } } @@ -333,12 +333,12 @@ index edb8e2d..9c88feb 100644 + abort_all_slots("update_score_suffixes() failed: " + std::string(e.what())); + } } - - void pre_decode() { -@@ -3325,6 +3585,16 @@ private: + + void process_tts_slots() { +@@ -3987,6 +4247,16 @@ private: n_past = std::min(n_past, slot.alora_invocation_start - 1); } - + + // score tasks need the logits that predict the first candidate + // token, so the last shared-prompt token must be (re-)decoded + // even when the cache already covers it @@ -350,12 +350,12 @@ index edb8e2d..9c88feb 100644 + } + const auto n_cache_reuse = slot.task->params.n_cache_reuse; - + const bool can_cache_reuse = -@@ -3578,8 +3848,12 @@ private: - +@@ -4257,8 +4527,12 @@ private: + bool do_checkpoint = params_base.n_ctx_checkpoints > 0; - + - // make checkpoints only for completion tasks - do_checkpoint = do_checkpoint && slot.task->type == SERVER_TASK_TYPE_COMPLETION; + // make checkpoints for completion tasks, and for score tasks at the @@ -364,10 +364,10 @@ index edb8e2d..9c88feb 100644 + // prompt for every candidate of a scoring call + do_checkpoint = do_checkpoint && (slot.task->type == SERVER_TASK_TYPE_COMPLETION || + slot.task->type == SERVER_TASK_TYPE_SCORE); - + // make a checkpoint of the parts of the memory that cannot be rolled back. // checkpoints are created only if: -@@ -3806,16 +4080,49 @@ private: +@@ -4356,16 +4630,49 @@ private: // embedding requires all tokens in the batch to be output; // MTP also wants logits at every prompt position so the // streaming hook can mirror t_h_nextn into ctx_dft. @@ -388,7 +388,7 @@ index edb8e2d..9c88feb 100644 batch.set_decision_order(batch.size() - 1, slot.task->decision.order[slot.prompt.n_tokens()]); } slot.prompt.tokens.push_back(cur_tok); - + + // score tasks: break at the shared-prompt boundary so the checkpoint + // below lands exactly there — the other candidates of the same + // scoring call re-process only their own tokens. Also break at the @@ -418,10 +418,10 @@ index edb8e2d..9c88feb 100644 // break at the last user message, or at user messages at least min step past the last checkpoint if (do_checkpoint && spans.is_user_start(slot.prompt.n_tokens())) { const auto pos = slot.prompt.n_tokens(); -@@ -3719,6 +4026,15 @@ private: +@@ -4408,6 +4715,15 @@ private: const bool is_user_start = spans.is_user_start(n_tokens_start); const bool is_last_user_message = n_tokens_start == last_user_pos; - + + // a batch starting at the score boundary or divergence point must + // always checkpoint — min-step spacing would otherwise suppress it + // and every candidate / next scoring call would re-process the prompt @@ -434,7 +434,7 @@ index edb8e2d..9c88feb 100644 // entire prompt has been processed if (slot.prompt.n_tokens() == slot.task->n_tokens()) { slot.state = SLOT_STATE_DONE_PROMPT; -@@ -3734,8 +4050,8 @@ private: +@@ -4423,8 +4739,8 @@ private: slot.init_sampler(); } else { // skip ordinary mid-prompt checkpoints, unless the batch starts a user @@ -445,10 +445,10 @@ index edb8e2d..9c88feb 100644 do_checkpoint = false; } } -@@ -3752,10 +4068,10 @@ private: +@@ -4441,10 +4757,10 @@ private: // do not checkpoint after mtmd chunks do_checkpoint = do_checkpoint && !has_mtmd; - + - // no need to create checkpoints that are too close together, unless it's the last user message + // no need to create checkpoints that are too close together, unless it's the last user message or the score boundary do_checkpoint = do_checkpoint && ( @@ -457,11 +457,11 @@ index edb8e2d..9c88feb 100644 + is_last_user_message || near_prompt_end || is_score_boundary || n_tokens_start > slot.prompt.checkpoints.back().n_tokens + params_base.checkpoint_min_step); SLT_DBG(slot, "main/do_checkpoint = %s, pos_min = %d, pos_max = %d\n", do_checkpoint ? "yes" : "no", pos_min, pos_max); - -@@ -3943,6 +4259,13 @@ private: + +@@ -4690,6 +5006,13 @@ private: } } - + + // score slots harvest logprobs from every view that contains + // their outputs, not just the one holding the final token + if (slot.task && slot.task->type == SERVER_TASK_TYPE_SCORE && @@ -472,10 +472,10 @@ index edb8e2d..9c88feb 100644 if (!is_inside_view(slot.i_batch)) { // the required token not in this sub-batch, skip return; -@@ -3971,6 +4294,25 @@ private: +@@ -4718,6 +5041,25 @@ private: return; } - + + if (slot.task->type == SERVER_TASK_TYPE_SCORE) { + // shared-prefix logprobs (and every candidate's first + // suffix logprob) were accumulated per view above; @@ -496,16 +496,16 @@ index edb8e2d..9c88feb 100644 + } + GGML_ASSERT(slot.task->need_sampling()); - + // prompt evaluated for next-token prediction diff --git a/tools/server/server-task.h b/tools/server/server-task.h -index 8c5fa9a..4b38805 100644 +index 810cefb..3e1ab8d 100644 --- a/tools/server/server-task.h +++ b/tools/server/server-task.h -@@ -12,11 +12,26 @@ - #include "server-common.h" - - +@@ -13,11 +13,26 @@ + #include "mtmd-helper.h" + + +// SERVER_TASK_TYPE_SCORE emits one logits output per candidate token (plus +// the forced last-token output), and the context's output budget +// (n_outputs_max) is reserved up front — so candidate length must be @@ -527,12 +527,12 @@ index 8c5fa9a..4b38805 100644 SERVER_TASK_TYPE_DECISION, + SERVER_TASK_TYPE_SCORE, SERVER_TASK_TYPE_INFILL, + SERVER_TASK_TYPE_TRANSCRIBE_RT, SERVER_TASK_TYPE_CANCEL, - SERVER_TASK_TYPE_CONTROL, -@@ -156,6 +171,18 @@ struct server_task { +@@ -168,6 +183,18 @@ struct server_task { task_params params; server_tokens tokens; - + + // used by SERVER_TASK_TYPE_SCORE: `tokens` holds the shared prefix + // (prompt + longest common candidate token prefix) and logprobs are + // returned for its tokens from n_score_prompt onward. Each candidate's @@ -548,18 +548,18 @@ index 8c5fa9a..4b38805 100644 // only used by CLI, this allow tokenizing CLI inputs on server side // we need this because mtmd_context and vocab are not accessible outside of server_context bool cli = false; -@@ -234,6 +261,7 @@ struct server_task { - switch (type) { +@@ -249,6 +276,7 @@ struct server_task { case SERVER_TASK_TYPE_COMPLETION: case SERVER_TASK_TYPE_INFILL: + case SERVER_TASK_TYPE_TRANSCRIBE_RT: + case SERVER_TASK_TYPE_SCORE: return true; - default: - return false; -@@ -509,6 +537,25 @@ struct server_task_result_decision : server_task_result { + case SERVER_TASK_TYPE_DECISION: + return !decision.labels.empty(); +@@ -558,6 +586,25 @@ struct server_task_result_decision : server_task_result { virtual json to_json() override; }; - + +struct server_task_result_score : server_task_result { + // log P(token | prefix) for the shared-prefix tokens after + // n_score_prompt, in order; NaN marks positions the decode never diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch deleted file mode 100644 index 07cdb8901..000000000 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ /dev/null @@ -1,850 +0,0 @@ -diff --git a/tools/mtmd/mtmd-helper-gen.cpp b/tools/mtmd/mtmd-helper-gen.cpp -index 5fb7ea9..b554bf2 100644 ---- a/tools/mtmd/mtmd-helper-gen.cpp -+++ b/tools/mtmd/mtmd-helper-gen.cpp -@@ -50,29 +50,38 @@ static llama_token find_special_token(const llama_vocab * vocab, const std::stri - return LLAMA_TOKEN_NULL; - } - -+static void put_bytes(std::vector & buf, const void * p, size_t n) { -+ const char * c = (const char *) p; -+ buf.insert(buf.end(), c, c + n); -+} -+ -+// data_sz == UINT32_MAX writes the "unknown length" sentinel (streaming), same as ffmpeg does on a pipe -+static void write_wav16_header(std::vector & buf, uint32_t data_sz, int32_t rate) { -+ const uint32_t riff_sz = data_sz == UINT32_MAX ? UINT32_MAX : 36 + data_sz; -+ const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; -+ const uint16_t fmt = 1, ch = 1, align = 2, bits = 16; -+ const uint32_t rate32 = (uint32_t) rate; -+ put_bytes(buf, "RIFF", 4); put_bytes(buf, &riff_sz, 4); put_bytes(buf, "WAVE", 4); -+ put_bytes(buf, "fmt ", 4); put_bytes(buf, &fmt_sz, 4); -+ put_bytes(buf, &fmt, 2); put_bytes(buf, &ch, 2); put_bytes(buf, &rate32, 4); -+ put_bytes(buf, &byte_rate, 4); put_bytes(buf, &align, 2); put_bytes(buf, &bits, 2); -+ put_bytes(buf, "data", 4); put_bytes(buf, &data_sz, 4); -+} -+ -+static void append_wav16_pcm(std::vector & buf, const float * pcm, size_t n) { -+ for (size_t i = 0; i < n; i++) { -+ int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, pcm[i])) * 32767.0f); -+ put_bytes(buf, &s, 2); -+ } -+} -+ - static bool write_wav16(std::vector & buf, const std::vector & pcm, int32_t rate) { - // RIFF chunk sizes are 32-bit; refuse to emit a file with a truncated header - if (pcm.size() > ((size_t) UINT32_MAX - 36) / 2) { - return false; - } -- const uint32_t data_sz = (uint32_t) (pcm.size() * 2); -- const uint32_t riff_sz = 36 + data_sz; -- const uint32_t fmt_sz = 16, byte_rate = (uint32_t) rate * 2; -- const uint16_t fmt = 1, ch = 1, align = 2, bits = 16; -- const uint32_t rate32 = (uint32_t) rate; -- auto put = [&](const void * p, size_t n) { -- const char * c = (const char *) p; -- buf.insert(buf.end(), c, c + n); -- }; -- put("RIFF", 4); put(&riff_sz, 4); put("WAVE", 4); -- put("fmt ", 4); put(&fmt_sz, 4); -- put(&fmt, 2); put(&ch, 2); put(&rate32, 4); -- put(&byte_rate, 4); put(&align, 2); put(&bits, 2); -- put("data", 4); put(&data_sz, 4); -- for (float v : pcm) { -- int16_t s = (int16_t) (std::max(-1.0f, std::min(1.0f, v)) * 32767.0f); -- put(&s, 2); -- } -+ write_wav16_header(buf, (uint32_t) (pcm.size() * 2), rate); -+ append_wav16_pcm(buf, pcm.data(), pcm.size()); - return true; - } - -@@ -92,6 +101,8 @@ public: - // set out_stop on end-of-speech, h_state_out must be null if no frame is generated - virtual int32_t step_gen(llama_token sampled, const float * h_state_in, const float ** h_state_out, bool * out_stop) = 0; - virtual int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) = 0; -+ // forces any buffered codes through code2wav now, regardless of window_frames -+ virtual int32_t flush() { return 0; } - - protected: - llama_context * lctx; -@@ -121,6 +132,9 @@ public: - prompt_batch.reset(); - n_prompt = 0; - prompt_pos = 0; -+ stream = false; -+ pcm_sent = 0; -+ wav_header_sent = false; - } - - int32_t set_input(const mtmd_helper_gen_audio_inp * inp) override { -@@ -208,6 +222,7 @@ public: - top_p = inp->top_p > 0 ? inp->top_p : def.top_p; - seed = inp->seed; - out_type = inp->out_type; -+ stream = inp->stream; - - // the prompt above holds the whole text stream up to tts_eos, so every generated - // frame adds tts_pad on top of the codes embedding -@@ -301,31 +316,60 @@ public: - } - - int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples) override { -- if (!flush_gen_wav()) { -- return 1; -+ *out_sample_rate = info.sample_rate; -+ -+ if (!stream) { -+ // one-shot call: force out whatever's left, regardless of window_frames -+ if (!flush_gen_wav()) { -+ return 1; -+ } -+ if (out_n_samples) { -+ *out_n_samples = (int64_t) audio_pcm.size(); -+ } -+ if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { -+ *out_data = (const char *) audio_pcm.data(); -+ *out_data_len = audio_pcm.size() * sizeof(float); -+ return 0; -+ } -+ out_buf.clear(); -+ if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { -+ LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); -+ return 1; -+ } -+ *out_data = out_buf.data(); -+ *out_data_len = out_buf.size(); -+ return 0; - } - -- *out_sample_rate = info.sample_rate; -+ // streaming: only return audio produced since the previous call -+ const size_t n_new = audio_pcm.size() - pcm_sent; - if (out_n_samples) { -- *out_n_samples = (int64_t) audio_pcm.size(); -+ *out_n_samples = (int64_t) n_new; - } - - if (out_type == MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM) { -- *out_data = (const char *) audio_pcm.data(); -- *out_data_len = audio_pcm.size() * sizeof(float); -+ *out_data = (const char *) (audio_pcm.data() + pcm_sent); -+ *out_data_len = n_new * sizeof(float); -+ pcm_sent = audio_pcm.size(); - return 0; - } - - out_buf.clear(); -- if (!write_wav16(out_buf, audio_pcm, info.sample_rate)) { -- LOG_ERR("mtmd_helper_gen_audio: output too large for WAV\n"); -- return 1; -+ if (!wav_header_sent) { -+ write_wav16_header(out_buf, UINT32_MAX, info.sample_rate); -+ wav_header_sent = true; - } -+ append_wav16_pcm(out_buf, audio_pcm.data() + pcm_sent, n_new); -+ pcm_sent = audio_pcm.size(); - *out_data = out_buf.data(); - *out_data_len = out_buf.size(); - return 0; - } - -+ int32_t flush() override { -+ return flush_gen_wav() ? 0 : 1; -+ } -+ - private: - bool ensure_cache() { - if (specials_ok) { -@@ -369,7 +413,7 @@ private: - LOG_ERR("mtmd_helper_gen_audio: mmproj has no speaker/audio encoder\n"); - return false; - } -- const std::string marker = mtmd_default_marker(); -+ const std::string marker = mtmd_get_marker(mctx); - mtmd_input_text text{ marker.c_str(), marker.size(), false, true }; - mtmd_input_chunks * chunks = mtmd_input_chunks_init(); - const mtmd_bitmap * bptr = bitmap; -@@ -455,6 +499,9 @@ private: - std::vector h_state_buf; - mtmd_helper_gen_audio_outtype out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; - std::vector out_buf; -+ bool stream = false; -+ size_t pcm_sent = 0; // samples already returned by get_output() -+ bool wav_header_sent = false; - }; - - // settings that only live in the reference's per-pack yaml, not in the checkpoint -@@ -1022,6 +1069,14 @@ void mtmd_helper_gen_audio_reset(mtmd_helper_gen_audio * ctx) { - } - } - -+struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void) { -+ mtmd_helper_gen_audio_inp inp{}; -+ inp.top_k = 50; -+ inp.top_p = 1.0f; -+ inp.out_type = MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; -+ return inp; -+} -+ - int32_t mtmd_helper_gen_audio_set_input(mtmd_helper_gen_audio * ctx, const mtmd_helper_gen_audio_inp * inp) { - if (!ctx->pipeline) { - LOG_ERR("mtmd_helper_gen_audio: unsupported or missing gen-audio pipeline\n"); -@@ -1058,3 +1113,10 @@ int32_t mtmd_helper_gen_audio_get_output(mtmd_helper_gen_audio * ctx, int32_t * - } - return ctx->pipeline->get_output(out_sample_rate, out_data, out_data_len, out_n_samples); - } -+ -+int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx) { -+ if (!ctx->pipeline) { -+ return 1; -+ } -+ return ctx->pipeline->flush(); -+} -diff --git a/tools/mtmd/mtmd-helper.h b/tools/mtmd/mtmd-helper.h -index 7436230..acbfefc 100644 ---- a/tools/mtmd/mtmd-helper.h -+++ b/tools/mtmd/mtmd-helper.h -@@ -204,6 +204,7 @@ enum mtmd_helper_gen_audio_outtype { - MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV, // WAV PCM 16-bit LE, mono - }; - struct mtmd_helper_gen_audio_inp { -+ bool stream; // if true, output() must be called after each step_gen() - llama_seq_id seq_id; - - const char * prompt; -@@ -219,6 +220,8 @@ struct mtmd_helper_gen_audio_inp { - enum mtmd_helper_gen_audio_outtype out_type; - }; - -+MTMD_API struct mtmd_helper_gen_audio_inp mtmd_helper_gen_audio_inp_default(void); -+ - MTMD_API mtmd_helper_gen_audio * mtmd_helper_gen_audio_init( - struct llama_context * lctx, - struct mtmd_context * mctx); -@@ -250,6 +253,8 @@ MTMD_API int32_t mtmd_helper_gen_audio_step_gen( - - // out_data valid until next get_output() or reset() call - // out_n_samples (optional, can be NULL) receives the number of generated PCM samples -+// if inp->stream is true: returns only audio produced since the previous call, and -+// *out_data_len == 0 whenever a full window_frames batch hasn't accumulated yet - MTMD_API int32_t mtmd_helper_gen_audio_get_output( - mtmd_helper_gen_audio * ctx, - int32_t * out_sample_rate, -@@ -257,6 +262,10 @@ MTMD_API int32_t mtmd_helper_gen_audio_get_output( - size_t * out_data_len, - int64_t * out_n_samples); - -+// forces any buffered codes through code2wav now, regardless of window_frames; -+// call once when generation has ended, before the last get_output() in stream mode -+MTMD_API int32_t mtmd_helper_gen_audio_flush(mtmd_helper_gen_audio * ctx); -+ - #ifdef __cplusplus - } // extern "C" - #endif -@@ -283,8 +292,41 @@ struct mtmd_helper_gen_audio_deleter { - }; - using gen_audio_ptr = std::unique_ptr; - struct gen_audio { -+ -+ // sub-struct, RAII wrapper for mtmd_helper_gen_audio_inp -+ struct inp { -+ mtmd_helper_gen_audio_inp data = mtmd_helper_gen_audio_inp_default(); -+ std::string prompt_str; -+ std::string lang_str; -+ mtmd::bitmap_ptr speaker_ref_ptr; -+ -+ inp() = default; -+ inp(inp &&) = default; -+ inp & operator=(inp &&) = default; -+ inp(const inp &) = delete; -+ inp & operator=(const inp &) = delete; -+ -+ void set_prompt (std::string p) { prompt_str = std::move(p); } -+ void set_lang (std::string l) { lang_str = std::move(l); } -+ void set_speaker_ref(mtmd::bitmap_ptr bmp) { speaker_ref_ptr = std::move(bmp); } -+ -+ // pointers are only valid as long as *this is alive -+ const mtmd_helper_gen_audio_inp * get() { -+ data.prompt = prompt_str.c_str(); -+ data.prompt_len = prompt_str.size(); -+ data.lang = lang_str.empty() ? nullptr : lang_str.c_str(); -+ data.speaker_ref = speaker_ref_ptr.get(); -+ return &data; -+ } -+ }; -+ - gen_audio_ptr ctx; -- gen_audio(struct llama_context * lctx, struct mtmd_context * mctx) : ctx(mtmd_helper_gen_audio_init(lctx, mctx)) {} -+ void init(struct llama_context * lctx, struct mtmd_context * mctx) { -+ ctx.reset(mtmd_helper_gen_audio_init(lctx, mctx)); -+ } -+ bool valid() const { -+ return ctx.get() != nullptr; -+ } - void reset() { - mtmd_helper_gen_audio_reset(ctx.get()); - } -@@ -300,6 +342,9 @@ struct gen_audio { - int32_t get_output(int32_t * out_sample_rate, const char ** out_data, size_t * out_data_len, int64_t * out_n_samples = nullptr) { - return mtmd_helper_gen_audio_get_output(ctx.get(), out_sample_rate, out_data, out_data_len, out_n_samples); - } -+ int32_t flush() { -+ return mtmd_helper_gen_audio_flush(ctx.get()); -+ } - }; - - } // namespace mtmd_helper -diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp -index 9c88feb..064bb51 100644 ---- a/tools/server/server-context.cpp -+++ b/tools/server/server-context.cpp -@@ -17,6 +17,7 @@ - #include "speculative.h" - #include "mtmd.h" - #include "mtmd-helper.h" -+#include "base64.hpp" - - #include - #include -@@ -41,8 +42,9 @@ constexpr int HTTP_POLLING_SECONDS = 1; - - static common_speculative_output_limits server_output_limits(const common_params & params) { - if (params.embedding || -- (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE)) { -- return { params.n_batch, 1 }; -+ (params.pooling_type != LLAMA_POOLING_TYPE_UNSPECIFIED && params.pooling_type != LLAMA_POOLING_TYPE_NONE) || -+ !params.mmproj.path.empty()) { // gen-audio (TTS) capability isn't known until the mmproj loads, size generously -+ return { params.n_batch, params.n_batch }; - } - - auto result = common_speculative_get_output_limits( -@@ -215,6 +217,30 @@ struct server_slot { - mtmd_context * mctx = nullptr; - mtmd::batch_ptr mbatch = nullptr; - -+ struct tts_ctx { -+ mtmd_helper::gen_audio ctx; -+ const float * h_state; -+ llama_token sampled; -+ int32_t n_decoded; -+ bool is_supported() const { -+ return ctx.valid(); -+ } -+ void reset() { -+ // mtmd_helper_gen_audio_reset() dereferences its argument before it -+ // null-checks the pipeline, and the pipeline is only allocated for -+ // models that actually carry a gen-audio mmproj. server_slot::reset() -+ // runs for every slot of every model, so without this guard any -+ // non-TTS model segfaults during slot initialization. -+ if (is_supported()) { -+ ctx.reset(); -+ } -+ h_state = nullptr; -+ sampled = LLAMA_TOKEN_NULL; -+ n_decoded = 0; -+ } -+ }; -+ tts_ctx tts; -+ - // speculative decoding - common_speculative * spec; - -@@ -395,6 +421,8 @@ struct server_slot { - - // clear multimodal state - mbatch.reset(); -+ -+ tts.reset(); - } - - void init_sampler() const { -@@ -871,6 +899,14 @@ public: - - server_decision_context decision; - -+ bool has_cap_tts() const { -+ return mctx != nullptr && mtmd_gen_audio_get_info(mctx).type != MTMD_GEN_AUDIO_TYPE_NONE; -+ } -+ -+ bool has_cap_chat() const { -+ return mctx == nullptr || mtmd_helper_model_can_chat(ctx_tgt, mctx); -+ } -+ - server_queue queue_tasks; - server_response queue_results; - -@@ -1350,6 +1386,10 @@ private: - slot.mctx = mctx; - slot.prompt.tokens.has_mtmd = mctx != nullptr; - -+ if (has_cap_tts()) { -+ slot.tts.ctx.init(ctx_tgt, mctx); -+ } -+ - SLT_TRC(slot, "new slot, n_ctx = %d\n", slot.n_ctx); - - slot.callback_on_release = [this](int id_slot) { -@@ -1830,6 +1870,28 @@ private: - - SLT_DBG(slot, "launching slot : %s\n", safe_json_to_str(slot.to_json()).c_str()); - -+ if (task.type == SERVER_TASK_TYPE_TTS) { -+ GGML_ASSERT(has_cap_tts()); // should already checked in route handler -+ if (!slot.tts.is_supported()) { -+ slot.tts.ctx.init(ctx_tgt, slot.mctx); -+ } -+ -+ // TTS slots never enter the shared batch: pre_decode() returns early for -+ // them and process_tts_slots() drives them instead, so they skip the -+ // prompt-cache bookkeeping that clears this sequence between requests. -+ // The gen-audio pipeline always decodes from position 0, and its own -+ // reset() only clears host-side buffers, so without this the second and -+ // later tasks on a slot decode over the previous request's tokens and -+ // step_prompt() fails immediately. -+ slot.prompt_clear(); -+ -+ task.tts_inp.data.seq_id = slot.id; -+ if (slot.tts.ctx.set_input(task.tts_inp.get()) != 0) { -+ send_error(task, "failed to process TTS prompt", ERROR_TYPE_SERVER); -+ return false; -+ } -+ } -+ - // initialize samplers - if (task.need_sampling()) { - try { -@@ -1847,6 +1909,9 @@ private: - // TODO: getting pre sampling logits is not yet supported with backend sampling - use_backend_sampling &= !need_pre_sample_logits; - -+ // TODO: check verify if this actually works with TTS -+ use_backend_sampling &= task.type != SERVER_TASK_TYPE_TTS; -+ - // TODO: tmp until backend sampling is fully implemented - if (use_backend_sampling) { - llama_set_sampler(ctx_tgt, slot.id, common_sampler_get(slot.smpl.get())); -@@ -1872,9 +1937,13 @@ private: - - slot.task = std::make_unique(std::move(task)); - -- slot.state = slot.task->is_child() -- ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt -- : SLOT_STATE_STARTED; -+ if (slot.task->type == SERVER_TASK_TYPE_TTS) { -+ slot.state = SLOT_STATE_PROCESSING_PROMPT; -+ } else { -+ slot.state = slot.task->is_child() -+ ? SLOT_STATE_WAIT_OTHER // wait for the parent to process prompt -+ : SLOT_STATE_STARTED; -+ } - - // reset server kill-switch counter - n_empty_consecutive = 0; -@@ -2139,6 +2208,18 @@ private: - queue_results.send(std::move(res)); - } - -+ void send_tts_result(server_slot & slot, int32_t sample_rate, const char * data, size_t data_len, bool final) { -+ auto res = std::make_unique(); -+ -+ res->id = slot.task->id; -+ res->index = slot.task->index; -+ res->sample_rate = sample_rate; -+ res->audio.assign(data, data_len); -+ res->final = final; -+ -+ queue_results.send(std::move(res)); -+ } -+ - void send_final_response(server_slot & slot) { - auto res = std::make_unique(); - -@@ -2721,6 +2802,7 @@ private: - case SERVER_TASK_TYPE_RERANK: - case SERVER_TASK_TYPE_DECISION: - case SERVER_TASK_TYPE_SCORE: -+ case SERVER_TASK_TYPE_TTS: - { - // special case: if input is provided via CLI, tokenize it first - // otherwise, no need to tokenize as it's already done inside the HTTP thread -@@ -3179,6 +3261,14 @@ private: - return; - } - -+ // note: TTS slots bypass the shared batch entirely -+ try { -+ process_tts_slots(); -+ } catch (const std::exception & e) { -+ SRV_ERR("process_tts_slots() failed: %s\n", e.what()); -+ abort_all_slots("process_tts_slots() failed: " + std::string(e.what())); -+ } -+ - GGML_ASSERT(batch.slot_batched || batch.size() == 0); - - if (batch.slot_batched) { -@@ -3248,10 +3338,77 @@ private: - } - } - -+ void process_tts_slots() { -+ iterate(slots, [&](server_slot & slot) { -+ if (!slot.is_processing() || slot.task->type != SERVER_TASK_TYPE_TTS) { -+ return; -+ } -+ -+ llama_set_embeddings(ctx_tgt, true); -+ -+ if (slot.state == SLOT_STATE_PROCESSING_PROMPT) { -+ const int32_t ret = slot.tts.ctx.step_prompt(llama_n_batch(ctx_tgt)); -+ if (ret < 0) { -+ send_error(slot, "TTS prompt processing failed", ERROR_TYPE_SERVER); -+ slot.release(); -+ } else if (ret == 0) { -+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1); -+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true); -+ slot.tts.h_state = llama_get_embeddings_ith(ctx_tgt, -1); -+ slot.state = SLOT_STATE_GENERATING; -+ } -+ return; -+ } -+ -+ const int32_t n_predict = slot.task->params.n_predict > 0 ? slot.task->params.n_predict : 512; -+ if (slot.tts.n_decoded >= n_predict || llama_vocab_is_eog(vocab, slot.tts.sampled)) { -+ int32_t sample_rate = 0; -+ const char * data = nullptr; -+ size_t data_len = 0; -+ // generation truly ends here: force out any sub-window remainder still buffered -+ if (slot.tts.ctx.flush() != 0 || slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) { -+ send_error(slot, "failed to finalize TTS output", ERROR_TYPE_SERVER); -+ } else { -+ send_tts_result(slot, sample_rate, data, data_len, true); -+ } -+ slot.release(); -+ return; -+ } -+ -+ const float * h_state_next = nullptr; -+ if (slot.tts.ctx.step_gen(slot.tts.sampled, slot.tts.h_state, &h_state_next) != 0) { -+ send_error(slot, "TTS generation failed", ERROR_TYPE_SERVER); -+ slot.release(); -+ return; -+ } -+ slot.tts.h_state = h_state_next; -+ slot.tts.n_decoded++; -+ -+ slot.tts.sampled = common_sampler_sample(slot.smpl.get(), ctx_tgt, -1); -+ common_sampler_accept(slot.smpl.get(), slot.tts.sampled, true); -+ -+ if (slot.task->params.stream) { -+ int32_t sample_rate = 0; -+ const char * data = nullptr; -+ size_t data_len = 0; -+ if (slot.tts.ctx.get_output(&sample_rate, &data, &data_len) != 0) { -+ send_error(slot, "TTS streaming output failed", ERROR_TYPE_SERVER); -+ slot.release(); -+ } else if (data_len > 0) { -+ send_tts_result(slot, sample_rate, data, data_len, false); -+ } -+ } -+ }); -+ } -+ - void pre_decode() { - // apply context-shift if needed - // TODO: simplify and improve - iterate(slots, [&](server_slot & slot) { -+ if (slot.task && slot.task->type == SERVER_TASK_TYPE_TTS) { -+ // TTS slots drive their own decode loop in process_tts_slots(), never enter the shared batch -+ return; -+ } - if (slot.state == SLOT_STATE_GENERATING && slot.prompt.n_tokens() + 1 >= slot.n_ctx) { - if (!params_base.ctx_shift) { - // this check is redundant (for good) -@@ -3324,7 +3481,7 @@ private: - - // determine which slots are generating and drafting - iterate(slots, [&](server_slot & slot) { -- if (slot.state != SLOT_STATE_GENERATING) { -+ if (slot.state != SLOT_STATE_GENERATING || slot.task->type == SERVER_TASK_TYPE_TTS) { - return; - } - -@@ -3458,7 +3615,7 @@ private: - return; // batch is full, skip remaining slots - } - -- if (!slot.is_processing()) { -+ if (!slot.is_processing() || slot.task->type == SERVER_TASK_TYPE_TTS) { - return; - } - -@@ -4678,6 +4835,8 @@ server_context_meta server_context::get_meta() const { - /* has_inp_image */ impl->chat_params.allow_image, - /* has_inp_audio */ impl->chat_params.allow_audio, - /* has_inp_video */ impl->chat_params.allow_video, -+ /* has_cap_chat */ impl->has_cap_chat(), -+ /* has_cap_tts */ impl->has_cap_tts(), - /* json_ui_settings */ impl->json_ui_settings, - /* slot_n_ctx */ impl->n_ctx_slot(), - /* pooling_type */ llama_pooling_type(impl->ctx_tgt), -@@ -4751,6 +4910,11 @@ std::unique_ptr server_routes::handle_completions_impl( - - res->set_req(&req); // will also set spipe if needed - -+ if (!ctx_server.has_cap_chat()) { -+ res->error(format_error_response("this server does not support chat/completions", ERROR_TYPE_NOT_SUPPORTED)); -+ return res; -+ } -+ - int32_t sse_ping_interval = params.sse_ping_interval; - - try { -@@ -5776,6 +5940,150 @@ void server_routes::init_routes() { - return res; - }; - -+ this->post_tts = [this](const server_http_req & req) { -+ auto res = create_response(); -+ res->set_req(&req); // will also set spipe if needed -+ -+ if (!ctx_server.has_cap_tts()) { -+ res->error(format_error_response("this server does not support audio generation", ERROR_TYPE_NOT_SUPPORTED)); -+ return res; -+ } -+ -+ const json body = json::parse(req.body); -+ -+ std::string prompt = json_value(body, "input", json_value(body, "prompt", std::string())); -+ if (prompt.empty()) { -+ res->error(format_error_response("\"input\" must be a non-empty string", ERROR_TYPE_INVALID_REQUEST)); -+ return res; -+ } -+ -+ const std::string response_format = json_value(body, "response_format", std::string("wav")); -+ const bool stream = json_value(body, "stream", false); -+ -+ server_task task(SERVER_TASK_TYPE_TTS); -+ task.tts_inp.set_prompt(prompt); -+ task.tts_inp.set_lang(json_value(body, "lang", std::string())); -+ task.tts_inp.data.top_k = json_value(body, "top_k", 0); -+ task.tts_inp.data.top_p = json_value(body, "top_p", 0.0f); -+ task.tts_inp.data.stream = stream; -+ task.tts_inp.data.out_type = response_format == "pcm" -+ ? MTMD_HELPER_GEN_AUDIO_OUTTYPE_PCM -+ : MTMD_HELPER_GEN_AUDIO_OUTTYPE_WAV; -+ task.params.stream = stream; -+ task.params.n_predict = json_value(body, "n_predict", -1); -+ task.params.sampling = params.sampling; // baseline defaults, then apply overrides below -+ task.params.sampling.penalty_repeat = json_value(body, "repeat_penalty", 1.05f); -+ task.params.sampling.penalty_last_n = -1; -+ if (task.tts_inp.data.top_k > 0) { -+ task.params.sampling.top_k = task.tts_inp.data.top_k; -+ } -+ if (task.tts_inp.data.top_p > 0) { -+ task.params.sampling.top_p = task.tts_inp.data.top_p; -+ } -+ -+ // speaker reference: either an uploaded form file ("speaker_ref") or a base64 JSON field ("speaker_ref_b64") -+ const unsigned char * speaker_ref_data = nullptr; -+ size_t speaker_ref_len = 0; -+ std::string speaker_ref_b64_decoded; -+ -+ auto speaker_ref_file = req.files.find("speaker_ref"); -+ if (speaker_ref_file != req.files.end()) { -+ speaker_ref_data = speaker_ref_file->second.data.data(); -+ speaker_ref_len = speaker_ref_file->second.data.size(); -+ } else { -+ std::string speaker_ref_b64 = json_value(body, "speaker_ref_b64", std::string()); -+ if (!speaker_ref_b64.empty()) { -+ speaker_ref_b64_decoded = base64::decode(speaker_ref_b64); -+ speaker_ref_data = (const unsigned char *) speaker_ref_b64_decoded.data(); -+ speaker_ref_len = speaker_ref_b64_decoded.size(); -+ } -+ } -+ -+ if (speaker_ref_len > 0) { -+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); -+ if (!wrapper.bitmap) { -+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); -+ return res; -+ } -+ task.tts_inp.set_speaker_ref(mtmd::bitmap_ptr(wrapper.bitmap)); -+ } else { -+ // SRV_WRN expands __VA_ARGS__ without the GNU comma-elision extension, -+ // so a bare format string leaves a trailing comma and will not compile -+ SRV_WRN("%s", "no speaker reference provided, the model may behave randomly\n"); -+ } -+ -+ auto & rd = res->rd; -+ task.id = rd.get_new_id(); -+ rd.post_task(std::move(task)); -+ -+ const std::string content_type = response_format == "pcm" ? "audio/L16" : "audio/wav"; -+ -+ if (!stream) { -+ auto result = rd.next(req.should_stop); -+ if (!result) { -+ GGML_ASSERT(req.should_stop()); -+ return res; // connection is closed -+ } -+ if (result->is_error()) { -+ res->error(result->to_json()); -+ return res; -+ } -+ auto * tts_res = dynamic_cast(result.get()); -+ GGML_ASSERT(tts_res != nullptr); -+ res->status = 200; -+ res->content_type = content_type; -+ res->data = std::move(tts_res->audio); -+ return res; -+ } else { -+ auto first_result = rd.next(req.should_stop); -+ if (!first_result) { -+ GGML_ASSERT(req.should_stop()); -+ return res; // connection is closed -+ } -+ if (first_result->is_error()) { -+ res->error(first_result->to_json()); -+ return res; -+ } -+ auto * first_tts_res = dynamic_cast(first_result.get()); -+ GGML_ASSERT(first_tts_res != nullptr); -+ -+ res->status = 200; -+ res->content_type = content_type; -+ res->data = std::move(first_tts_res->audio); -+ bool is_done = first_tts_res->final; -+ -+ res->set_next([res_this = res.get(), is_done](std::string & output) mutable -> bool { -+ if (is_done) { -+ return false; -+ } -+ if (res_this->should_stop()) { -+ return false; -+ } -+ if (!res_this->data.empty()) { -+ output = std::move(res_this->data); -+ res_this->data.clear(); -+ return true; -+ } -+ -+ server_response_reader & rd = res_this->rd; -+ if (!rd.has_next()) { -+ return false; -+ } -+ auto result = rd.next([&res_this]() { return res_this->should_stop(); }); -+ if (!result || result->is_error()) { -+ return false; -+ } -+ auto * tts_res = dynamic_cast(result.get()); -+ GGML_ASSERT(tts_res != nullptr); -+ output = std::move(tts_res->audio); -+ is_done = tts_res->final; -+ return true; -+ }); -+ } -+ -+ return res; -+ }; -+ - this->get_lora_adapters = [this](const server_http_req & req) { - auto res = create_response(); - -diff --git a/tools/server/server-context.h b/tools/server/server-context.h -index c554bb9..f025792 100644 ---- a/tools/server/server-context.h -+++ b/tools/server/server-context.h -@@ -23,6 +23,8 @@ struct server_context_meta { - bool has_inp_image; - bool has_inp_audio; - bool has_inp_video; -+ bool has_cap_chat; -+ bool has_cap_tts; - json json_ui_settings; - int slot_n_ctx; - enum llama_pooling_type pooling_type; -@@ -152,6 +154,7 @@ struct server_routes { - server_http_context::handler_t post_embeddings; - server_http_context::handler_t post_embeddings_oai; - server_http_context::handler_t post_rerank; -+ server_http_context::handler_t post_tts; - server_http_context::handler_t post_systemone; - server_http_context::handler_t get_lora_adapters; - server_http_context::handler_t post_lora_adapters; -diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cpp -index a5c33c0..7e59745 100644 ---- a/tools/server/server-task.cpp -+++ b/tools/server/server-task.cpp -@@ -1506,6 +1506,17 @@ json server_task_result_decision::to_json() { - }; - } - -+// -+// server_task_result_tts -+// -+json server_task_result_tts::to_json() { -+ return json { -+ {"sample_rate", sample_rate}, -+ {"n_bytes", audio.size()}, -+ {"final", final}, -+ }; -+} -+ - // - // server_task_result_error - // -diff --git a/tools/server/server-task.h b/tools/server/server-task.h -index 4b38805..d79ea4f 100644 ---- a/tools/server/server-task.h -+++ b/tools/server/server-task.h -@@ -10,6 +10,7 @@ - - // TODO: prevent including the whole server-common.h as we only use server_tokens - #include "server-common.h" -+#include "mtmd-helper.h" - - - // SERVER_TASK_TYPE_SCORE emits one logits output per candidate token (plus -@@ -43,6 +44,7 @@ enum server_task_type { - SERVER_TASK_TYPE_SLOT_ERASE, - SERVER_TASK_TYPE_GET_LORA, - SERVER_TASK_TYPE_SET_LORA, -+ SERVER_TASK_TYPE_TTS, - }; - - // TODO: change this to more generic "response_format" to replace the "format_response_*" in server-common -@@ -225,6 +227,9 @@ struct server_task { - // used by SERVER_TASK_TYPE_SET_LORA - std::map set_lora; // mapping adapter ID -> scale - -+ // used by SERVER_TASK_TYPE_TTS -+ mtmd_helper::gen_audio::inp tts_inp; -+ - server_task() = default; - - server_task(server_task_type type) : type(type) {} -@@ -249,6 +254,7 @@ struct server_task { - switch (type) { - case SERVER_TASK_TYPE_COMPLETION: - case SERVER_TASK_TYPE_INFILL: -+ case SERVER_TASK_TYPE_TTS: - return true; - case SERVER_TASK_TYPE_DECISION: - return !decision.labels.empty(); -@@ -521,5 +527,15 @@ struct server_task_result_embd : server_task_result { - json to_json_oaicompat(); - }; - -+struct server_task_result_tts : server_task_result { -+ std::string audio; // raw bytes for this chunk (WAV or PCM, per request's out_type) -+ int32_t sample_rate = 0; -+ bool final = false; // true for the last chunk of a request -+ -+ virtual bool is_stop() override { return final; } -+ -+ virtual json to_json() override; -+}; -+ - struct server_task_result_rerank : server_task_result { - float score = -1e6; diff --git a/gallery/index.yaml b/gallery/index.yaml index 981cc3563..8491bd92b 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -77044,3 +77044,120 @@ - detector_interval:1 parameters: model: gem-x/gem-x-contact-f32.gguf +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: flint + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q4_K_XL GGUF quantization (25 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + sha256: 63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + variants: + - model: flint-q2 + - model: flint-q6 +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: flint-q2 + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q2_K_XL GGUF quantization (13.6 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + sha256: 7f43dd99e7d574c43c3f1b24e0e8269cf1ec92ba6c25f11b3ff126cde4c7a0cb + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true +- url: github:mudler/LocalAI/gallery/virtual.yaml@master + name: flint-q6 + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q6_K_XL GGUF quantization (36 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + sha256: 086231093581c67f8bf2964f8b49198889c4f0775b39857ba63b4c37041279c2 + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true diff --git a/gallery/models/index.yaml b/gallery/models/index.yaml index da124ab3a..4a448f245 100644 --- a/gallery/models/index.yaml +++ b/gallery/models/index.yaml @@ -94741,3 +94741,144 @@ - detector_interval:1 parameters: model: gem-x/gem-x-contact-f32.gguf +- kind: model + id: flint + name: flint + version: 0.0.0-gallery.a824f12de5cd + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q4_K_XL GGUF quantization (25 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + sha256: 63c2ecc461ca545134b670c660e76fa20f47240f8d39b6510bfd21c4dd6e072f + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q4_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true + variants: + - model: flint-q2 + - model: flint-q6 +- kind: model + id: flint-q2 + name: flint-q2 + version: 0.0.0-gallery.0522ec7a82c2 + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q2_K_XL GGUF quantization (13.6 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + sha256: 7f43dd99e7d574c43c3f1b24e0e8269cf1ec92ba6c25f11b3ff126cde4c7a0cb + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q2_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true +- kind: model + id: flint-q6 + name: flint-q6 + version: 0.0.0-gallery.6df3c309be93 + description: | + Flint is a mixture-of-experts coding-agent model from Recursive, with + 39.8B total and about 2.5B active parameters per token. It takes text in + and gives text out, and supports reasoning and multi-turn tool use. This + entry uses the Unsloth UD-Q6_K_XL GGUF quantization (36 GB). + license: apache-2.0 + urls: + - https://huggingface.co/RecursiveSI/flint + - https://huggingface.co/unsloth/flint-GGUF + tags: + - llm + - gguf + - cpu + - gpu + - moe + - coding + - agent + - reasoning + - tools + channel: gallery + policy: + requireSignature: false + schemaVersion: 1 + model: + url: github:mudler/LocalAI/gallery/virtual.yaml@master + files: + - filename: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + sha256: 086231093581c67f8bf2964f8b49198889c4f0775b39857ba63b4c37041279c2 + uri: huggingface://unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + overrides: + backend: llama-cpp + known_usecases: + - chat + options: + - use_jinja:true + parameters: + model: llama-cpp/models/unsloth/flint-GGUF/flint-UD-Q6_K_XL.gguf + temperature: 1 + top_k: 20 + top_p: 0.95 + template: + use_tokenizer_template: true