From 959de86761acf3d4dc7260c6e252781cbbd7c900 Mon Sep 17 00:00:00 2001 From: "LocalAI [bot]" <139863280+localai-bot@users.noreply.github.com> Date: Thu, 21 May 2026 16:31:48 +0200 Subject: [PATCH] feat(llama-cpp): make server-side prompt cache work by default (#9925) Aligns LocalAI's llama-cpp gRPC backend with upstream's auto-on prompt cache path so repeated system prompts (agents, OpenAI/Anthropic-compatible CLIs, coding assistants) skip prefill on subsequent calls without any YAML changes. Reported in #9921. Upstream's server enables `kv_unified=true` (and bumps `n_parallel` to 4) when slot count is auto, which unlocks `cache_idle_slots`. LocalAI hardcodes `n_parallel=1` and so far also hardcoded `kv_unified=false`, which silently force-disables idle-slot saving at server init. The host prompt cache was allocated but never written across requests. Changes in backend/cpp/llama-cpp/grpc-server.cpp: - params.kv_unified: false -> true (single-slot path now benefits from the prompt cache; users can opt out with `kv_unified:false`) - params.n_ctx_checkpoints: 8 -> 32 (match upstream default) - params.cache_idle_slots = true initialized explicitly (upstream default) - params.checkpoint_every_nt = 8192 initialized explicitly (upstream default) - New option parsers: cache_idle_slots / idle_slots_cache, checkpoint_every_nt / checkpoint_every_n_tokens Docs: - features/text-generation.md: fix misleading `cache_ram` description (it's the host-side prompt cache, not the KV cache), document the kv_unified + cache_ram + cache_idle_slots interaction, add rows for the two newly-exposed options, and add a worked example for the agent/CLI workload from the issue. - advanced/model-configuration.md: mark the legacy `prompt_cache_path` / `prompt_cache_all` / `prompt_cache_ro` YAML fields as unused by the llama-cpp gRPC backend (they target upstream's CLI completion tool and are not consumed by grpc-server.cpp) and point readers at the new prompt-cache explainer. Closes #9921 Assisted-by: claude:opus-4.7 Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto --- backend/cpp/llama-cpp/grpc-server.cpp | 49 ++++++++++++++++++-- docs/content/advanced/model-configuration.md | 10 ++-- docs/content/features/text-generation.md | 29 ++++++++++-- 3 files changed, 77 insertions(+), 11 deletions(-) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 61e3f7ee3..bc51f06e5 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -517,10 +517,27 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt params.warmup = true; // no_op_offload: disable host tensor op offload (default: false) params.no_op_offload = false; - // kv_unified: enable unified KV cache (default: false) - params.kv_unified = false; - // n_ctx_checkpoints: max context checkpoints per slot (default: 8) - params.n_ctx_checkpoints = 8; + // kv_unified: enable unified KV cache. Upstream's server auto-enables this + // when the slot count is auto (-np <0), bumping n_parallel to 4 alongside. + // LocalAI keeps n_parallel=1 by default, which would skip that auto path + // and leave kv_unified=false. We flip the default to true here so the + // server-side prompt cache (cache_idle_slots) is actually usable on the + // single-slot path that LocalAI ships with: without it, idle slots are + // never persisted across requests and the prompt cache is dead weight. + // Users can opt out with `options: [ "kv_unified:false" ]`. + params.kv_unified = true; + // n_ctx_checkpoints: max context checkpoints per slot. Match upstream's + // default (32); the previous LocalAI-specific 8 was unnecessarily tight + // and limits partial-prefix recovery without a clear memory rationale. + params.n_ctx_checkpoints = 32; + // cache_idle_slots: save and clear idle slot KV to the prompt cache on + // task switch. Upstream default is true; the server auto-disables it if + // kv_unified=false or cache_ram_mib=0, so flipping kv_unified above is + // what actually unlocks it. + params.cache_idle_slots = true; + // checkpoint_every_nt: create a context checkpoint every N tokens during + // prefill (-1 disables). Match upstream's default (8192). + params.checkpoint_every_nt = 8192; // decode options. Options are in form optname:optvale, or if booleans only optname. for (int i = 0; i < request->options_size(); i++) { @@ -679,7 +696,29 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt try { params.n_ctx_checkpoints = std::stoi(optval_str); } catch (const std::exception& e) { - // If conversion fails, keep default value (8) + // If conversion fails, keep default value (32) + } + } + + // --- server-side idle-slot prompt cache toggle (upstream --cache-idle-slots) --- + // Saves the slot's KV state into the host-side prompt cache on task + // switch so a later request with the same prefix can warm-load it. + // Auto-disabled by the server if kv_unified=false or cache_ram=0. + } else if (!strcmp(optname, "cache_idle_slots") || !strcmp(optname, "idle_slots_cache")) { + if (optval_str == "true" || optval_str == "1" || optval_str == "yes" || optval_str == "on" || optval_str == "enabled") { + params.cache_idle_slots = true; + } else if (optval_str == "false" || optval_str == "0" || optval_str == "no" || optval_str == "off" || optval_str == "disabled") { + params.cache_idle_slots = false; + } + + // --- prefill checkpoint cadence (upstream -cpent / --checkpoint-every-n-tokens) --- + // -1 disables checkpointing during prefill. + } else if (!strcmp(optname, "checkpoint_every_nt") || !strcmp(optname, "checkpoint_every_n_tokens")) { + if (optval != NULL) { + try { + params.checkpoint_every_nt = std::stoi(optval_str); + } catch (const std::exception& e) { + // If conversion fails, keep default value (8192) } } diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index b53c33858..20277f8de 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -444,11 +444,15 @@ These llama.cpp options are passed through the `options:` array. ### Prompt Caching +The recommended way to enable prompt caching for the `llama-cpp` backend is the **server-side prompt cache** controlled by `cache_ram` / `kv_unified` / `cache_idle_slots` in the `options:` array (see [llama.cpp backend options]({{%relref "features/text-generation#server-side-prompt-cache-repeated-system-prompts" %}})). It's on by default since LocalAI v4.3 and is what gives repeated system prompts a near-zero prefill on the second call. + +The fields below come from upstream llama.cpp's **CLI completion tool** and are passed through to the gRPC backend for compatibility, but the gRPC server itself does not consume them: keep them empty unless you're targeting a non-llama-cpp backend that reads them. + | Field | Type | Description | |-------|------|-------------| -| `prompt_cache_path` | string | Path to store prompt cache (relative to models directory) | -| `prompt_cache_all` | bool | Cache all prompts automatically | -| `prompt_cache_ro` | bool | Read-only prompt cache | +| `prompt_cache_path` | string | (legacy / unused by llama-cpp gRPC server) Path to a file-backed prompt cache for upstream's CLI completion tool. | +| `prompt_cache_all` | bool | (legacy / unused by llama-cpp gRPC server) | +| `prompt_cache_ro` | bool | (legacy / unused by llama-cpp gRPC server) | ### Text Processing diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index 5ef0ba489..ae2646e58 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -499,7 +499,7 @@ The `llama.cpp` backend supports additional configuration options that can be sp |--------|------|-------------|---------| | `use_jinja` or `jinja` | boolean | Enable Jinja2 template processing for chat templates. When enabled, the backend uses Jinja2-based chat templates from the model for formatting messages. | `use_jinja:true` | | `context_shift` | boolean | Enable context shifting, which allows the model to dynamically adjust context window usage. | `context_shift:true` | -| `cache_ram` | integer | Set the maximum RAM cache size in MiB for KV cache. Use `-1` for unlimited (default). | `cache_ram:2048` | +| `cache_ram` | integer | Size budget in MiB for the **server-side prompt cache** (a host-RAM store of idle slot KV states that's reloaded on a prompt-prefix hit, see [upstream PR #16391](https://github.com/ggml-org/llama.cpp/pull/16391)). Default: `-1` (no limit). `0` disables the prompt cache entirely. Together with `kv_unified` and `cache_idle_slots` this is what makes a repeated system prompt skip prefill on subsequent calls. | `cache_ram:4096` | | `parallel` or `n_parallel` | integer | Enable parallel request processing. When set to a value greater than 1, enables continuous batching for handling multiple requests concurrently. | `parallel:4` | | `grpc_servers` or `rpc_servers` | string | Comma-separated list of gRPC server addresses for distributed inference. Allows distributing workload across multiple llama.cpp workers. | `grpc_servers:localhost:50051,localhost:50052` | | `fit_params` or `fit` | boolean | Enable auto-adjustment of model/context parameters to fit available device memory. Default: `true`. | `fit_params:true` | @@ -512,8 +512,10 @@ The `llama.cpp` backend supports additional configuration options that can be sp | `check_tensors` | boolean | Validate tensor data for invalid values during model loading. Default: `false`. | `check_tensors:true` | | `warmup` | boolean | Enable warmup run after model loading. Default: `true`. | `warmup:false` | | `no_op_offload` | boolean | Disable offloading host tensor operations to device. Default: `false`. | `no_op_offload:true` | -| `kv_unified` or `unified_kv` | boolean | Enable unified KV cache. Default: `false`. | `kv_unified:true` | -| `n_ctx_checkpoints` or `ctx_checkpoints` | integer | Maximum number of context checkpoints per slot. Default: `8`. | `ctx_checkpoints:4` | +| `kv_unified` or `unified_kv` | boolean | Use a single unified KV buffer shared across all sequences. Default: `true` (LocalAI override; upstream defaults to `false` but auto-enables it when slot count is auto). **Required for `cache_idle_slots` to work**: without it the server force-disables idle-slot saving at init, and the prompt cache is never written across requests. | `kv_unified:false` | +| `cache_idle_slots` or `idle_slots_cache` | boolean | On a new task, save the previous slot's KV state into the prompt cache (and clear the slot) so a later request with the same prefix can warm-load it. Default: `true`. Auto-disabled by the server if `kv_unified=false` or `cache_ram=0`. | `cache_idle_slots:false` | +| `n_ctx_checkpoints` or `ctx_checkpoints` | integer | Maximum number of context checkpoints per slot (used for partial-prefix recovery, e.g. SWA). Default: `32`. | `ctx_checkpoints:16` | +| `checkpoint_every_nt` or `checkpoint_every_n_tokens` | integer | Create a context checkpoint every N tokens during prefill. `-1` disables checkpointing. Default: `8192`. | `checkpoint_every_nt:4096` | | `split_mode` or `sm` | string | How to split the model across multiple GPUs: `none` (single GPU only), `layer` (default — split layers and KV across GPUs), `row` (split rows across GPUs), `tensor` (experimental tensor parallelism — requires `flash_attention: true`, no KV-cache quantization, manually set `context_size`, and a llama.cpp build that includes [#19378](https://github.com/ggml-org/llama.cpp/pull/19378)). | `split_mode:tensor` | **Example configuration with options:** @@ -535,6 +537,27 @@ options: **Note:** The `parallel` option can also be set via the `LLAMACPP_PARALLEL` environment variable, and `grpc_servers` can be set via the `LLAMACPP_GRPC_SERVERS` environment variable. Options specified in the YAML file take precedence over environment variables. +##### Server-side prompt cache (repeated system prompts) + +Agents, coding assistants, and Anthropic/OpenAI-compatible CLIs typically resend the same large system prompt on every turn. The llama.cpp server can short-circuit prefill for the matching prefix by stashing idle slot KV states in host RAM and reloading them on a hit. Three settings interact: + +| Setting | Default | Role | +|---|---|---| +| `cache_ram:N` | `-1` (no limit) | Allocates the host-side prompt cache. `0` disables it. | +| `kv_unified:true` | `true` | Single unified KV buffer (**prerequisite** for idle-slot saving). | +| `cache_idle_slots:true` | `true` | Persists the idle slot's KV into the prompt cache on task switch. | + +All three are on by default since LocalAI v4.3, so the prompt cache works out of the box for the common single-slot setup. If you're on an older release, or you've explicitly disabled one of them, add the following to recover the behaviour: + +```yaml +options: + - cache_ram:4096 # or -1 for no limit + - kv_unified:true + - cache_idle_slots:true +``` + +Set `cache_ram:0` to opt out of the prompt cache entirely (saves host RAM at the cost of re-prefilling repeated prompts). + #### Reference - [llama](https://github.com/ggerganov/llama.cpp)