From a3fdfbc0d1afc8064d82776fd2b3093413b512e1 Mon Sep 17 00:00:00 2001 From: Roman Mazurenko <92804957+rvmz@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:09:05 +0300 Subject: [PATCH] feat(llama-cpp): add device selection option (#10724) Allow llama.cpp model configs to select the backend devices used for offload, matching upstream --device behavior so users can exclude a display or debug GPU. Signed-off-by: rvmzes Co-authored-by: rvmzes --- backend/cpp/llama-cpp/grpc-server.cpp | 22 ++++++++++++++++++++ docs/content/advanced/model-configuration.md | 2 ++ docs/content/features/text-generation.md | 2 ++ 3 files changed, 26 insertions(+) diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 660c4367e..cd4fb341d 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -613,6 +613,24 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt // starts with '-'. Applied once after the loop via common_params_parse. std::vector extra_argv; + auto add_device_options = [&](const std::string & devices) { + const std::regex regex{ R"([,]+)" }; + std::sregex_token_iterator it{ devices.begin(), devices.end(), regex, -1 }; + std::vector split_arg{ it, {} }; + + for (std::string device : split_arg) { + const auto start = device.find_first_not_of(" \t\n\r"); + if (start == std::string::npos) { + continue; + } + const auto end = device.find_last_not_of(" \t\n\r"); + device = device.substr(start, end - start + 1); + + extra_argv.push_back("--device"); + extra_argv.push_back(device); + } + }; + // decode options. Options are in form optname:optvale, or if booleans only optname. for (int i = 0; i < request->options_size(); i++) { std::string opt = request->options(i); @@ -744,6 +762,10 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt } else if (optval_str == "false" || optval_str == "0" || optval_str == "no" || optval_str == "off" || optval_str == "disabled") { params.no_op_offload = false; } + } else if (!strcmp(optname, "device") || !strcmp(optname, "devices")) { + if (optval != NULL) { + add_device_options(optval_str); + } } else if (!strcmp(optname, "split_mode") || !strcmp(optname, "sm")) { // Accepts: none | layer | row | tensor (the latter requires a llama.cpp build // that includes ggml-org/llama.cpp#19378, FlashAttention enabled, and KV-cache diff --git a/docs/content/advanced/model-configuration.md b/docs/content/advanced/model-configuration.md index 8092c162a..f71cf3d0d 100644 --- a/docs/content/advanced/model-configuration.md +++ b/docs/content/advanced/model-configuration.md @@ -493,6 +493,7 @@ These llama.cpp options are passed through the `options:` array. | `threads_batch` / `n_threads_batch` | int | same as `threads` | Threads used during prompt processing. `<= 0` means `hardware_concurrency()`. | | `direct_io` / `use_direct_io` | bool | `false` | Open the model with `O_DIRECT` (faster cold loads on NVMe; ignored if not supported). | | `verbosity` | int | `3` | llama.cpp internal log verbosity threshold. Higher = more verbose. | +| `device` / `devices` | string | all devices | Select the llama.cpp backend devices to use. Repeat the option or pass a comma-separated list; unlisted devices are excluded. Use the names reported by `llama-server --list-devices` / `--list-devices`. | | `override_tensor` / `tensor_buft_overrides` | string | "" | Per-tensor buffer-type overrides for the main model. Format: `=,=,...`. Mirrors the existing `draft_override_tensor` syntax for the draft model. | | `cpu_moe` | bool | false | Keep all MoE expert weights of the main model on CPU (upstream `--cpu-moe`). Frees VRAM on large MoE models (DeepSeek, Qwen3 `*-A3B`). | | `n_cpu_moe` | int | 0 | Keep MoE expert weights of the first N main-model layers on CPU (upstream `--n-cpu-moe`). | @@ -514,6 +515,7 @@ options: - "--cpu-moe" # boolean flag - "--n-cpu-moe:4" # flag with a value - "--override-tensor:exps=CPU" + - "devices:CUDA1,CUDA2,CUDA3" # skip CUDA0, e.g. a display GPU ``` Notes: diff --git a/docs/content/features/text-generation.md b/docs/content/features/text-generation.md index cadc67808..3d45ec6b9 100644 --- a/docs/content/features/text-generation.md +++ b/docs/content/features/text-generation.md @@ -512,6 +512,7 @@ The `llama.cpp` backend supports additional configuration options that can be sp | `check_tensors` | boolean | Validate tensor data for invalid values during model loading. Default: `false`. | `check_tensors:true` | | `warmup` | boolean | Enable warmup run after model loading. Default: `true`. | `warmup:false` | | `no_op_offload` | boolean | Disable offloading host tensor operations to device. Default: `false`. | `no_op_offload:true` | +| `device` or `devices` | string | Select the llama.cpp backend devices to use. Repeat the option or pass a comma-separated list; unlisted devices are excluded. Use the names reported by `llama-server --list-devices` / `--list-devices`. | `devices:CUDA1,CUDA2,CUDA3` | | `kv_unified` or `unified_kv` | boolean | Use a single unified KV buffer shared across all sequences. Default: `true` (LocalAI override; upstream defaults to `false` but auto-enables it when slot count is auto). **Required for `cache_idle_slots` to work**: without it the server force-disables idle-slot saving at init, and the prompt cache is never written across requests. | `kv_unified:false` | | `cache_idle_slots` or `idle_slots_cache` | boolean | On a new task, save the previous slot's KV state into the prompt cache (and clear the slot) so a later request with the same prefix can warm-load it. Default: `true`. Auto-disabled by the server if `kv_unified=false` or `cache_ram=0`. | `cache_idle_slots:false` | | `n_ctx_checkpoints` or `ctx_checkpoints` | integer | Maximum number of context checkpoints per slot (used for partial-prefix recovery, e.g. SWA). Default: `32`. | `ctx_checkpoints:16` | @@ -530,6 +531,7 @@ options: - context_shift:true - cache_ram:4096 - parallel:2 + - devices:CUDA1,CUDA2,CUDA3 - fit_params:true - fit_target:1024 - slot_prompt_similarity:0.5