mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-08 15:28:23 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(llama-cpp): add device selection option (#10724)

Browse Source 
Allow llama.cpp model configs to select the backend devices used for offload, matching upstream --device behavior so users can exclude a display or debug GPU.

Signed-off-by: rvmzes <rvmzes@rvmzess-MacBook-Pro.local>
Co-authored-by: rvmzes <rvmzes@rvmzess-MacBook-Pro.local>
This commit is contained in:
Roman Mazurenko
2026-07-07 23:09:05 +03:00
committed by GitHub
parent 2f33cc7bc4
commit a3fdfbc0d1
3 changed files with 26 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
backend/cpp/llama-cpp/grpc-server.cpp
View File

@@ -613,6 +613,24 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
// starts with '-'. Applied once after the loop via common_params_parse.
std::vector<std::string> extra_argv;
auto add_device_options = [&](const std::string & devices) {
const std::regex regex{ R"([,]+)" };
std::sregex_token_iterator it{ devices.begin(), devices.end(), regex, -1 };
std::vector<std::string> split_arg{ it, {} };
for (std::string device : split_arg) {
const auto start = device.find_first_not_of(" \t\n\r");
if (start == std::string::npos) {
continue;
}
const auto end = device.find_last_not_of(" \t\n\r");
device = device.substr(start, end - start + 1);
extra_argv.push_back("--device");
extra_argv.push_back(device);
}
};
// decode options. Options are in form optname:optvale, or if booleans only optname.
for (int i = 0; i < request->options_size(); i++) {
std::string opt = request->options(i);
@@ -744,6 +762,10 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
} else if (optval_str == "false" || optval_str == "0" || optval_str == "no" || optval_str == "off" || optval_str == "disabled") {
params.no_op_offload = false;
}
} else if (!strcmp(optname, "device") || !strcmp(optname, "devices")) {
if (optval != NULL) {
add_device_options(optval_str);
}
} else if (!strcmp(optname, "split_mode") || !strcmp(optname, "sm")) {
// Accepts: none | layer | row | tensor (the latter requires a llama.cpp build
// that includes ggml-org/llama.cpp#19378, FlashAttention enabled, and KV-cache