mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-07-31 10:28:43 -04:00
Code Issues Packages Projects Releases Wiki Activity

fix(llama-cpp): preserve GPU layers during option passthrough (#11193)

Browse Source 
Stage the negative GPU-layer sentinels expected by the upstream argument parser, then restore LocalAI resolved values unless a passthrough flag explicitly overrides them. This avoids the parser assertion that terminated the backend for any generic option.

Assisted-by: Codex:gpt-5

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>
This commit is contained in:
localai-org-maint-bot
2026-07-30 15:54:05 +02:00
committed by GitHub
parent 79e88581c8
commit 5e541894df
5 changed files with 143 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
backend/cpp/llama-cpp/grpc-server.cpp
View File

@@ -54,6 +54,7 @@
#include "chat-auto-parser.h"
#include "llama_compat.h" // fork-skew switches, generated by prepare.sh
#include "message_content.h"
#include "passthrough_options.h"
#include <getopt.h>
#include <grpcpp/ext/proto_server_reflection_plugin.h>
#include <grpcpp/grpcpp.h>
@@ -579,6 +580,8 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
// Raw upstream llama-server flags collected from any option entry that
// starts with '-'. Applied once after the loop via common_params_parse.
std::vector<std::string> extra_argv;
bool passthrough_main_gpu_layers = false;
bool passthrough_draft_gpu_layers = false;
// O_DIRECT intent from the `direct_io` option. Upstream folded
// use_mmap/use_mlock/use_direct_io into a single common_params::load_mode
@@ -1186,6 +1189,17 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
flag.c_str());
} else {
extra_argv.push_back(flag);
passthrough_main_gpu_layers =
passthrough_main_gpu_layers ||
flag == "-ngl" ||
flag == "--gpu-layers" ||
flag == "--n-gpu-layers";
passthrough_draft_gpu_layers =
passthrough_draft_gpu_layers ||
flag == "--spec-draft-ngl" ||
flag == "-ngld" ||
flag == "--gpu-layers-draft" ||
flag == "--n-gpu-layers-draft";
// Preserve the whole value after the first ':' so embedded
// colons (e.g. host:port) survive strtok's truncation of optval.
auto colon = opt.find(':');
@@ -1349,6 +1363,14 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
// (n_parallel -> -1, use_color). Snapshot n_parallel so an unrelated
// passthrough flag can't silently clobber LocalAI's resolved value.
const int saved_n_parallel = params.n_parallel;
// Newer upstream parsers assert that these fields still contain their
// negative initialization sentinels. LocalAI resolves them from the
// model request before applying passthrough options, so stage the
// sentinels and restore the values unless a raw flag overrides them.
const auto saved_gpu_layers =
llama_grpc::prepare_passthrough_gpu_layers(
params.n_gpu_layers,
params.speculative.draft.n_gpu_layers);
std::vector<char *> argv;
std::string prog = "llama-server";
@@ -1372,6 +1394,12 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
if (params.n_parallel == -1) {
params.n_parallel = saved_n_parallel;
}
llama_grpc::restore_passthrough_gpu_layers(
params.n_gpu_layers,
params.speculative.draft.n_gpu_layers,
saved_gpu_layers,
passthrough_main_gpu_layers,
passthrough_draft_gpu_layers);
}
#ifndef LOCALAI_LLAMA_CPP_NO_SCORE_TASK