diff --git a/backend/cpp/llama-cpp/CMakeLists.txt b/backend/cpp/llama-cpp/CMakeLists.txt index 47852d400..f861713ee 100644 --- a/backend/cpp/llama-cpp/CMakeLists.txt +++ b/backend/cpp/llama-cpp/CMakeLists.txt @@ -110,4 +110,9 @@ if(LLAMA_GRPC_BUILD_TESTS) target_link_libraries(parent_watch_test PRIVATE Threads::Threads) target_compile_features(parent_watch_test PRIVATE cxx_std_17) add_test(NAME parent_watch_test COMMAND parent_watch_test) + + add_executable(passthrough_options_test passthrough_options_test.cpp passthrough_options.h) + target_include_directories(passthrough_options_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) + target_compile_features(passthrough_options_test PRIVATE cxx_std_17) + add_test(NAME passthrough_options_test COMMAND passthrough_options_test) endif() diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 904327209..adb1bf063 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -54,6 +54,7 @@ #include "chat-auto-parser.h" #include "llama_compat.h" // fork-skew switches, generated by prepare.sh #include "message_content.h" +#include "passthrough_options.h" #include #include #include @@ -579,6 +580,8 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt // Raw upstream llama-server flags collected from any option entry that // starts with '-'. Applied once after the loop via common_params_parse. std::vector extra_argv; + bool passthrough_main_gpu_layers = false; + bool passthrough_draft_gpu_layers = false; // O_DIRECT intent from the `direct_io` option. Upstream folded // use_mmap/use_mlock/use_direct_io into a single common_params::load_mode @@ -1186,6 +1189,17 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt flag.c_str()); } else { extra_argv.push_back(flag); + passthrough_main_gpu_layers = + passthrough_main_gpu_layers || + flag == "-ngl" || + flag == "--gpu-layers" || + flag == "--n-gpu-layers"; + passthrough_draft_gpu_layers = + passthrough_draft_gpu_layers || + flag == "--spec-draft-ngl" || + flag == "-ngld" || + flag == "--gpu-layers-draft" || + flag == "--n-gpu-layers-draft"; // Preserve the whole value after the first ':' so embedded // colons (e.g. host:port) survive strtok's truncation of optval. auto colon = opt.find(':'); @@ -1349,6 +1363,14 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt // (n_parallel -> -1, use_color). Snapshot n_parallel so an unrelated // passthrough flag can't silently clobber LocalAI's resolved value. const int saved_n_parallel = params.n_parallel; + // Newer upstream parsers assert that these fields still contain their + // negative initialization sentinels. LocalAI resolves them from the + // model request before applying passthrough options, so stage the + // sentinels and restore the values unless a raw flag overrides them. + const auto saved_gpu_layers = + llama_grpc::prepare_passthrough_gpu_layers( + params.n_gpu_layers, + params.speculative.draft.n_gpu_layers); std::vector argv; std::string prog = "llama-server"; @@ -1372,6 +1394,12 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt if (params.n_parallel == -1) { params.n_parallel = saved_n_parallel; } + llama_grpc::restore_passthrough_gpu_layers( + params.n_gpu_layers, + params.speculative.draft.n_gpu_layers, + saved_gpu_layers, + passthrough_main_gpu_layers, + passthrough_draft_gpu_layers); } #ifndef LOCALAI_LLAMA_CPP_NO_SCORE_TASK diff --git a/backend/cpp/llama-cpp/passthrough_options.h b/backend/cpp/llama-cpp/passthrough_options.h new file mode 100644 index 000000000..b2c897d8f --- /dev/null +++ b/backend/cpp/llama-cpp/passthrough_options.h @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: MIT + +#pragma once + +namespace llama_grpc { + +struct passthrough_gpu_layers_state { + int main; + int draft; +}; + +inline passthrough_gpu_layers_state prepare_passthrough_gpu_layers( + int & main_gpu_layers, + int & draft_gpu_layers) { + const passthrough_gpu_layers_state saved{ + main_gpu_layers, + draft_gpu_layers, + }; + main_gpu_layers = -1; + draft_gpu_layers = -1; + return saved; +} + +inline void restore_passthrough_gpu_layers( + int & main_gpu_layers, + int & draft_gpu_layers, + passthrough_gpu_layers_state saved, + bool main_overridden = false, + bool draft_overridden = false) { + if (!main_overridden && main_gpu_layers == -1) { + main_gpu_layers = saved.main; + } + if (!draft_overridden && draft_gpu_layers == -1) { + draft_gpu_layers = saved.draft; + } +} + +} // namespace llama_grpc diff --git a/backend/cpp/llama-cpp/passthrough_options_test.cpp b/backend/cpp/llama-cpp/passthrough_options_test.cpp new file mode 100644 index 000000000..c78825e5c --- /dev/null +++ b/backend/cpp/llama-cpp/passthrough_options_test.cpp @@ -0,0 +1,68 @@ +// SPDX-License-Identifier: MIT + +#include + +#include "passthrough_options.h" + +static int failures = 0; + +static void check(bool ok, const char * name) { + if (!ok) { + ++failures; + std::fprintf(stderr, "FAIL: %s\n", name); + } +} + +static void test_stages_resolved_gpu_layers_for_upstream_parser() { + int main_gpu_layers = 99; + int draft_gpu_layers = 12; + + const auto saved = llama_grpc::prepare_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers); + + check(main_gpu_layers == -1, "main GPU layers use parser sentinel"); + check(draft_gpu_layers == -1, "draft GPU layers use parser sentinel"); + + llama_grpc::restore_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers, saved); + + check(main_gpu_layers == 99, "main GPU layers restored"); + check(draft_gpu_layers == 12, "draft GPU layers restored"); +} + +static void test_keeps_explicit_passthrough_overrides() { + int main_gpu_layers = 99; + int draft_gpu_layers = 12; + + const auto saved = llama_grpc::prepare_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers); + + main_gpu_layers = 4; + draft_gpu_layers = 2; + llama_grpc::restore_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers, saved); + + check(main_gpu_layers == 4, "main passthrough override retained"); + check(draft_gpu_layers == 2, "draft passthrough override retained"); +} + +static void test_keeps_explicit_auto_passthrough_overrides() { + int main_gpu_layers = 99; + int draft_gpu_layers = 12; + + const auto saved = llama_grpc::prepare_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers); + + llama_grpc::restore_passthrough_gpu_layers( + main_gpu_layers, draft_gpu_layers, saved, true, true); + + check(main_gpu_layers == -1, "main auto passthrough override retained"); + check(draft_gpu_layers == -1, "draft auto passthrough override retained"); +} + +int main() { + test_stages_resolved_gpu_layers_for_upstream_parser(); + test_keeps_explicit_passthrough_overrides(); + test_keeps_explicit_auto_passthrough_overrides(); + return failures == 0 ? 0 : 1; +} diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index e99561179..b86924d2d 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -2,6 +2,7 @@ set -e + ## Patches ## Apply patches from the `patches` directory. Runs under set -e so a @@ -24,6 +25,9 @@ cp -r grpc-server.cpp llama.cpp/tools/grpc-server/ # unit test (compiled only when -DLLAMA_GRPC_BUILD_TESTS=ON). cp -r message_content.h llama.cpp/tools/grpc-server/ cp -r message_content_test.cpp llama.cpp/tools/grpc-server/ +# Generic passthrough parser staging and its standalone regression test. +cp -r passthrough_options.h llama.cpp/tools/grpc-server/ +cp -r passthrough_options_test.cpp llama.cpp/tools/grpc-server/ # Parent-death watcher (included by grpc-server.cpp) and its standalone unit # test (run via backend/cpp/run-unit-tests.sh; also buildable under ctest). cp -r parent_watch.h llama.cpp/tools/grpc-server/ @@ -58,4 +62,3 @@ else echo "add_subdirectory(grpc-server)" >> llama.cpp/tools/CMakeLists.txt fi set -e -