mirror of
https://github.com/mudler/LocalAI.git
synced 2026-10-05 20:42:10 -04:00
fix(llama-cpp): let parallel:1 in the model options win over LLAMACPP_PARALLEL (#12426)
The environment fallback was applied whenever n_parallel was still 1 after option parsing. An explicit `parallel: 1` in the model YAML is indistinguishable from the default that way, so it was replaced by LLAMACPP_PARALLEL. The docs say options in the YAML take precedence over environment variables; a single model could not be forced to one slot while the global variable was set. Track whether the options set the slot count and resolve it in a small helper (parallel_params.h): option first, then LLAMACPP_PARALLEL, then 1. The helper gets a standalone unit test picked up by `make test-backend-cpp`. Assisted-by: Claude:claude-opus-5-5 Signed-off-by: Stefan Walcz <stefan.walcz@walcz.de>
This commit is contained in:
1 parent
dd19ee8912
commit
6aa7b9b871
6 files changed
+77
-18
No files matched your search
@@ -0,0 +1,29 @@
|
||||
#include "parallel_params.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
int main() {
|
||||
// parallel:1 in the model options must not be replaced by the environment.
|
||||
if (llama_grpc::resolve_n_parallel(1, "4") != 1) {
|
||||
std::fprintf(stderr, "explicit parallel:1 was overwritten by LLAMACPP_PARALLEL\n");
|
||||
return 1;
|
||||
}
|
||||
if (llama_grpc::resolve_n_parallel(8, "4") != 8) {
|
||||
std::fprintf(stderr, "explicit parallel was overwritten by LLAMACPP_PARALLEL\n");
|
||||
return 1;
|
||||
}
|
||||
// Without an option the environment applies, otherwise the default.
|
||||
if (llama_grpc::resolve_n_parallel(std::nullopt, "4") != 4) {
|
||||
std::fprintf(stderr, "LLAMACPP_PARALLEL was not used as fallback\n");
|
||||
return 1;
|
||||
}
|
||||
if (llama_grpc::resolve_n_parallel(std::nullopt, nullptr) != 1) {
|
||||
std::fprintf(stderr, "default slot count is not 1\n");
|
||||
return 1;
|
||||
}
|
||||
if (llama_grpc::resolve_n_parallel(std::nullopt, "many") != 1) {
|
||||
std::fprintf(stderr, "unparsable LLAMACPP_PARALLEL was not ignored\n");
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
Reference in new issue
Block a user