mirror/LocalAI
1
0
Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-05-04 05:57:10 -04:00
Code Issues Packages Projects Releases Wiki Activity

feat(llama.cpp): allow to set kv-overrides (#5745)

Browse Source 
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
This commit is contained in:
Ettore Di Giacinto
2025-06-28 21:26:07 +02:00
committed by GitHub
parent dbcf5fb4fc
commit dfadc3696e
4 changed files with 13 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
backend/cpp/llama/grpc-server.cpp
View File

@@ -12,6 +12,7 @@
#include "backend.pb.h"
#include "backend.grpc.pb.h"
#include "common.h"
#include <getopt.h>
#include <grpcpp/ext/proto_server_reflection_plugin.h>
#include <grpcpp/grpcpp.h>
@@ -260,6 +261,13 @@ static void params_parse(const backend::ModelOptions* request,
}
}
// Add kv_overrides
if (request->overrides_size() > 0) {
for (int i = 0; i < request->overrides_size(); i++) {
string_parse_kv_override(request->overrides(i).c_str(), params.kv_overrides);
}
}
// TODO: Add yarn
if (!request->tensorsplit().empty()) {