mirror/LocalAI
1
0 Fork 0
mirror of https://github.com/mudler/LocalAI.git synced 2026-09-25 23:54:56 -04:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into fix/launcher-autostart-11673

Browse Source
This commit is contained in:
Ettore Di Giacinto committed 2026-08-30 21:14:42 +00:00
2 parents 1045e553b9 dd4e75983d
commit 7e76b08d22
131 files changed
+6960 -704
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.github/backend-matrix.yml
+13
View File
@@ -3754,6 +3754,19 @@ include:
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'hipblas'
cuda-major-version: ""
cuda-minor-version: ""
platforms: 'linux/amd64'
tag-latest: 'auto'
tag-suffix: '-gpu-rocm-hipblas-stablediffusion-ggml'
runs-on: 'ubuntu-latest'
base-image: "rocm/dev-ubuntu-24.04:7.2.1"
skip-drivers: 'false'
backend: "stablediffusion-ggml"
dockerfile: "./backend/Dockerfile.golang"
context: "./"
ubuntu-version: '2404'
- build-type: 'sycl_f16'
cuda-major-version: ""
cuda-minor-version: ""
.github/dependabot.yml
+1 -65
View File
@@ -29,10 +29,6 @@ updates:
schedule:
# Check for updates to GitHub Actions every weekday
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/bark"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/common/template"
schedule:
@@ -55,30 +51,10 @@ updates:
ignore:
- dependency-name: "torch"
- dependency-name: "transformers"
- package-ecosystem: "pip"
directory: "/backend/python/exllama"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/exllama2"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/mamba"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/openvoice"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/rerankers"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/sentencetransformers"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/backend/python/transformers"
schedule:
@@ -86,44 +62,4 @@ updates:
- package-ecosystem: "pip"
directory: "/backend/python/vllm"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/examples/chainlit"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/examples/functions"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/examples/langchain/langchainpy-localai-example"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/examples/langchain-chroma"
schedule:
interval: "weekly"
- package-ecosystem: "pip"
directory: "/examples/streamlit-bot"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/examples/k8sgpt"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/examples/kubernetes"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/examples/langchain"
schedule:
interval: "weekly"
- package-ecosystem: "gomod"
directory: "/examples/semantic-todo"
schedule:
interval: "weekly"
- package-ecosystem: "docker"
directory: "/examples/telegram-bot"
schedule:
interval: "weekly"
interval: "weekly"
.github/workflows/notify-releases.yaml
+4 -3
View File
@@ -31,13 +31,14 @@ jobs:
messages: [
{
role: "system",
content: "Write a discord message with a bullet point summary of the release notes."
content: "Write a Discord message with a bullet point summary of the release notes. Keep the complete message under 1800 characters."
},
{
role: "user",
content: $input
}
]
],
max_tokens: 450
}')
# Send the request to LocalAI API
@@ -46,7 +47,7 @@ jobs:
-d "$json_payload")
# Extract the summary from the response
summary=$(echo $response | jq -r '.choices[0].message.content')
summary=$(printf '%s' "$response" | jq -er '.choices[0].message.content | strings | .[0:1800]')
# Print the summary
# -H "Authorization: Bearer $API_KEY" \
backend/cpp/audio-cpp/Makefile
+1 -1
View File
@@ -9,7 +9,7 @@
# recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
# rebuild and so the bump bot can see the pin.
AUDIO_CPP_VERSION?=4d383be1bff107e823ffc19120dcb6c78d493c0f
AUDIO_CPP_VERSION?=89a0e9803380880305e9e1b83c93614f9df2c893
AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp
CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
backend/cpp/bonsai/Makefile
+3 -1
View File
@@ -1,7 +1,7 @@
# Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp.
# Auto-bumped nightly by .github/workflows/bump_deps.yaml.
BONSAI_VERSION?=9ca265a57f85f2117942490f421f64a226dd9847
BONSAI_VERSION?=312bb2a93ea2bf798333fa859614fbf913ecb9e2
LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp
CMAKE_ARGS?=
@@ -41,6 +41,7 @@ define bonsai-build
# and are applied by apply-patches.sh below.
rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge
bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp
$(info $(GREEN)I bonsai build info:$(1)$(RESET))
@@ -79,6 +80,7 @@ bonsai-cpu-all:
# and are applied by apply-patches.sh below.
rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches
$(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge
bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp
$(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET))
backend/cpp/bonsai/patch-grpc-server.sh
+24
View File
@@ -0,0 +1,24 @@
#!/bin/bash
# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai.
set -euo pipefail
if [[ $# -ne 1 ]]; then
echo "usage: $0 <grpc-server.cpp>" >&2
exit 2
fi
SRC=$1
if [[ ! -f "$SRC" ]]; then
echo "grpc-server.cpp not found at $SRC" >&2
exit 2
fi
if grep -q 'common_json_error' "$SRC"; then
echo "==> patching $SRC to use the Bonsai JSON exception type"
awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
mv "$SRC.tmp" "$SRC"
echo "==> Bonsai JSON exception patch OK"
else
echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping"
fi
backend/cpp/ds4/Makefile
+2 -2
View File
@@ -1,10 +1,10 @@
# ds4 backend Makefile.
#
# Upstream pin lives below as DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28
# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
# (.github/bump_deps.sh) can find and update it - matches the
# llama-cpp / ik-llama-cpp / turboquant convention.
DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28
DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577
DS4_REPO?=https://github.com/antirez/ds4
CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
backend/cpp/ik-llama-cpp/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9
IK_LLAMA_VERSION?=15dddc60b3fc937a9e2a210359ecce392ccdf446
LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp
CMAKE_ARGS?=
backend/cpp/llama-cpp/Makefile
+1 -1
View File
@@ -1,5 +1,5 @@
LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd
LLAMA_VERSION?=d7bd3bfcad3e29c7e49fd26f38c79ee3e9a3fd6b
LLAMA_REPO?=https://github.com/ggerganov/llama.cpp
CMAKE_ARGS?=
backend/cpp/llama-cpp/grpc-server.cpp
+47 -40
View File
@@ -88,6 +88,12 @@ using grpc::ServerBuilder;
using grpc::ServerContext;
using grpc::Status;
#if LOCALAI_HAS_MTMD_INIT_OPT
#define LOCALAI_MTMD_INIT_OPT_ARG(value) , value
#else
#define LOCALAI_MTMD_INIT_OPT_ARG(value)
#endif
// gRPC bearer token auth for distributed mode.
// Reads LOCALAI_GRPC_AUTH_TOKEN from the environment. When set, rejects
// requests without a matching "authorization: Bearer <token>" metadata header.
@@ -294,7 +300,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
} else {
SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str());
}
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what());
SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str());
}
@@ -324,7 +330,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str());
}
SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str());
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
// If parsing fails, treat as string
data["tool_choice"] = predict->toolchoice();
SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str());
@@ -353,7 +359,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
// Add to data - llama.cpp server expects it as an object (map)
data["logit_bias"] = logit_bias_json;
SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str());
} catch (const json::parse_error& e) {
} catch (const common_json_error& e) {
SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what());
}
}
@@ -398,7 +404,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const
});
}
data["stop"] = predict->stopprompts();
data["stop"] = json::array();
for (const auto & stop : predict->stopprompts()) {
data["stop"].push_back(stop);
}
// data["n_probs"] = predict->nprobs();
//TODO: images,
@@ -1116,14 +1125,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
try {
int n = std::stoi(optval_str);
if (n < 0) n = 0;
// Keep override-name storage alive for the lifetime of the params struct
// (mirrors upstream arg.cpp behavior with a function-local static).
#if LOCALAI_HAS_N_CPU_FFN_HELPER
llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides);
#else
static std::list<std::string> buft_overrides_draft;
for (int i = 0; i < n; ++i) {
buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i));
params.speculative.draft.tensor_buft_overrides.push_back(
{buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()});
}
#endif
} catch (...) {}
}
@@ -1141,14 +1152,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt
try {
int n = std::stoi(optval_str);
if (n < 0) n = 0;
// Keep override-name storage alive for the lifetime of the
// params struct (mirrors upstream arg.cpp's function-local static).
#if LOCALAI_HAS_N_CPU_FFN_HELPER
llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides);
#else
static std::list<std::string> buft_overrides_main;
for (int i = 0; i < n; ++i) {
buft_overrides_main.push_back(llm_ffn_exps_block_regex(i));
params.tensor_buft_overrides.push_back(
{buft_overrides_main.back().c_str(), ggml_backend_cpu_buffer_type()});
}
#endif
} catch (...) {}
}
@@ -1795,7 +1808,7 @@ public:
for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
}
messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
}
// Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -1988,7 +2001,7 @@ public:
if (!body_json.contains("chat_template_kwargs")) {
body_json["chat_template_kwargs"] = json::object();
}
for (auto& el : ctk.items()) {
for (auto el : ctk.items()) {
body_json["chat_template_kwargs"][el.key()] = el.value();
}
}
@@ -2074,30 +2087,27 @@ public:
// If not using chat templates, extract files from image_data/audio_data fields
// (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
const auto &images_data = data.find("image_data");
if (images_data != data.end() && images_data->is_array())
if (data.contains("image_data") && data.at("image_data").is_array())
{
for (const auto &img : *images_data)
for (const auto &img : data.at("image_data"))
{
auto decoded_data = base64_decode(img["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &audio_data = data.find("audio_data");
if (audio_data != data.end() && audio_data->is_array())
if (data.contains("audio_data") && data.at("audio_data").is_array())
{
for (const auto &audio : *audio_data)
for (const auto &audio : data.at("audio_data"))
{
auto decoded_data = base64_decode(audio["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &video_data = data.find("video_data");
if (video_data != data.end() && video_data->is_array())
if (data.contains("video_data") && data.at("video_data").is_array())
{
for (const auto &video : *video_data)
for (const auto &video : data.at("video_data"))
{
auto decoded_data = base64_decode(video["data"].get<std::string>());
files.push_back(decoded_data);
@@ -2111,10 +2121,10 @@ public:
std::vector<server_tokens> inputs;
if (has_mtmd) {
// multimodal
inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files));
inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)));
} else {
// Everything else, including multimodal completions.
inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true);
inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
}
tasks.reserve(inputs.size());
@@ -2370,7 +2380,7 @@ public:
for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j));
for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j));
}
messages_json.push_back(llama_grpc::build_reconstructed_message(rin));
messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump()));
}
// Final safety check: Ensure no message has null content (Jinja templates require strings)
@@ -2563,7 +2573,7 @@ public:
if (!body_json.contains("chat_template_kwargs")) {
body_json["chat_template_kwargs"] = json::object();
}
for (auto& el : ctk.items()) {
for (auto el : ctk.items()) {
body_json["chat_template_kwargs"][el.key()] = el.value();
}
}
@@ -2649,11 +2659,10 @@ public:
// If not using chat templates, extract files from image_data/audio_data fields
// (If using chat templates, files were already extracted by oaicompat_chat_params_parse)
if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) {
const auto &images_data = data.find("image_data");
if (images_data != data.end() && images_data->is_array())
if (data.contains("image_data") && data.at("image_data").is_array())
{
std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl;
for (const auto &img : *images_data)
std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl;
for (const auto &img : data.at("image_data"))
{
std::cout << "[PREDICT] Processing image" << std::endl;
auto decoded_data = base64_decode(img["data"].get<std::string>());
@@ -2661,20 +2670,18 @@ public:
}
}
const auto &audio_data = data.find("audio_data");
if (audio_data != data.end() && audio_data->is_array())
if (data.contains("audio_data") && data.at("audio_data").is_array())
{
for (const auto &audio : *audio_data)
for (const auto &audio : data.at("audio_data"))
{
auto decoded_data = base64_decode(audio["data"].get<std::string>());
files.push_back(decoded_data);
}
}
const auto &video_data = data.find("video_data");
if (video_data != data.end() && video_data->is_array())
if (data.contains("video_data") && data.at("video_data").is_array())
{
for (const auto &video : *video_data)
for (const auto &video : data.at("video_data"))
{
auto decoded_data = base64_decode(video["data"].get<std::string>());
files.push_back(decoded_data);
@@ -2689,10 +2696,10 @@ public:
std::vector<server_tokens> inputs;
if (has_mtmd) {
// multimodal
inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files));
inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)));
} else {
// Everything else, including multimodal completions.
inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true);
inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
}
tasks.reserve(inputs.size());
@@ -2879,7 +2886,7 @@ public:
json prompt = body.at("embeddings");
auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true);
auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
for (const auto & tokens : tokenized_prompts) {
// this check is necessary for models that do not add BOS token to the input
if (tokens.empty()) {
@@ -2984,7 +2991,7 @@ public:
tasks.reserve(documents.size());
for (size_t i = 0; i < documents.size(); i++) {
auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i]);
auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
server_task task = server_task(SERVER_TASK_TYPE_RERANK);
task.id = rd.queue_tasks.get_new_id();
task.index = i;
@@ -3005,7 +3012,7 @@ public:
}
// Collect responses
json responses = json::array();
std::vector<json> responses;
for (auto & res : all_results.results) {
GGML_ASSERT(dynamic_cast<server_task_result_rerank*>(res.get()) != nullptr);
responses.push_back(res->to_json());
@@ -3018,7 +3025,7 @@ public:
// Crop results by request.top_n if specified
int top_n = request->top_n();
if (top_n > 0 && top_n < static_cast<int>(responses.size())) {
responses = json(responses.begin(), responses.begin() + top_n);
responses.resize(top_n);
}
// Set usage information
backend::Usage* usage = rerankResult->mutable_usage();
@@ -3065,7 +3072,7 @@ public:
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, opts.error);
}
auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false);
auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt));
if (!wrapper.bitmap) {
return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT,
"failed to read speaker reference audio: " + opts.voice_path);
backend/cpp/llama-cpp/message_content.h
+4 -3
View File
@@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role,
// (#7528). A multimodal user message legitimately carries a typed-part array
// ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the
// streaming and non-streaming paths so this invariant cannot drift between them.
inline void normalize_template_message(nlohmann::ordered_json& msg) {
template <typename Json>
inline void normalize_template_message(Json& msg) {
if (!msg.contains("content")) {
msg["content"] = ""; // templates expect the field to exist
return;
}
nlohmann::ordered_json& content = msg["content"];
auto& content = msg["content"];
const std::string role = (msg.contains("role") && msg["role"].is_string())
? msg["role"].get<std::string>()
? msg["role"].template get<std::string>()
: std::string();
if (content.is_null()) {
content = ""; // #7324: null would crash content[:N] slicing
backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch
+1 -11
View File
@@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch
---
common/common.cpp | 6 +-
common/common.h | 3 +
tools/CMakeLists.txt | 1 +
tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++-
tools/server/server-task.h | 47 +++++
5 files changed, 406 insertions(+), 9 deletions(-)
4 files changed, 405 insertions(+), 9 deletions(-)
diff --git a/common/common.cpp b/common/common.cpp
index 2e3f14c..0cec0dc 100644
@@ -42,15 +41,6 @@ index 878534d..4001df2 100644
int32_t n_sequences = 1; // number of sequences to decode
int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch)
int32_t n_outputs_max_per_seq = 1; // max outputs per sequence
diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index 780df32..1d2fe8f 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -41,3 +41,4 @@ else()
add_subdirectory(fit-params)
add_subdirectory(results)
endif()
+add_subdirectory(grpc-server)
diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp
index 3b5f6a1..d0e18e6 100644
--- a/tools/server/server-context.cpp
backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch
+1 -1
View File
@@ -659,7 +659,7 @@ index 9069463fe..b7fa1e534 100644
+ }
+
+ if (speaker_ref_len > 0) {
+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false);
+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt);
+ if (!wrapper.bitmap) {
+ res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST));
+ return res;
backend/cpp/llama-cpp/prepare.sh
+12
View File
@@ -61,11 +61,23 @@ if grep -q "server_metrics metrics;" llama.cpp/tools/server/server-task.h; then
else
HAS_SERVER_METRICS=0
fi
if grep -q "mtmd_helper_init_opt" llama.cpp/tools/mtmd/mtmd-helper.h; then
HAS_MTMD_INIT_OPT=1
else
HAS_MTMD_INIT_OPT=0
fi
if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then
HAS_N_CPU_FFN_HELPER=1
else
HAS_N_CPU_FFN_HELPER=0
fi
cat > llama.cpp/tools/grpc-server/llama_compat.h <<EOF
// Generated by backend/cpp/llama-cpp/prepare.sh. Do not edit.
#pragma once
#define LOCALAI_LEGACY_LOAD_MODE ${LEGACY_LOAD_MODE}
#define LOCALAI_HAS_SERVER_METRICS ${HAS_SERVER_METRICS}
#define LOCALAI_HAS_MTMD_INIT_OPT ${HAS_MTMD_INIT_OPT}
#define LOCALAI_HAS_N_CPU_FFN_HELPER ${HAS_N_CPU_FFN_HELPER}
EOF
set +e
backend/cpp/turboquant/patch-grpc-server.sh
+14
View File
@@ -8,6 +8,8 @@
# so the grpc-server option parser skips the two references to
# common_params::checkpoint_min_step (the default and the option handler).
# That field does not exist in the fork yet; drop this once it does.
# 3. Use nlohmann's parse_error type in JSON catch clauses because the fork
# predates upstream's common_json_error wrapper.
#
# The fork used to lag upstream on the whole common_params_speculative refactor
# (ggml-org/llama.cpp#22397/#22838/#22964), the model_tgt rename (#22838) and
@@ -100,4 +102,16 @@ else
echo "==> LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK"
fi
# 3. The shared source follows current upstream and catches common_json_error.
# TurboQuant still exposes nlohmann::json directly, so its equivalent parse
# failures use json::parse_error instead.
if grep -q 'common_json_error' "$SRC"; then
echo "==> patching $SRC to use the TurboQuant JSON exception type"
awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp"
mv "$SRC.tmp" "$SRC"
echo "==> TurboQuant JSON exception patch OK"
else
echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping"
fi
echo "==> all patches applied"
backend/go/crispasr/Makefile
+1 -1
View File
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
# CrispASR version (release tag)
CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR
CRISPASR_VERSION?=74bb374a8cc74284348d76a0a6e944180fbe6b07
CRISPASR_VERSION?=ae4474dd8306384a0e697183d863dfc52e69a2fb
SO_TARGET?=libgocrispasr.so
CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
backend/go/depth-anything-cpp/Makefile
+1 -1
View File
@@ -14,7 +14,7 @@ JOBS?=$(shell nproc --ignore=1)
# It is kept alive by the upstream tag da2-support (survives a squash-merge);
# repoint to the master merge commit once mudler/depth-anything.cpp PR #1 lands.
DEPTHANYTHING_REPO?=https://github.com/mudler/depth-anything.cpp.git
DEPTHANYTHING_VERSION?=54abd5c0abfd1f394e01cb3c38f2e3af4daedf85
DEPTHANYTHING_VERSION?=739992d10bf9472c46dcd4622b14d2b20766c58d
ifeq ($(NATIVE),false)
CMAKE_ARGS+=-DGGML_NATIVE=OFF
backend/go/stablediffusion-ggml/Makefile
+6 -3
View File
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
# stablediffusion.cpp (ggml)
STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
STABLEDIFFUSION_GGML_VERSION?=97d2990807fe6d558e395f8764198d7c7e7b411c
STABLEDIFFUSION_GGML_VERSION?=be0e34480dada95f8ce9a021bbb95c5de85d67c7
CMAKE_ARGS+=-DGGML_MAX_NAME=128
@@ -38,8 +38,11 @@ else ifeq ($(BUILD_TYPE),hipblas)
ROCM_PATH ?= /opt/rocm
export CXX=$(ROCM_HOME)/llvm/bin/clang++
export CC=$(ROCM_HOME)/llvm/bin/clang
AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201
CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201
# SD_HIPBLAS turns on ggml's HIP backend itself; GGML_HIPBLAS is the name ggml
# used before it was renamed to GGML_HIP, so passing it here only produced an
# unused-variable warning.
CMAKE_ARGS+=-DSD_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS)
else ifeq ($(BUILD_TYPE),vulkan)
CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON
else ifeq ($(BUILD_TYPE),metal)
backend/go/vllm-cpp/Makefile
+1 -1
View File
@@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e
# vllm.cpp version
VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp
VLLM_CPP_VERSION?=438305e1577768ec0f75729456a4c8b9f425e2ee
VLLM_CPP_VERSION?=150b37852c123f7855fb219b37347572ca9427e7
# MLX GEMM provider (darwin/metal only; see the metal branch below for why).
# Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun
backend/go/vllm-cpp/govllmcpp.go
+3 -2
View File
@@ -1,6 +1,6 @@
package main
// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v21).
// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23).
//
// The structs below are hand-mirrored PODs of the C declarations, with
// explicit padding so the Go layout matches the C layout on linux/darwin
@@ -21,7 +21,7 @@ import (
// the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks
// the two against each other, because a mismatch is only caught at runtime by
// registerLib, where it takes the backend down on every load (issue #11379).
const abiVersion = 21
const abiVersion = 23
// The ABI's tri-state toggles (enable_prefix_caching ABI v7,
// enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is
@@ -83,6 +83,7 @@ type cModelParams struct {
LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19)
_ [4]byte
LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19)
MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22)
}
// cSamplingParams mirrors vllm_sampling_params (structured fields included).
backend/go/vllm-cpp/vllmcpp_test.go
+4 -3
View File
@@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) {
RunSpecs(t, "vllm-cpp suite")
}
// The Go POD mirrors must match the C struct layout of vllm.h (ABI v21)
// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23)
// byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin
// amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h.
var _ = Describe("C ABI struct mirrors", func() {
@@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() {
// VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile).
// Moving the pin past this without growing the mirrors below ships a
// backend that refuses every load at startup (issue #11379).
Expect(abiVersion).To(Equal(21))
Expect(abiVersion).To(Equal(23))
})
It("cModelParams matches vllm_model_params", func() {
@@ -51,7 +51,8 @@ var _ = Describe("C ABI struct mirrors", func() {
Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104)))
Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112)))
Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120)))
Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128)))
Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128)))
Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136)))
})
It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() {
backend/go/whisper/Makefile
+1 -1
View File
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)
# whisper.cpp version
WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp
WHISPER_CPP_VERSION?=233fe1fc9b48a09e361d3594520838ca266537fe
WHISPER_CPP_VERSION?=978113305b2ead22249b881deafa131dc8884911
SO_TARGET?=libgowhisper.so
CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF
backend/index.yaml
+12 -2
View File
@@ -510,7 +510,7 @@
default: "cpu-stablediffusion-ggml"
nvidia: "cuda12-stablediffusion-ggml"
intel: "intel-sycl-f16-stablediffusion-ggml"
# amd: "rocm-stablediffusion-ggml"
amd: "rocm-stablediffusion-ggml"
vulkan: "vulkan-stablediffusion-ggml"
nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml"
metal: "metal-stablediffusion-ggml"
@@ -2109,7 +2109,7 @@
default: "cpu-stablediffusion-ggml-development"
nvidia: "cuda12-stablediffusion-ggml-development"
intel: "intel-sycl-f16-stablediffusion-ggml-development"
# amd: "rocm-stablediffusion-ggml-development"
amd: "rocm-stablediffusion-ggml-development"
vulkan: "vulkan-stablediffusion-ggml-development"
nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml-development"
metal: "metal-stablediffusion-ggml-development"
@@ -3904,6 +3904,11 @@
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml"
mirrors:
- localai/localai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml
- !!merge <<: *stablediffusionggml
name: "rocm-stablediffusion-ggml"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml"
mirrors:
- localai/localai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml
- !!merge <<: *stablediffusionggml
name: "intel-sycl-f32-stablediffusion-ggml"
uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-stablediffusion-ggml"
@@ -3917,6 +3922,11 @@
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml"
mirrors:
- localai/localai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml
- !!merge <<: *stablediffusionggml
name: "rocm-stablediffusion-ggml-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml"
mirrors:
- localai/localai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml
- !!merge <<: *stablediffusionggml
name: "intel-sycl-f32-stablediffusion-ggml-development"
uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-stablediffusion-ggml"
backend/python/sglang/backend.py
+4 -3
View File
@@ -323,7 +323,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
if not hasattr(request, proto_field):
continue
value = getattr(request, proto_field)
if value in (None, 0, 0.0, [], False, ""):
if proto_field != "Temperature" and value in (None, 0, 0.0, [], False, ""):
continue
# repeated fields come back as RepeatedScalarContainer — convert
if hasattr(value, "__iter__") and not isinstance(value, (str, bytes)):
@@ -363,8 +363,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
template_kwargs["tools"] = json.loads(request.Tools)
except json.JSONDecodeError:
pass
if request.Metadata.get("enable_thinking", "").lower() == "true":
template_kwargs["enable_thinking"] = True
_thinking = request.Metadata.get("enable_thinking", "").lower()
if _thinking in ("true", "false"):
template_kwargs["enable_thinking"] = (_thinking == "true")
try:
return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
backend/python/sglang/test.py
+61
View File
@@ -96,6 +96,67 @@ class TestSglangHelpers(unittest.TestCase):
servicer._apply_engine_args({}, "[1,2,3]")
self.assertIn("must be a JSON object", str(ctx.exception))
def test_build_prompt_forwards_enable_thinking(self):
from types import SimpleNamespace
class Tok:
def __init__(self):
self.kwargs = None
def apply_chat_template(self, messages, **kwargs):
self.kwargs = kwargs
return "PROMPT"
def kwargs_for(metadata):
servicer = self._servicer()
tok = Tok()
servicer.tokenizer = tok
msg = SimpleNamespace(
role="user", content="hi", name="",
tool_call_id="", reasoning_content="", tool_calls="",
)
req = SimpleNamespace(
Prompt="", UseTokenizerTemplate=True,
Messages=[msg], Tools="", Metadata=metadata,
)
self.assertEqual(servicer._build_prompt(req), "PROMPT")
return tok.kwargs
self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True)
# "false" used to be dropped, so Qwen3 kept thinking on
self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False)
self.assertNotIn("enable_thinking", kwargs_for({}))
self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False)
def test_explicit_zero_temperature_is_preserved(self):
"""Temperature=0 is valid greedy decoding, not an unset value."""
from types import SimpleNamespace
servicer = self._servicer()
request = SimpleNamespace(
Temperature=0,
N=0,
PresencePenalty=0,
FrequencyPenalty=0,
RepetitionPenalty=0,
TopP=0,
TopK=0,
MinP=0,
Seed=0,
StopPrompts=[],
StopTokenIds=[],
IgnoreEOS=False,
Tokens=0,
MinTokens=0,
SkipSpecialTokens=False,
Grammar="",
)
params = servicer._build_sampling_params(request)
self.assertEqual(params["temperature"], 0)
# Other protobuf-default scalar fields must remain filtered.
self.assertNotIn("top_p", params)
if __name__ == "__main__":
unittest.main()
backend/python/vllm/backend.py
+11 -7
View File
@@ -523,9 +523,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
context.set_details(str(e))
return backend_pb2.ScoreResponse()
async def _predict(self, request, context, streaming=False):
# Build the sampling parameters
# NOTE: this must stay in sync with the vllm backend
def _build_sampling_params(self, request):
request_to_sampling_params = {
"N": "n",
"PresencePenalty": "presence_penalty",
@@ -555,9 +553,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
for request_field, param_field in request_to_sampling_params.items():
if hasattr(request, request_field):
value = getattr(request, request_field)
if value not in (None, 0, [], False):
if request_field == "Temperature" or value not in (None, 0, [], False):
setattr(sampling_params, param_field, value)
return sampling_params
async def _predict(self, request, context, streaming=False):
# Build the sampling parameters
sampling_params = self._build_sampling_params(request)
# Structured-output decoding: use Grammar field to pass JSON schema or BNF
if HAS_GUIDED_DECODING and request.Grammar:
try:
@@ -587,9 +591,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer):
except json.JSONDecodeError:
pass
# Enable thinking mode if requested
if request.Metadata.get("enable_thinking", "").lower() == "true":
template_kwargs["enable_thinking"] = True
_thinking = request.Metadata.get("enable_thinking", "").lower()
if _thinking in ("true", "false"):
template_kwargs["enable_thinking"] = (_thinking == "true")
try:
prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs)
backend/python/vllm/requirements-cublas13-after.txt
+2 -2
View File
@@ -3,8 +3,8 @@
# on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
# instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
# so uv consults this index alongside PyPI.
--extra-index-url https://wheels.vllm.ai/0.27.1/cu130
--extra-index-url https://wheels.vllm.ai/0.28.0/cu130
# VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
# which pins this exact vLLM version. Bumping vllm here means coordinating with a
# vllm-metal release that supports the new version, or macOS/Metal builds break.
vllm==0.27.1
vllm==0.28.0
backend/python/vllm/requirements-l4t13-after.txt
+1 -1
View File
@@ -9,4 +9,4 @@
# memory architecture crash deterministically with an empty "Engine core init
# failed" set (mudler/LocalAI#10722). Leaving this unpinned let the L4T image
# drift onto whatever wheel was latest at build time.
vllm==0.26.0
vllm==0.28.0
backend/python/vllm/test.py
+13
View File
@@ -121,6 +121,19 @@ class TestBackendServicer(unittest.TestCase):
finally:
self.tearDown()
def test_explicit_zero_temperature_is_preserved(self):
"""Temperature=0 is valid greedy decoding, not an unset value."""
import sys, os
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from backend import BackendServicer
servicer = BackendServicer()
request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0)
sampling_params = servicer._build_sampling_params(request)
self.assertEqual(sampling_params.temperature, 0)
# Other protobuf-default scalar fields must remain filtered.
self.assertEqual(sampling_params.top_p, 0.9)
def test_messages_to_dicts(self):
"""
core/application/distributed.go
+9
View File
@@ -162,6 +162,15 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade
}
xlog.Info("Node registry initialized")
// Let scheduling rules be keyed by a model alias. The registry resolves a
// rule's name through the config loader to find the model it governs, so an
// operator can pin placement to a stable name like "production" and have it
// follow the alias when the alias is repointed. Wired before the seed below
// and before the reconciler starts, so the first tick already resolves.
if configLoader != nil {
registry.SetAliasResolver(configLoader)
}
// Seed declarative per-model scheduling config (LOCALAI_MODEL_SCHEDULING /
// LOCALAI_MODEL_SCHEDULING_CONFIG). Authoritative: overwrites matching models
// on every boot. Runs before the reconciler starts so the first tick already
core/application/startup.go
+19
View File
@@ -267,6 +267,10 @@ func New(opts ...config.AppOption) (*Application, error) {
}
// Initialize distributed mode services (NATS, object storage, node registry)
// revisionStore is built inside the distributed block below but used after
// the model configs are loaded, so it is declared out here.
var revisionStore modeladmin.RevisionStore
distSvc, err := initDistributed(options, application.authDB, application.ModelConfigLoader())
if err != nil {
return nil, fmt.Errorf("distributed mode initialization failed: %w", err)
@@ -373,6 +377,9 @@ func New(opts ...config.AppOption) (*Application, error) {
cfgLoaderOpts := options.ToConfigLoaderOptions()
modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup)
gs.SetModelRevisionLifecycle(modelRevisionLifecycle)
// Captured here, used after the model configs are loaded below: the
// resync reads the loader, which is still empty at this point.
revisionStore = modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle)
gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) {
// ApplyRemoteChange honors the op: a "delete" prunes the element
// (a reload-from-path is additive and cannot drop it), anything
@@ -419,6 +426,18 @@ func New(opts ...config.AppOption) (*Application, error) {
xlog.Error("error loading config files", "error", err)
}
// Bring the controller's stored revisions back in line with the
// configuration just loaded. An inference request may only establish a
// revision, never replace one, so a model whose stored value has drifted
// stays unroutable until something republishes it. This has to run after
// the load above: the loader is empty until then, and a resync against an
// empty loader silently reconciles nothing.
if revisionStore != nil {
if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), options, revisionStore); err != nil {
xlog.Warn("Failed to resync model config revisions", "error", err)
}
}
if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil {
xlog.Error("error registering external backends", "error", err)
}
core/backend/options.go
+9 -9
View File
@@ -202,18 +202,18 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo
model.WithContext(so.Context),
model.WithModelID(c.ModelID()),
}
// Prefer the revision stamped when the configuration was loaded. c has since
// been merged with this request's prediction parameters (temperature, top_p,
// stop, ...), and hashing it here would produce a different revision for
// every distinct request body — which the controller reads as a config
// change and rejects as stale. Recomputing is the fallback for a config that
// never passed through the loader.
// Use the revision stamped when the configuration was parsed, and only
// that. By this point c has been merged with the request's prediction
// parameters and had SetDefaults applied, so hashing it here would produce
// a revision that depends on the request body and on whether the model file
// parsed, which the controller reads as a config change and rejects. Every
// config the loader hands out is stamped; an unstamped one was synthesized
// elsewhere and is routed without a revision rather than with a wrong one.
if revision := c.PersistedConfigRevision(); revision != "" {
defOpts = append(defOpts, model.WithConfigRevision(revision))
} else if revision, err := config.ModelConfigRevision(&c); err == nil {
defOpts = append(defOpts, model.WithConfigRevision(revision))
} else {
xlog.Warn("Failed to compute model configuration revision", "model", c.ModelID(), "error", err)
xlog.Warn("Model configuration carries no revision stamp; routing without one",
"model", c.ModelID())
}
managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil
if managedPrimary {
core/config/inference_defaults.json
+3 -1
View File
@@ -1,6 +1,8 @@
{
"_comment": "Auto-generated from unsloth inference_defaults.json. DO NOT EDIT. Run go generate ./core/config/ to update.",
"families": {
"moss-tts-local-transformer-v1.5": {"min_p":0,"repeat_penalty":1,"temperature":1.7,"top_k":25,"top_p":0.8},
"moss-tts-nano": {"min_p":0,"repeat_penalty":1,"temperature":1.7,"top_k":25,"top_p":0.8},
"qwen3.8": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8},
"qwen3.6": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8},
"qwen3.5": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8},
@@ -60,5 +62,5 @@
"grok": {"min_p":0.01,"repeat_penalty":1,"temperature":1,"top_k":-1,"top_p":0.95},
"mimo": {"min_p":0.01,"repeat_penalty":1,"temperature":0.7,"top_k":-1,"top_p":0.95}
},
"patterns": ["qwen3.8","qwen3.6","qwen3.5","qwen3-coder","qwen3-next","qwen3-vl","qwen3","qwen2.5-coder","qwen2.5-vl","qwen2.5-omni","qwen2.5-math","qwen2.5","qwen2-vl","qwen2","qwq","gemma-4","gemma-3n","gemma-3","medgemma","gemma-2","muse-glimmer","llama-4","llama-3.3","llama-3.2","llama-3.1","llama-3","phi-4","phi-3","mistral-nemo","mistral-small","mistral-large","magistral","ministral","devstral","pixtral","deepseek-v4","deepseek-r1","deepseek-v3","deepseek-ocr","glm-5","glm-4","nemotron","minimax-m2.7","minimax-m2.5","minimax","gpt-oss","granite-4","kimi-k3","kimi-k2","kimi","lfm2","smollm","olmo","falcon","ernie","seed","grok","mimo"]
"patterns": ["moss-tts-local-transformer-v1.5","moss-tts-nano","qwen3.8","qwen3.6","qwen3.5","qwen3-coder","qwen3-next","qwen3-vl","qwen3","qwen2.5-coder","qwen2.5-vl","qwen2.5-omni","qwen2.5-math","qwen2.5","qwen2-vl","qwen2","qwq","gemma-4","gemma-3n","gemma-3","medgemma","gemma-2","muse-glimmer","llama-4","llama-3.3","llama-3.2","llama-3.1","llama-3","phi-4","phi-3","mistral-nemo","mistral-small","mistral-large","magistral","ministral","devstral","pixtral","deepseek-v4","deepseek-r1","deepseek-v3","deepseek-ocr","glm-5","glm-4","nemotron","minimax-m2.7","minimax-m2.5","minimax","gpt-oss","granite-4","kimi-k3","kimi-k2","kimi","lfm2","smollm","olmo","falcon","ernie","seed","grok","mimo"]
}
core/config/model_config.go
+1 -1
View File
@@ -1864,7 +1864,7 @@ func (c *ModelConfig) PersistedConfigRevision() string {
// persisted. It is computed from the receiver as-is, so callers must invoke it
// only on a configuration that has not been merged with request overrides.
func (c *ModelConfig) StampPersistedConfigRevision() error {
revision, err := ModelConfigRevision(c)
revision, err := modelConfigRevision(c)
if err != nil {
return err
}
core/config/model_config_loader.go
+76 -9
View File
@@ -168,6 +168,14 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model
if err := yaml.Unmarshal(f, &configs); err == nil && len(configs) > 0 {
for _, cc := range configs {
cc.modelConfigFile = file
// Stamp before SetDefaults: the revision describes what is on disk.
// SetDefaults folds in the GGUF guess, hardware defaults and
// app-level options, none of which are persisted configuration, and
// the GGUF guess in particular depends on whether the model file
// parses at that moment.
if err := cc.StampPersistedConfigRevision(); err != nil {
return nil, fmt.Errorf("stamping config revision for %q: %w", cc.Name, err)
}
cc.SetDefaults(opts...)
cc.syncKnownUsecasesFromString()
}
@@ -182,6 +190,9 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model
c.modelConfigFile = file
c.syncKnownUsecasesFromString()
if err := c.StampPersistedConfigRevision(); err != nil {
return nil, fmt.Errorf("stamping config revision for %q: %w", c.Name, err)
}
c.SetDefaults(opts...)
return []*ModelConfig{c}, nil
@@ -218,17 +229,18 @@ func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath str
}
}
cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
// Stamp the revision here, at the boundary between the persisted
// configuration and the request that is about to override parts of it.
// Everything downstream of this point (the request middleware) merges
// per-request prediction parameters into cfg, so a revision computed later
// would identify the request rather than the configuration.
if err := cfg.StampPersistedConfigRevision(); err != nil {
return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err)
// Stamp before SetDefaults, and only when this config did not come from
// disk already carrying one (a name with no config file on disk is
// synthesized above). Re-stamping a loaded config here would hash it after
// SetDefaults and reintroduce the dependency on the GGUF guess.
if cfg.PersistedConfigRevision() == "" {
if err := cfg.StampPersistedConfigRevision(); err != nil {
return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err)
}
}
cfg.SetDefaults(append(opts, ModelPath(modelPath))...)
return cfg, nil
}
@@ -429,6 +441,26 @@ func (bcl *ModelConfigLoader) ResolveAlias(cfg *ModelConfig) (*ModelConfig, bool
return &target, true, nil
}
// ResolveAliasName maps a model name to the name of the model that actually
// serves it: an alias resolves to its target, anything else resolves to
// itself. The second return reports whether name was an alias.
//
// Unlike ResolveAlias this never errors. A name with no config (a rule may be
// authored before the model is installed), a dangling alias, and a chained
// alias all resolve to themselves, so callers keep a usable name that simply
// has no model behind it rather than silently governing a different model.
func (bcl *ModelConfigLoader) ResolveAliasName(name string) (string, bool) {
cfg, exists := bcl.GetModelConfig(name)
if !exists || !cfg.IsAlias() {
return name, false
}
target, exists := bcl.GetModelConfig(cfg.Alias)
if !exists || target.IsAlias() {
return name, true
}
return target.Name, true
}
// ValidateAliasTarget checks an alias config's target at create/swap time:
// the target must exist, must not be an alias, and must not be disabled.
// Returns nil for non-alias configs.
@@ -953,3 +985,38 @@ func hasAnyMappingKey(mapping *yaml.Node, keys ...string) bool {
func nonemptyScalar(node *yaml.Node) bool {
return node != nil && node.Kind == yaml.ScalarNode && node.Tag == "!!str" && strings.TrimSpace(node.Value) != ""
}
// RevisionFor returns the config revision for modelName: the one an inference
// request for that model will carry.
//
// This is the only way to obtain a revision outside this package. Every
// publisher must use it, so that what is published and what is checked are
// the same value by construction rather than by two implementations happening
// to agree. Hashing a ModelConfig directly is not available to callers, because
// a config that has been through SetDefaults or the request middleware hashes
// to something no request will ever present.
func (bcl *ModelConfigLoader) RevisionFor(modelName string, appConfig *ApplicationConfig) (string, error) {
cfg, err := bcl.LoadModelConfigFileByNameDefaultOptions(modelName, appConfig)
if err != nil {
return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err)
}
return stampedRevision(cfg, modelName)
}
// RevisionForPath is RevisionFor for callers that hold loader options and a
// models path rather than an ApplicationConfig.
func (bcl *ModelConfigLoader) RevisionForPath(modelName, modelPath string, opts ...ConfigLoaderOption) (string, error) {
cfg, err := bcl.LoadModelConfigFileByName(modelName, modelPath, opts...)
if err != nil {
return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err)
}
return stampedRevision(cfg, modelName)
}
func stampedRevision(cfg *ModelConfig, modelName string) (string, error) {
revision := cfg.PersistedConfigRevision()
if revision == "" {
return "", fmt.Errorf("no config revision stamped for %q", modelName)
}
return revision, nil
}
core/config/model_config_loader_test.go
+54
View File
@@ -314,3 +314,57 @@ var _ = Describe("ModelConfigLoader alias resolution", func() {
Expect(loader.ValidateAliasTarget(&bad)).To(MatchError(ContainSubstring("itself an alias")))
})
})
var _ = Describe("ModelConfigLoader ResolveAliasName", func() {
var loader *ModelConfigLoader
BeforeEach(func() {
loader = NewModelConfigLoader("")
loader.configs["real"] = ModelConfig{Name: "real", Backend: "llama-cpp"}
loader.configs["production"] = ModelConfig{Name: "production", Alias: "real"}
loader.configs["chain"] = ModelConfig{Name: "chain", Alias: "production"}
loader.configs["dangling"] = ModelConfig{Name: "dangling", Alias: "nope"}
})
It("maps an alias name to the model that actually serves it", func() {
target, isAlias := loader.ResolveAliasName("production")
Expect(isAlias).To(BeTrue())
Expect(target).To(Equal("real"))
})
It("maps a real model name to itself", func() {
target, isAlias := loader.ResolveAliasName("real")
Expect(isAlias).To(BeFalse())
Expect(target).To(Equal("real"))
})
// A rule may be authored for a model that is not installed yet (pre-staging
// placement before standing up a node), so an unknown name must resolve to
// itself rather than to the empty string.
It("maps an unknown name to itself", func() {
target, isAlias := loader.ResolveAliasName("not-installed-yet")
Expect(isAlias).To(BeFalse())
Expect(target).To(Equal("not-installed-yet"))
})
// A broken alias has no model behind it. Resolving to itself keeps the
// caller on a name that simply has no replicas, instead of silently
// governing some other model.
It("maps a dangling alias to itself", func() {
target, isAlias := loader.ResolveAliasName("dangling")
Expect(isAlias).To(BeTrue())
Expect(target).To(Equal("dangling"))
})
It("maps a chained alias to itself rather than following the chain", func() {
target, isAlias := loader.ResolveAliasName("chain")
Expect(isAlias).To(BeTrue())
Expect(target).To(Equal("chain"))
})
It("maps the empty name to itself", func() {
target, isAlias := loader.ResolveAliasName("")
Expect(isAlias).To(BeFalse())
Expect(target).To(BeEmpty())
})
})
core/config/model_config_revision.go
+9 -2
View File
@@ -10,10 +10,17 @@ import (
"google.golang.org/protobuf/proto"
)
// ModelConfigRevision returns a stable revision of the persisted semantic
// modelConfigRevision returns a stable revision of the persisted semantic
// configuration. ModelConfig's JSON tags exclude runtime-derived state and
// source bookkeeping, while encoding/json orders map keys deterministically.
func ModelConfigRevision(cfg *ModelConfig) (string, error) {
//
// Deliberately unexported. It must only ever be called on a configuration as
// parsed from disk, before SetDefaults folds in the GGUF guess, the hardware
// defaults and app-level options. Callers outside this package cannot tell
// which they hold, and every time one hashed a defaulted or request-merged
// config it published a revision no inference request would carry, which makes
// the model unroutable. Use ModelConfigLoader.RevisionFor instead.
func modelConfigRevision(cfg *ModelConfig) (string, error) {
if cfg == nil {
return "", errors.New("model config is nil")
}
core/config/model_config_revision_stability_test.go
+64 -8
View File
@@ -51,9 +51,8 @@ template:
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
cfg, ok := loader.GetModelConfig("example")
Expect(ok).To(BeTrue())
revision, err := config.ModelConfigRevision(&cfg)
Expect(err).ToNot(HaveOccurred())
return revision
Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty())
return cfg.PersistedConfigRevision()
}
It("does not change when the same file is loaded repeatedly", func() {
@@ -73,18 +72,75 @@ template:
})
// The request pipeline reloads the config through LoadModelConfigFileByName,
// which applies SetDefaults a second time. That must not move the revision
// away from the one model administration publishes from the loader map.
// which applies SetDefaults a second time. The stamp is taken before those
// defaults, so both the stored config and the one a request resolves carry
// the same revision.
It("survives the extra SetDefaults the request path applies", func() {
loader := config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
stored, ok := loader.GetModelConfig("example")
Expect(ok).To(BeTrue())
adminRevision, err := config.ModelConfigRevision(&stored)
Expect(err).ToNot(HaveOccurred())
Expect(stored.PersistedConfigRevision()).ToNot(BeEmpty())
requestCfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig)
Expect(err).ToNot(HaveOccurred())
Expect(requestCfg.PersistedConfigRevision()).To(Equal(adminRevision))
Expect(requestCfg.PersistedConfigRevision()).To(Equal(stored.PersistedConfigRevision()))
})
})
// The revision must describe the configuration as persisted, and nothing else.
// SetDefaults folds in values that are not persisted config: the GGUF guess
// (which reads the model file and can fail on slow or remote storage), the
// hardware defaults, and app-level options like threads. Hashing after that
// made the revision a function of whether a multi-gigabyte file happened to
// parse, so one unchanged YAML produced two different revisions depending on
// the moment, and the controller rejected every request carrying the other one.
var _ = Describe("Model config revision independence from runtime defaults", func() {
It("does not change when SetDefaults is applied", func() {
dir := GinkgoT().TempDir()
body := "backend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\n" +
"mmproj: llama-cpp/mmproj/example/mmproj.gguf\nname: example\n" +
"parameters:\n model: llama-cpp/models/example/example.gguf\n"
Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), []byte(body), 0o600)).To(Succeed())
appConfig := config.NewApplicationConfig()
appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
loader := config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
stored, ok := loader.GetModelConfig("example")
Expect(ok).To(BeTrue())
before := stored.PersistedConfigRevision()
Expect(before).ToNot(BeEmpty())
// Applying defaults again is what the request path does.
stored.SetDefaults(appConfig.ToConfigLoaderOptions()...)
Expect(stored.PersistedConfigRevision()).To(Equal(before))
resolved, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig)
Expect(err).ToNot(HaveOccurred())
Expect(resolved.PersistedConfigRevision()).To(Equal(before),
"the request path must carry the same revision as the stored config")
})
It("does not change when app-level defaults differ", func() {
dir := GinkgoT().TempDir()
Expect(os.WriteFile(filepath.Join(dir, "example.yaml"),
[]byte("name: example\nbackend: llama-cpp\nparameters:\n model: m.gguf\n"), 0o600)).To(Succeed())
revWith := func(threads int, f16 bool) string {
appConfig := config.NewApplicationConfig()
appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
appConfig.Threads = threads
appConfig.F16 = f16
loader := config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
cfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig)
Expect(err).ToNot(HaveOccurred())
return cfg.PersistedConfigRevision()
}
Expect(revWith(8, false)).To(Equal(revWith(1, true)),
"an operator changing threads must not make every model unroutable")
})
})
core/config/model_config_revision_test.go
+4 -3
View File
@@ -19,10 +19,11 @@ var _ = Describe("Model configuration revisions", func() {
return cfg
}
// The raw hash is unexported on purpose, so these specs exercise it the way
// every caller now must: by stamping the parsed config.
revision := func(cfg *config.ModelConfig) string {
value, err := config.ModelConfigRevision(cfg)
Expect(err).NotTo(HaveOccurred())
return value
Expect(cfg.StampPersistedConfigRevision()).To(Succeed())
return cfg.PersistedConfigRevision()
}
It("is stable across equivalent YAML formatting and map order", func() {
core/gallery/cluster_env_test.go
+94
View File
@@ -0,0 +1,94 @@
package gallery_test
import (
"context"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/pkg/system"
)
// On a distributed controller the GPUs live on the workers, so a variant
// picker sized against the controller tells admins a cluster of A100s can only
// run the smallest CPU build.
var _ = Describe("ClusterResolveEnv", func() {
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
// The controller as Argus actually runs it: no GPU at all.
var controller *system.SystemState
BeforeEach(func() {
controller = system.NewCapabilityState("default")
})
It("sizes models against the cluster reading rather than the controller", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
Expect(env.AvailableMemory).To(Equal(gib(80)))
})
It("accepts a CUDA backend that only the workers can run", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
Expect(env.BackendCompatible).ToNot(BeNil())
// A name carrying the cuda token is what the controller rejects today;
// a bare engine name like "vllm" passes on any host and would prove
// nothing about the union.
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
Expect(env.BackendCompatible("llama-cpp")).To(BeTrue())
})
// The union must stay a filter, not an open door: a Linux NVIDIA fleet
// still cannot run an Apple-only build.
It("still rejects a backend no node in the cluster can run", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
Expect(env.BackendCompatible("mlx")).To(BeFalse())
})
It("accepts a backend that any one node in a mixed fleet can run", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13", "metal"})
Expect(env.BackendCompatible("mlx")).To(BeTrue())
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
})
// Ranking has to follow the hardware too, or a cluster of NVIDIA workers
// gets offered the GGUF build over the vLLM one it should prefer.
It("ranks engines by the workers' hardware, not the controller's", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
Expect(env.EnginePreference).To(Equal(system.NewCapabilityState("nvidia-cuda-13").EnginePreferenceTokens()))
})
// Every degradation path lands here, so it must be indistinguishable from
// the single-node behavior that shipped before any of this existed.
It("falls back to the host description when the cluster reports nothing", func() {
host := gallery.HostResolveEnv(context.Background(), controller)
env := gallery.ClusterResolveEnv(context.Background(), controller, 0, nil)
Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
Expect(env.EnginePreference).To(Equal(host.EnginePreference))
Expect(env.BackendCompatible("cuda-13-vllm")).To(Equal(host.BackendCompatible("cuda-13-vllm")))
Expect(env.BackendCompatible("mlx")).To(Equal(host.BackendCompatible("mlx")))
})
// A cluster that reports capabilities but no usable memory reading should
// still gain the hardware view; only the size question falls back.
It("keeps the host memory when only the memory reading is missing", func() {
host := gallery.HostResolveEnv(context.Background(), controller)
env := gallery.ClusterResolveEnv(context.Background(), controller, 0, []string{"nvidia-cuda-13"})
Expect(env.AvailableMemory).To(Equal(host.AvailableMemory))
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
})
It("keeps the probe wired so variant sizes are still measured", func() {
env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"})
Expect(env.ProbeMemory).ToNot(BeNil())
Expect(env.ServingFeaturePreference).To(Equal(system.ServingFeaturePreferenceTokens()))
})
})
core/gallery/models.go
+70
View File
@@ -938,3 +938,73 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error {
}
return nil
}
// ClusterResolveEnv describes a CLUSTER to variant selection, where
// HostResolveEnv describes one machine.
//
// It exists because a distributed controller is the wrong machine to ask. The
// controller is typically a GPU-less pod while every model actually runs on a
// worker, so a picker sized against it reports that a fleet of A100s can only
// run the smallest CPU build, and auto-selection then installs exactly that.
//
// availableMemory is the largest single healthy node's budget, and capabilities
// are the capability strings present in the cluster. Either may be empty: a
// zero memory reading keeps the host's own figure and an empty capability list
// keeps the host's own hardware verdict, so every degradation path lands back
// on the single-node behavior rather than on a cluster described as having
// nothing.
func ClusterResolveEnv(ctx context.Context, systemState *system.SystemState, availableMemory uint64, capabilities []string) ResolveEnv {
env := HostResolveEnv(ctx, systemState)
if availableMemory > 0 {
env.AvailableMemory = availableMemory
}
if len(capabilities) == 0 {
return env
}
// One state pinned per capability, mirroring AvailableBackendsForCapabilities:
// the controller's own detection must not leak into a worker's verdict, and
// a forced capability on the controller image must not either.
nodeStates := make([]*system.SystemState, 0, len(capabilities))
for _, capability := range capabilities {
nodeStates = append(nodeStates, system.NewCapabilityState(capability,
system.WithBackendPath(systemState.Backend.BackendsPath)))
}
hostCompatible := env.BackendCompatible
// A union, because a variant only has to run SOMEWHERE. The controller
// stays in the union so a cluster whose workers all went offline still
// describes itself the way it did before distributed mode existed.
env.BackendCompatible = func(backend string) bool {
if hostCompatible != nil && hostCompatible(backend) {
return true
}
for _, nodeState := range nodeStates {
if nodeState.IsBackendCompatible(backend, "") {
return true
}
}
return false
}
// Ranking follows the same hardware as the filter. Left on the controller's
// tokens, an NVIDIA fleet would be offered the GGUF build over the vLLM one
// even though nothing filtered the vLLM build out.
seen := make(map[string]struct{})
preference := make([]string, 0, len(nodeStates))
for _, nodeState := range nodeStates {
for _, token := range nodeState.EnginePreferenceTokens() {
if _, dup := seen[token]; dup {
continue
}
seen[token] = struct{}{}
preference = append(preference, token)
}
}
if len(preference) > 0 {
env.EnginePreference = preference
}
return env
}
core/http/endpoints/localai/backend.go
+3 -3
View File
@@ -356,7 +356,7 @@ func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc {
// local system state is the only thing worth filtering against.
type ClusterCapabilityProvider func(ctx context.Context) ([]string, error)
// resolveClusterCapabilities reads the capabilities present in the cluster,
// ResolveClusterCapabilities reads the capabilities present in the cluster,
// degrading to the local-only listing on error.
//
// Every capability-filtered discovery endpoint shares this: on a distributed
@@ -364,7 +364,7 @@ type ClusterCapabilityProvider func(ctx context.Context) ([]string, error)
// (usually GPU-less) host hides GPU-only backends the cluster can actually
// run. A registry hiccup must never blank the catalog, so a failure falls back
// to the pre-existing local-only behavior rather than erroring the request.
func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
func ResolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string {
if provider == nil {
return nil
}
@@ -423,7 +423,7 @@ func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[st
// @Router /backends/available [get]
func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
return func(c echo.Context) error {
capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
backends, err := gallery.AvailableBackendsForCapabilities(mgs.galleries, systemState, capabilities)
if err != nil {
core/http/endpoints/localai/edit_model_test.go
+4 -4
View File
@@ -294,9 +294,9 @@ var _ = Describe("Edit Model test", func() {
Expect(client.published[0]).To(Equal(messaging.CacheInvalidateEvent{
Element: "old", Op: "delete", ConfigRevision: modeladmin.DeletedModelConfigRevision("old"),
}))
newConfig, ok := loader.GetModelConfig("new")
_, ok := loader.GetModelConfig("new")
Expect(ok).To(BeTrue())
newRevision, err := config.ModelConfigRevision(&newConfig)
newRevision, err := loader.RevisionForPath("new", tempDir)
Expect(err).ToNot(HaveOccurred())
Expect(client.published[1]).To(Equal(messaging.CacheInvalidateEvent{
Element: "new", Op: "install", ConfigRevision: newRevision,
@@ -313,9 +313,9 @@ var _ = Describe("Edit Model test", func() {
}
_, oldOnPeer := peerLoader.GetModelConfig("old")
Expect(oldOnPeer).To(BeFalse())
peerConfig, newOnPeer := peerLoader.GetModelConfig("new")
_, newOnPeer := peerLoader.GetModelConfig("new")
Expect(newOnPeer).To(BeTrue())
peerRevision, err := config.ModelConfigRevision(&peerConfig)
peerRevision, err := peerLoader.RevisionForPath("new", tempDir)
Expect(err).ToNot(HaveOccurred())
Expect(peerRevision).To(Equal(newRevision))
Expect(peerLifecycle.batches).To(Equal([][]modeladmin.ModelRevisionTransition{
core/http/endpoints/localai/finetune.go
+1 -1
View File
@@ -276,7 +276,7 @@ func DownloadExportedModelEndpoint(ftService *finetune.FineTuneService) echo.Han
// ListFineTuneBackendsEndpoint returns installed backends tagged with "fine-tuning".
func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
return func(c echo.Context) error {
capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
if err != nil {
core/http/endpoints/localai/nodes.go
+15
View File
@@ -1218,6 +1218,20 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, err.Error()))
}
// A rule may be keyed by an alias, in which case it governs whatever
// that alias currently points at. Reject an alias that resolves to
// nothing, and reject a second rule for a model some other rule already
// governs, so the operator hears about the clash instead of silently
// writing a rule that never takes effect.
target, err := registry.ValidateSchedulingTarget(ctx, req.ModelName)
if err != nil {
status := http.StatusBadRequest
if errors.Is(err, nodes.ErrSchedulingConflict) {
status = http.StatusConflict
}
return c.JSON(status, nodeError(status, err.Error()))
}
// Serialize node selector to JSON
var selectorJSON string
if len(req.NodeSelector) > 0 {
@@ -1230,6 +1244,7 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc {
config := &nodes.ModelSchedulingConfig{
ModelName: req.ModelName,
TargetModel: target,
NodeSelector: selectorJSON,
MinReplicas: req.MinReplicas,
MaxReplicas: req.MaxReplicas,
core/http/endpoints/localai/nodes_backends_list_test.go
+2
View File
@@ -42,6 +42,8 @@ func (s *stubNodeCommandSender) StopBackend(_, _ string) error { return nil }
func (s *stubNodeCommandSender) UnloadModelOnNode(_, _ string) error { return nil }
func (s *stubNodeCommandSender) PingNode(_ string) error { return nil }
var _ = Describe("ListBackendsOnNodeEndpoint", func() {
var registry *nodes.NodeRegistry
core/http/endpoints/localai/nodes_scheduling_alias_test.go
+125
View File
@@ -0,0 +1,125 @@
package localai
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"github.com/labstack/echo/v4"
"github.com/mudler/LocalAI/core/services/nodes"
"github.com/mudler/LocalAI/core/services/testutil"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
// aliasResolverStub maps alias names to targets in place of a config loader.
type aliasResolverStub struct{ aliases map[string]string }
func (s *aliasResolverStub) ResolveAliasName(name string) (string, bool) {
target, ok := s.aliases[name]
if !ok {
return name, false
}
return target, true
}
var _ = Describe("Scheduling endpoints with model aliases", func() {
var (
registry *nodes.NodeRegistry
resolver *aliasResolverStub
)
BeforeEach(func() {
db := testutil.SetupTestDB()
var err error
registry, err = nodes.NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
resolver = &aliasResolverStub{aliases: map[string]string{"production": "qwen3"}}
registry.SetAliasResolver(resolver)
})
post := func(body string) *httptest.ResponseRecorder {
e := echo.New()
req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body))
req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
ExpectWithOffset(1, SetSchedulingEndpoint(registry)(c)).To(Succeed())
return rec
}
It("accepts a rule keyed by an alias and reports the model it governs", func() {
rec := post(`{"model_name":"production","min_replicas":2,"node_selector":{"tier":"gpu"}}`)
Expect(rec.Code).To(Equal(http.StatusOK))
var resp map[string]any
Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed())
Expect(resp["model_name"]).To(Equal("production"))
Expect(resp["target_model"]).To(Equal("qwen3"))
})
It("rejects a second rule for a model an alias rule already governs", func() {
Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK))
rec := post(`{"model_name":"qwen3","min_replicas":1}`)
Expect(rec.Code).To(Equal(http.StatusConflict))
Expect(rec.Body.String()).To(ContainSubstring("production"))
})
It("rejects an alias rule for a model that already has its own rule", func() {
Expect(post(`{"model_name":"qwen3","min_replicas":1}`).Code).To(Equal(http.StatusOK))
rec := post(`{"model_name":"production","min_replicas":2}`)
Expect(rec.Code).To(Equal(http.StatusConflict))
Expect(rec.Body.String()).To(ContainSubstring("qwen3"))
})
It("still allows editing a rule in place", func() {
Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK))
rec := post(`{"model_name":"production","min_replicas":4}`)
Expect(rec.Code).To(Equal(http.StatusOK))
stored, err := registry.GetModelScheduling(context.Background(), "production")
Expect(err).ToNot(HaveOccurred())
Expect(stored.MinReplicas).To(Equal(4))
})
It("rejects a rule keyed by an alias that does not resolve", func() {
resolver.aliases["orphan"] = "orphan"
rec := post(`{"model_name":"orphan","min_replicas":1}`)
Expect(rec.Code).To(Equal(http.StatusBadRequest))
Expect(rec.Body.String()).To(ContainSubstring("does not resolve"))
})
It("still accepts a rule for a model that is not installed yet", func() {
rec := post(`{"model_name":"not-installed-yet","min_replicas":1}`)
Expect(rec.Code).To(Equal(http.StatusOK))
})
It("labels a rule that another rule shadows when listing", func() {
// A seed file or a repointed alias can leave two rules on one model,
// which the write path above rejects but cannot retract.
Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})).To(Succeed())
Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "qwen3", MinReplicas: 1})).To(Succeed())
e := echo.New()
req := httptest.NewRequest(http.MethodGet, "/", nil)
rec := httptest.NewRecorder()
c := e.NewContext(req, rec)
Expect(ListSchedulingEndpoint(registry)(c)).To(Succeed())
var listed []map[string]any
Expect(json.Unmarshal(rec.Body.Bytes(), &listed)).To(Succeed())
byName := map[string]map[string]any{}
for _, item := range listed {
byName[item["model_name"].(string)] = item
}
Expect(byName["qwen3"]["shadowed"]).To(BeNil())
Expect(byName["production"]["shadowed"]).To(Equal(true))
})
})
core/http/endpoints/localai/quantization.go
+1 -1
View File
@@ -195,7 +195,7 @@ func DownloadQuantizedModelEndpoint(qService *quantization.QuantizationService)
// ListQuantizationBackendsEndpoint returns installed backends tagged with "quantization".
func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc {
return func(c echo.Context) error {
capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities)
installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled)
backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities)
if err != nil {
core/http/endpoints/ollama/models_test.go
+14
View File
@@ -146,6 +146,20 @@ parameters:
Expect(resp.Details.Format).To(Equal("gguf"))
Expect(resp.Details.Families).ToNot(BeEmpty())
})
It("looks up the model when the Ollama :latest tag is included", func() {
writeConfig("chat", `
name: chat
backend: llama-cpp
template:
chat: "{{ .Input }}"
parameters:
model: Llama-3-8B-Q4_K_M.gguf
`)
resp := callShow("chat:latest")
Expect(resp.Details.Format).To(Equal("gguf"))
Expect(resp.Capabilities).To(ContainElement("completion"))
})
})
Describe("ListModelsEndpoint", func() {
core/http/endpoints/openai/realtime_webrtc.go
+61 -6
View File
@@ -1,6 +1,9 @@
package openai
import (
"encoding/json"
"io"
"mime"
"net/http"
"time"
@@ -27,6 +30,61 @@ type RealtimeCallResponse struct {
SessionID string `json:"session_id"`
}
func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) {
var req RealtimeCallRequest
mediaType := ""
contentType := c.Request().Header.Get(echo.HeaderContentType)
if contentType != "" {
var err error
mediaType, _, err = mime.ParseMediaType(contentType)
if err != nil {
return req, false, err
}
}
switch mediaType {
case echo.MIMEMultipartForm:
if err := c.Request().ParseMultipartForm(32 << 20); err != nil {
return req, true, err
}
req.SDP = c.FormValue("sdp")
var session struct {
Model string `json:"model"`
LocalAIAssistant bool `json:"localai_assistant,omitempty"`
}
if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil {
return req, true, err
}
req.Model = session.Model
req.LocalAIAssistant = session.LocalAIAssistant
return req, true, nil
case "application/sdp":
sdp, err := readRealtimeSDP(c.Request().Body)
req.SDP = sdp
req.Model = c.QueryParam("model")
return req, true, err
default:
err := c.Bind(&req)
return req, false, err
}
}
func readRealtimeSDP(body io.Reader) (string, error) {
data, err := io.ReadAll(body)
return string(data), err
}
func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error {
if plainSDPResponse {
return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp))
}
return c.JSON(http.StatusCreated, RealtimeCallResponse{
SDP: sdp,
SessionID: sessionID,
})
}
// RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling.
func RealtimeCalls(application *application.Application) echo.HandlerFunc {
se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig())
@@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
if settingEngineErr != nil {
return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()})
}
var req RealtimeCallRequest
if err := c.Bind(&req); err != nil {
req, plainSDPResponse, err := decodeRealtimeCallRequest(c)
if err != nil {
return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"})
}
if req.SDP == "" {
@@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc {
runRealtimeSession(application, transport, req.Model, evaluator, opts)
}()
return c.JSON(http.StatusCreated, RealtimeCallResponse{
SDP: localDesc.SDP,
SessionID: sessionID,
})
return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID)
}
}
core/http/endpoints/openai/realtime_webrtc_request_test.go
+91
View File
@@ -0,0 +1,91 @@
package openai
import (
"bytes"
"mime/multipart"
"net/http"
"net/http/httptest"
"net/textproto"
"github.com/labstack/echo/v4"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("decodeRealtimeCallRequest", func() {
It("decodes the legacy JSON request", func() {
request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`))
request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON)
req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
Expect(err).NotTo(HaveOccurred())
Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
Expect(plainSDPResponse).To(BeFalse())
})
It("decodes the OpenAI multipart request", func() {
var body bytes.Buffer
writer := multipart.NewWriter(&body)
sdpHeader := make(textproto.MIMEHeader)
sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`)
sdpHeader.Set("Content-Type", "application/sdp")
sdpPart, err := writer.CreatePart(sdpHeader)
Expect(err).NotTo(HaveOccurred())
_, err = sdpPart.Write([]byte("offer"))
Expect(err).NotTo(HaveOccurred())
sessionHeader := make(textproto.MIMEHeader)
sessionHeader.Set("Content-Disposition", `form-data; name="session"`)
sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON)
sessionPart, err := writer.CreatePart(sessionHeader)
Expect(err).NotTo(HaveOccurred())
_, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`))
Expect(err).NotTo(HaveOccurred())
Expect(writer.Close()).To(Succeed())
request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body)
request.Header.Set(echo.HeaderContentType, writer.FormDataContentType())
req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
Expect(err).NotTo(HaveOccurred())
Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true}))
Expect(plainSDPResponse).To(BeTrue())
})
It("decodes a raw SDP request with the model query parameter", func() {
request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer"))
request.Header.Set(echo.HeaderContentType, "application/sdp")
req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder()))
Expect(err).NotTo(HaveOccurred())
Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"}))
Expect(plainSDPResponse).To(BeTrue())
})
})
var _ = Describe("writeRealtimeCallResponse", func() {
It("writes the bare SDP answer for OpenAI request formats", func() {
response := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
context := echo.New().NewContext(request, response)
Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed())
Expect(response.Code).To(Equal(http.StatusCreated))
Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp"))
Expect(response.Body.String()).To(Equal("answer"))
})
It("preserves the JSON response for legacy requests", func() {
response := httptest.NewRecorder()
request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil)
context := echo.New().NewContext(request, response)
Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed())
Expect(response.Code).To(Equal(http.StatusCreated))
Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON))
Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`))
})
})
core/http/middleware/request.go
+6
View File
@@ -141,6 +141,12 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR
}
modelName := input.ModelName(nil)
// Ollama-compat /api/tags appends ":latest" to untagged names.
// Strip it for lookup so the listed name works on /api/chat,
// /v1/chat/completions, and the other model-bearing endpoints.
if strings.HasSuffix(modelName, ":latest") {
modelName = strings.TrimSuffix(modelName, ":latest")
}
cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(modelName, re.applicationConfig)
if err != nil {
core/http/middleware/request_config_revision_test.go
+2 -2
View File
@@ -115,8 +115,8 @@ var _ = Describe("Model config revision seen by inference requests", func() {
Expect(admin.LoadModelConfigsFromPath(modelDir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
loaded, ok := admin.GetModelConfig("test-model")
Expect(ok).To(BeTrue())
adminRevision, err := config.ModelConfigRevision(&loaded)
Expect(err).ToNot(HaveOccurred())
adminRevision := loaded.PersistedConfigRevision()
Expect(adminRevision).ToNot(BeEmpty())
Expect(revisionFor(`{"model":"test-model","temperature":0.7,"messages":[{"role":"user","content":"hi"}]}`)).
To(Equal(adminRevision))
core/http/middleware/request_test.go
+14
View File
@@ -82,6 +82,13 @@ var _ = Describe("SetModelAndConfig middleware", func() {
Expect(resp.Error.Message).To(ContainSubstring("not found"))
Expect(resp.Error.Type).To(Equal("invalid_request_error"))
})
It("still 404s when :latest is appended to an unknown model", func() {
rec := postJSON(app, "/v1/chat/completions",
`{"model":"nonexistent-model:latest","messages":[{"role":"user","content":"hi"}]}`)
Expect(rec.Code).To(Equal(http.StatusNotFound))
})
})
Context("when the model exists as a config file", func() {
@@ -97,6 +104,13 @@ var _ = Describe("SetModelAndConfig middleware", func() {
Expect(rec.Code).To(Equal(http.StatusOK))
})
It("accepts the Ollama :latest tag that /api/tags appends", func() {
rec := postJSON(app, "/v1/chat/completions",
`{"model":"test-model:latest","messages":[{"role":"user","content":"hi"}]}`)
Expect(rec.Code).To(Equal(http.StatusOK))
})
})
Context("when the model exists as a pre-loaded config", func() {
core/http/react-ui/e2e/models-cluster-fit.spec.js
+120
View File
@@ -0,0 +1,120 @@
import { test, expect } from "./coverage-fixtures.js";
// On a distributed controller the models run on the workers, so every "will
// this fit" answer on this page is about their hardware. The controller is
// usually a GPU-less pod: sized against it, a cluster of A100s is told it can
// only run the smallest CPU build.
const GB = 1024 * 1024 * 1024;
const MODELS = [
{ name: "big-gpu-model", description: "Needs a real GPU", backend: "vllm", installed: false, tags: ["chat"] },
];
// 40GB: far past the controller's 8GB of RAM, comfortably inside one 80GB card.
const ESTIMATES = {
"big-gpu-model": {
sizeBytes: 40 * GB,
sizeDisplay: "40.0 GB",
estimates: { 8192: { vramBytes: 40 * GB, vramDisplay: "40.0 GB" } },
},
};
// The controller as Argus actually runs it: 8GB of system RAM, no GPU.
const CONTROLLER_ONLY = {
type: "ram",
available: true,
gpus: [],
aggregate: { total_memory: 8 * GB, used_memory: 2 * GB, free_memory: 6 * GB, gpu_count: 0 },
};
const WITH_CLUSTER = {
...CONTROLLER_ONLY,
cluster: {
enabled: true,
node_id: "n-1",
node_name: "dgx-01",
total_memory: 80 * GB,
is_gpu: true,
node_count: 4,
},
};
async function mockModels(page, resources) {
await page.route("**/api/models*", (route) =>
route.fulfill({
contentType: "application/json",
body: JSON.stringify({
models: MODELS,
allBackends: ["vllm"],
allTags: ["chat"],
availableModels: MODELS.length,
installedModels: 3,
totalPages: 1,
currentPage: 1,
}),
}),
);
await page.route("**/api/models/estimate/*", (route) => {
const name = decodeURIComponent(new URL(route.request().url()).pathname.split("/").pop());
return route.fulfill({ contentType: "application/json", body: JSON.stringify(ESTIMATES[name] || {}) });
});
await page.route("**/api/resources", (route) =>
route.fulfill({ contentType: "application/json", body: JSON.stringify(resources) }),
);
}
const railItems = (page) => page.locator('[data-testid="discover-rail-item"]');
const railItem = (page, name) => page.locator(`[data-entity="${name}"]`);
const railReady = (page) => expect(railItems(page).first()).toBeVisible({ timeout: 20_000 });
const PANE = '[data-testid="discover-pane"]';
test.describe("Models gallery - cluster-aware fit", () => {
test("a model that only a worker can hold is not called too large", async ({ page }) => {
await mockModels(page, WITH_CLUSTER);
await page.goto("/app/models");
await railReady(page);
// The whole defect in one assertion: 40GB against a 4-node cluster whose
// largest card holds 80GB.
await expect(railItem(page, "big-gpu-model")).toContainText("fits", { timeout: 20_000 });
await expect(railItem(page, "big-gpu-model")).not.toContainText("too large");
});
test("the fit verdict names the node it belongs to", async ({ page }) => {
await mockModels(page, WITH_CLUSTER);
await page.goto("/app/models");
await railReady(page);
await railItem(page, "big-gpu-model").click();
// Wait for the detail itself: until it renders, the pane still holds the
// zero-state hero, which names the node for its own reasons.
await expect(page.locator(PANE).getByText("40.0 GB")).toBeVisible({ timeout: 20_000 });
// The headroom this model has is headroom SOMEWHERE, and the stat says
// where rather than leaving it to read as this machine's.
await expect(page.locator(PANE)).toContainText(/headroom on dgx-01/i);
});
test("the host summary describes the cluster, not the controller", async ({ page }) => {
await mockModels(page, WITH_CLUSTER);
await page.goto("/app/models");
await railReady(page);
// 80 GB is the cluster's best node; 8 GB is this pod's own RAM and must
// not be what the page advertises.
await expect(page.locator(".zero-pane__title")).toContainText("80 GB");
await expect(page.locator(".zero-pane__title")).not.toContainText("8.00 GB");
});
// Single-node behavior is the fallback every degradation path lands on, so
// it has to stay exactly as it was.
test("without a cluster the verdict is still the local host's", async ({ page }) => {
await mockModels(page, CONTROLLER_ONLY);
await page.goto("/app/models");
await railReady(page);
await expect(railItem(page, "big-gpu-model")).toContainText("too large", { timeout: 20_000 });
});
});
core/http/react-ui/e2e/scheduling.spec.js
+150 -49
View File
@@ -36,35 +36,80 @@ async function mockScheduling(page, { rules = [rule], nodeList = nodes } = {}) {
}
test.describe('Scheduling page', () => {
test('groups node labels, collapses the reference, filters forgivingly, and expands results', async ({ page }) => {
// Node labels are only ever needed while writing a rule's node selector, so
// they live in that field rather than in a card standing open above the
// rules whether or not anyone is writing one.
test('keeps no standing label browser on the page', async ({ page }) => {
await mockScheduling(page)
await page.goto('/app/scheduling')
await expect(page.getByText('llama-3.3')).toBeVisible()
const reference = page.getByTestId('node-label-reference')
await expect(reference.getByText('Falcon GPU')).toBeVisible()
await expect(reference.getByText('No labels')).toBeVisible()
await expect(reference.locator('.scheduling-node-card')).toHaveCount(5)
await expect(reference.getByText('5 of 27 nodes')).toBeVisible()
await expect(page.getByTestId('node-label-reference')).toHaveCount(0)
await expect(page.getByRole('button', { name: /node labels/i })).toHaveCount(0)
await expect(page.locator('.scheduling-node-card')).toHaveCount(0)
// Falcon GPU is a node name, and nothing on this page has a reason to
// enumerate node names until a selector is being filled.
await expect(page.getByText('Falcon GPU')).toHaveCount(0)
})
const toggle = page.getByRole('button', { name: /node labels/i })
await expect(toggle).toHaveAttribute('aria-expanded', 'true')
await toggle.click()
await expect(toggle).toHaveAttribute('aria-expanded', 'false')
await expect(reference.getByRole('searchbox')).toBeHidden()
await toggle.click()
test('suggests the cluster\'s own label keys and values as the selector is typed', async ({ page }) => {
await mockScheduling(page)
await page.goto('/app/scheduling')
await page.getByRole('button', { name: 'Add Scheduling Rule' }).click()
await reference.getByRole('searchbox').fill('GPU.VENDOR=nvi')
await expect(reference.locator('.scheduling-node-card')).toHaveCount(1)
await expect(reference.getByText('Falcon GPU')).toBeVisible()
const keyInput = page.getByRole('combobox', { name: 'Selector key' })
await keyInput.click()
const suggestions = page.getByTestId('label-suggestions')
// Every key the cluster reports, before a single character is typed.
await expect(suggestions.getByRole('option', { name: 'gpu.vendor' })).toBeVisible()
await expect(suggestions.getByRole('option', { name: 'zone' })).toBeVisible()
await reference.getByRole('searchbox').fill('flcn')
await expect(reference.locator('.scheduling-node-card')).toHaveCount(1)
await expect(reference.getByText('Falcon GPU')).toBeVisible()
await keyInput.fill('vend')
await expect(suggestions.getByRole('option')).toHaveCount(1)
await suggestions.getByRole('option', { name: 'gpu.vendor' }).click()
await expect(keyInput).toHaveValue('gpu.vendor')
await reference.getByRole('searchbox').fill('')
await reference.getByRole('button', { name: 'Show 20 more nodes' }).click()
await expect(reference.locator('.scheduling-node-card')).toHaveCount(25)
await expect(reference.getByText('25 of 27 nodes')).toBeVisible()
// Values are scoped to the key being filled, so a selector cannot be built
// out of a pair no node matches.
const valueInput = page.getByRole('combobox', { name: 'Selector value' })
await valueInput.click()
await expect(suggestions.getByRole('option', { name: 'NVIDIA' })).toBeVisible()
await expect(suggestions.getByRole('option', { name: 'amd' })).toBeVisible()
await expect(suggestions.getByRole('option', { name: 'east' })).toHaveCount(0)
await valueInput.fill('nvi')
await suggestions.getByRole('option', { name: 'NVIDIA' }).click()
await expect(valueInput).toHaveValue('NVIDIA')
})
test('picks a suggestion from the keyboard', async ({ page }) => {
await mockScheduling(page)
await page.goto('/app/scheduling')
await page.getByRole('button', { name: 'Add Scheduling Rule' }).click()
const keyInput = page.getByRole('combobox', { name: 'Selector key' })
await keyInput.fill('zon')
await keyInput.press('ArrowDown')
await keyInput.press('Enter')
await expect(keyInput).toHaveValue('zone')
// Enter picked the suggestion rather than committing the chip, so the
// half-built pair is still in the inputs.
await expect(page.getByLabel('Node selector').getByText('zone=', { exact: true })).toHaveCount(0)
})
// The cluster's vocabulary is a suggestion, never a constraint: an admin
// labelling nodes for a rule they are about to write must still be able to
// type a key no node reports yet.
test('still accepts a label the cluster has never reported', async ({ page }) => {
await mockScheduling(page)
await page.goto('/app/scheduling')
await page.getByRole('button', { name: 'Add Scheduling Rule' }).click()
await page.getByRole('combobox', { name: 'Selector key' }).fill('tenant')
await page.getByRole('combobox', { name: 'Selector value' }).fill('acme')
await page.getByRole('button', { name: 'Add selector' }).click()
await expect(page.getByLabel('Node selector').getByText('tenant=acme', { exact: true })).toBeVisible()
})
test('edits all fields with a locked model and preserves values after a failed save', async ({ page }) => {
@@ -113,42 +158,98 @@ test.describe('Scheduling page', () => {
await expect(page.getByRole('combobox', { name: '' }).first()).toBeEnabled()
})
test('shows node loading, empty, no-match, and retry states independently from rules', async ({ page }) => {
let attempts = 0
// The roster feeds suggestions and nothing else now, so failing to load it
// must cost the admin nothing but the hints.
test('leaves the selector fully usable when the node roster fails to load', async ({ page }) => {
await page.route('**/api/nodes/scheduling', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([rule]) }))
await page.route('**/api/nodes', async route => {
attempts++
if (attempts === 1) {
await new Promise(resolve => setTimeout(resolve, 250))
await route.fulfill({ status: 500, body: 'failed' })
} else {
await route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })
}
})
await page.route('**/api/nodes', route => route.fulfill({ status: 500, body: 'failed' }))
await page.goto('/app/scheduling')
await expect(page.getByText('Loading node labels…')).toBeVisible()
await expect(page.getByText('llama-3.3')).toBeVisible()
await expect(page.getByText('Could not load node labels.')).toBeVisible()
await page.getByRole('button', { name: 'Retry loading node labels' }).click()
await expect(page.getByText('No nodes are available yet.')).toBeVisible()
await page.unroute('**/api/nodes')
await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) }))
await page.reload()
await page.getByRole('searchbox', { name: 'Search node labels' }).fill('not-a-real-label')
await expect(page.getByText('No nodes match your search.')).toBeVisible()
// The rules still render: the roster is not on their path.
await expect(page.getByText('llama-3.3')).toBeVisible()
await page.getByRole('button', { name: 'Add Scheduling Rule' }).click()
await page.getByRole('combobox', { name: 'Selector key' }).fill('gpu.vendor')
await page.getByRole('combobox', { name: 'Selector value' }).fill('nvidia')
await page.getByRole('button', { name: 'Add selector' }).click()
await expect(page.getByLabel('Node selector').getByText('gpu.vendor=nvidia', { exact: true })).toBeVisible()
})
test('uses one node column and accessible rule actions on a narrow viewport', async ({ page }) => {
// A rule may be keyed by an alias, in which case it governs whichever model
// the alias points at. The page has to say which model that is, because the
// rule's own name no longer tells you.
test.describe('rules keyed by a model alias', () => {
const aliasRule = {
model_name: 'production',
target_model: 'llama-3.3',
model_is_alias: true,
node_selector: { tier: 'gpu' },
min_replicas: 2,
max_replicas: 4,
}
async function mockAliases(page, aliases = [{ name: 'production', target: 'llama-3.3' }]) {
await page.route('**/api/aliases', route => route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify(aliases),
}))
await page.route('**/api/models/capabilities', route => route.fulfill({
status: 200,
contentType: 'application/json',
body: JSON.stringify({ object: 'list', data: [{ id: 'llama-3.3' }, { id: 'production' }] }),
}))
}
test('names the model an alias rule governs', async ({ page }) => {
await mockScheduling(page, { rules: [aliasRule] })
await mockAliases(page)
await page.goto('/app/scheduling')
await expect(page.getByText('production')).toBeVisible()
await expect(page.locator('.scheduling-rule-target')).toHaveText(/llama-3\.3/)
})
test('marks a rule another rule already governs as shadowed', async ({ page }) => {
await mockScheduling(page, { rules: [{ ...aliasRule, shadowed: true }, rule] })
await mockAliases(page)
await page.goto('/app/scheduling')
await expect(page.locator('.scheduling-rule-shadowed')).toHaveCount(1)
await expect(page.locator('.scheduling-rule-shadowed')).toContainText('Shadowed')
})
test('flags an alias rule that no longer resolves', async ({ page }) => {
await mockScheduling(page, {
rules: [{ model_name: 'orphan', target_model: 'orphan', model_is_alias: true, min_replicas: 1 }],
})
await mockAliases(page, [])
await page.goto('/app/scheduling')
await expect(page.locator('.scheduling-rule-target--broken')).toBeVisible()
})
test('offers aliases in the model picker, tagged with their target', async ({ page }) => {
await mockScheduling(page)
await mockAliases(page)
await page.goto('/app/scheduling')
await page.getByRole('button', { name: 'Add Scheduling Rule' }).click()
const picker = page.locator('.searchable-model-select input')
await picker.click()
await expect(page.locator('.sms-hint')).toHaveText('alias of llama-3.3')
await page.getByRole('option', { name: /production/ }).click()
await expect(page.getByText(/production is an alias for llama-3\.3/)).toBeVisible()
})
})
test('keeps rule actions reachable on a narrow viewport', async ({ page }) => {
await page.setViewportSize({ width: 390, height: 844 })
await mockScheduling(page, { nodeList: nodes.slice(0, 2) })
await page.goto('/app/scheduling')
const cards = page.locator('.scheduling-node-card')
const first = await cards.nth(0).boundingBox()
const second = await cards.nth(1).boundingBox()
expect(second.y).toBeGreaterThan(first.y + first.height - 1)
const actions = page.locator('.scheduling-rule-actions')
await expect(actions.getByRole('button', { name: 'Edit llama-3.3' })).toBeVisible()
await expect(actions.getByRole('button', { name: 'Delete llama-3.3' })).toBeVisible()
core/http/react-ui/inline-style-baseline.txt
+1 -1
View File
@@ -1 +1 @@
519
514
core/http/react-ui/public/locales/de/models.json
+4 -1
View File
@@ -142,7 +142,8 @@
"sha256": "SHA256",
"backToAll": "Alle Modelle",
"vramAt": "VRAM bei {{context}}",
"headroom": "Spielraum"
"headroom": "Spielraum",
"headroomOn": "Spielraum auf {{node}}"
},
"empty": {
"title": "Keine Modelle gefunden",
@@ -216,6 +217,8 @@
"browsing": "Durchsuchen",
"pickHint": "Wähle ein Modell, um die Details zu sehen.",
"heroWithRam": "{{ram}} Systemspeicher, {{count}} Modelle in der Galerie.",
"heroWithCluster": "{{vram}} auf {{node}}, dem größten von {{nodes}} Knoten, {{count}} Modelle in der Galerie.",
"heroWithNode": "{{vram}} auf {{node}}, {{count}} Modelle in der Galerie.",
"byUseCase": "Oder mit einem Anwendungsfall starten",
"pickText": "Chat, Reasoning, Embeddings",
"pickVision": "Bilder und Dokumente lesen",
core/http/react-ui/public/locales/en/chat.json
+2
View File
@@ -33,6 +33,8 @@
"title": "Chat Settings",
"manageMode": "Manage mode",
"manageModeDesc": "Let this chat install models, switch backends, and edit configs by talking to LocalAI.",
"focusMode": "Focus mode",
"focusModeDesc": "Collapse the sidebar and slim the header while a conversation is active. Esc restores them temporarily; turn this off to keep the full layout.",
"systemPrompt": "System Prompt",
"systemPromptPlaceholder": "You are a helpful assistant...",
"temperature": "Temperature",
core/http/react-ui/public/locales/en/models.json
+4 -1
View File
@@ -152,7 +152,8 @@
"sha256": "SHA256",
"backToAll": "All models",
"vramAt": "VRAM at {{context}}",
"headroom": "Headroom"
"headroom": "Headroom",
"headroomOn": "Headroom on {{node}}"
},
"empty": {
"title": "No models found",
@@ -232,6 +233,8 @@
"browsing": "Browsing",
"pickHint": "Select a model to see its detail.",
"heroWithRam": "{{ram}} of system memory, {{count}} models in the gallery.",
"heroWithCluster": "{{vram}} on {{node}}, the largest of {{nodes}} nodes, {{count}} models in the gallery.",
"heroWithNode": "{{vram}} on {{node}}, {{count}} models in the gallery.",
"byUseCase": "Or start with a use case",
"pickText": "Chat, reasoning, embeddings",
"pickVision": "Read images and documents",
core/http/react-ui/public/locales/es/models.json
+4 -1
View File
@@ -142,7 +142,8 @@
"sha256": "SHA256",
"backToAll": "Todos los modelos",
"vramAt": "VRAM a {{context}}",
"headroom": "Margen"
"headroom": "Margen",
"headroomOn": "Margen en {{node}}"
},
"empty": {
"title": "No se encontraron modelos",
@@ -216,6 +217,8 @@
"browsing": "Explorando",
"pickHint": "Selecciona un modelo para ver su detalle.",
"heroWithRam": "{{ram}} de memoria del sistema, {{count}} modelos en la galería.",
"heroWithCluster": "{{vram}} en {{node}}, el mayor de {{nodes}} nodos, {{count}} modelos en la galería.",
"heroWithNode": "{{vram}} en {{node}}, {{count}} modelos en la galería.",
"byUseCase": "O empieza por un caso de uso",
"pickText": "Chat, razonamiento, embeddings",
"pickVision": "Leer imágenes y documentos",
core/http/react-ui/public/locales/id/models.json
+4 -1
View File
@@ -149,7 +149,8 @@
"sha256": "SHA256",
"backToAll": "Semua model",
"vramAt": "VRAM pada {{context}}",
"headroom": "Sisa ruang"
"headroom": "Sisa ruang",
"headroomOn": "Sisa ruang di {{node}}"
},
"empty": {
"title": "Model tidak ditemukan",
@@ -229,6 +230,8 @@
"browsing": "Menjelajah",
"pickHint": "Pilih model untuk melihat detailnya.",
"heroWithRam": "Memori sistem {{ram}}, {{count}} model di galeri.",
"heroWithCluster": "{{vram}} di {{node}}, terbesar dari {{nodes}} node, {{count}} model di galeri.",
"heroWithNode": "{{vram}} di {{node}}, {{count}} model di galeri.",
"byUseCase": "Atau mulai dari kasus penggunaan",
"pickText": "Obrolan, penalaran, embedding",
"pickVision": "Membaca gambar dan dokumen",
core/http/react-ui/public/locales/it/models.json
+4 -1
View File
@@ -142,7 +142,8 @@
"sha256": "SHA256",
"backToAll": "Tutti i modelli",
"vramAt": "VRAM a {{context}}",
"headroom": "Margine"
"headroom": "Margine",
"headroomOn": "Margine su {{node}}"
},
"empty": {
"title": "Nessun modello trovato",
@@ -216,6 +217,8 @@
"browsing": "Esplorazione",
"pickHint": "Seleziona un modello per vederne i dettagli.",
"heroWithRam": "{{ram}} di memoria di sistema, {{count}} modelli nella galleria.",
"heroWithCluster": "{{vram}} su {{node}}, il più grande di {{nodes}} nodi, {{count}} modelli nella galleria.",
"heroWithNode": "{{vram}} su {{node}}, {{count}} modelli nella galleria.",
"byUseCase": "Oppure parti da un caso d’uso",
"pickText": "Chat, ragionamento, embedding",
"pickVision": "Leggere immagini e documenti",
core/http/react-ui/public/locales/ko/models.json
+4 -1
View File
@@ -148,7 +148,8 @@
"sha256": "SHA256",
"backToAll": "모든 모델",
"vramAt": "{{context}}에서의 VRAM",
"headroom": "여유 공간"
"headroom": "여유 공간",
"headroomOn": "{{node}}의 여유 공간"
},
"empty": {
"title": "모델을 찾을 수 없습니다",
@@ -200,6 +201,8 @@
"browsing": "둘러보기",
"pickHint": "모델을 선택하면 상세 정보가 표시됩니다.",
"heroWithRam": "시스템 메모리 {{ram}}, 갤러리에 모델 {{count}}개.",
"heroWithCluster": "{{nodes}}개 노드 중 가장 큰 {{node}}에 {{vram}}, 갤러리에 모델 {{count}}개.",
"heroWithNode": "{{node}}에 {{vram}}, 갤러리에 모델 {{count}}개.",
"byUseCase": "또는 용도로 시작하기",
"pickText": "채팅, 추론, 임베딩",
"pickVision": "이미지와 문서 읽기",
core/http/react-ui/public/locales/pt-BR/models.json
+4 -1
View File
@@ -152,7 +152,8 @@
"sha256": "SHA256",
"backToAll": "Todos os modelos",
"vramAt": "VRAM em {{context}}",
"headroom": "Margem de sobra"
"headroom": "Margem de sobra",
"headroomOn": "Folga em {{node}}"
},
"empty": {
"title": "Nenhum modelo encontrado",
@@ -232,6 +233,8 @@
"browsing": "Explorando",
"pickHint": "Selecione um modelo para ver seus detalhes.",
"heroWithRam": "{{ram}} de memória do sistema, {{count}} modelos na galeria.",
"heroWithCluster": "{{vram}} em {{node}}, o maior de {{nodes}} nós, {{count}} modelos na galeria.",
"heroWithNode": "{{vram}} em {{node}}, {{count}} modelos na galeria.",
"byUseCase": "Ou comece por um caso de uso",
"pickText": "Chat, raciocínio, embeddings",
"pickVision": "Leia imagens e documentos",
core/http/react-ui/public/locales/zh-CN/models.json
+4 -1
View File
@@ -142,7 +142,8 @@
"sha256": "SHA256",
"backToAll": "全部模型",
"vramAt": "{{context}} 时显存",
"headroom": "剩余显存"
"headroom": "剩余显存",
"headroomOn": "{{node}} 上的余量"
},
"empty": {
"title": "未找到模型",
@@ -216,6 +217,8 @@
"browsing": "浏览中",
"pickHint": "选择一个模型以查看详情。",
"heroWithRam": "{{ram}} 系统内存，图库中有 {{count}} 个模型。",
"heroWithCluster": "{{node}} 上 {{vram}}，为 {{nodes}} 个节点中最大，图库中有 {{count}} 个模型。",
"heroWithNode": "{{node}} 上 {{vram}}，图库中有 {{count}} 个模型。",
"byUseCase": "或从用途开始",
"pickText": "对话、推理、向量",
"pickVision": "读取图像与文档",
core/http/react-ui/src/App.css
+91 -110
View File
@@ -2697,116 +2697,82 @@ select.input {
cursor: not-allowed;
}
.scheduling-node-reference {
margin-bottom: var(--spacing-md);
overflow: hidden;
/* Key-value chip builder (node selectors, node labels) */
.kvchips__chips {
display: flex;
flex-wrap: wrap;
gap: 4px;
margin-bottom: var(--spacing-xs);
}
.scheduling-node-reference__toggle {
.kvchips__chip {
align-items: center;
background: transparent;
border: 0;
background: var(--color-bg-tertiary);
border: 1px solid var(--color-border-subtle);
border-radius: var(--radius-sm);
display: inline-flex;
font-family: var(--font-mono);
font-size: 0.75rem;
gap: 4px;
padding: 2px 8px;
}
.kvchips__chip-remove {
background: none;
border: none;
color: var(--color-text-muted);
cursor: pointer;
font-size: 0.625rem;
padding: 0;
}
.kvchips__row {
align-items: stretch;
display: flex;
gap: var(--spacing-xs);
position: relative;
}
.kvchips__add {
min-height: 36px;
}
/* Anchored to the input row so the list covers what follows the field rather
than pushing the rest of the form down as the user types. */
.kvchips__suggestions {
background: var(--color-bg-secondary);
border: 1px solid var(--color-border);
border-radius: var(--radius-sm);
box-shadow: var(--shadow-md);
left: 0;
list-style: none;
margin: 4px 0 0;
max-height: 220px;
overflow-y: auto;
padding: 4px;
position: absolute;
right: 0;
top: 100%;
z-index: 20;
}
.kvchips__suggestion {
background: none;
border: none;
border-radius: var(--radius-sm);
color: var(--color-text-primary);
cursor: pointer;
display: flex;
font: inherit;
font-weight: var(--font-weight-semibold);
justify-content: space-between;
padding: var(--spacing-md);
display: block;
font-family: var(--font-mono);
font-size: 0.8125rem;
padding: var(--spacing-xs) var(--spacing-sm);
text-align: left;
width: 100%;
}
.scheduling-node-reference__toggle:focus-visible {
outline: 2px solid var(--color-primary);
outline-offset: -2px;
}
.scheduling-node-reference__content {
border-top: 1px solid var(--color-border-subtle);
padding: var(--spacing-md);
}
.scheduling-node-reference__content > .text-note {
margin: 0 0 var(--spacing-sm);
}
.scheduling-node-toolbar {
align-items: center;
display: flex;
gap: var(--spacing-md);
margin-bottom: var(--spacing-md);
}
.scheduling-node-toolbar .input {
flex: 1;
}
.scheduling-node-toolbar .text-meta {
flex: none;
}
.scheduling-node-grid {
display: grid;
gap: var(--spacing-sm);
grid-template-columns: repeat(auto-fit, minmax(240px, 1fr));
}
.scheduling-node-card {
.kvchips__suggestion:hover,
.kvchips__suggestion--active {
background: var(--color-bg-tertiary);
border: 1px solid var(--color-border-subtle);
border-radius: var(--radius-md);
min-width: 0;
padding: var(--spacing-sm);
}
.scheduling-node-card__header {
align-items: center;
display: flex;
gap: var(--spacing-sm);
justify-content: space-between;
margin-bottom: var(--spacing-xs);
}
.scheduling-node-status {
color: var(--color-text-muted);
font-size: 0.75rem;
text-transform: capitalize;
}
.scheduling-node-status--online,
.scheduling-node-status--ready,
.scheduling-node-status--healthy {
color: var(--color-success);
}
.scheduling-node-labels {
display: flex;
flex-wrap: wrap;
gap: 4px;
}
.scheduling-node-label {
border: 1px solid var(--color-border-subtle);
border-radius: var(--radius-sm);
font-family: var(--font-mono);
font-size: 0.75rem;
overflow-wrap: anywhere;
padding: 2px 6px;
}
.scheduling-node-message {
align-items: center;
color: var(--color-text-muted);
display: flex;
gap: var(--spacing-sm);
justify-content: center;
margin: var(--spacing-md) 0;
text-align: center;
}
.scheduling-show-more {
margin-top: var(--spacing-md);
}
.scheduling-rule-actions {
@@ -2815,17 +2781,32 @@ select.input {
justify-content: flex-end;
}
/* Second line of a rule's Model cell: the model an alias-keyed rule currently
governs. The cell itself is bold, so the weight is reset here rather than
inherited. */
.scheduling-rule-target {
font-weight: 400;
}
.scheduling-rule-target--broken {
font-weight: 400;
color: var(--color-warning);
}
/* Status pill for a rule another rule already governs, so it has no effect.
Mirrors the unsatisfiable pill's shape. */
.scheduling-rule-shadowed {
display: inline-block;
font-size: var(--text-xs);
padding: 2px 8px;
border-radius: var(--radius-sm);
font-weight: 600;
background: var(--color-bg-tertiary);
border: 1px solid var(--color-warning);
color: var(--color-warning);
}
@media (max-width: 640px) {
.scheduling-node-grid {
grid-template-columns: minmax(0, 1fr);
}
.scheduling-node-toolbar {
align-items: stretch;
flex-direction: column;
gap: var(--spacing-xs);
}
.scheduling-rule-actions {
width: 100%;
}
core/http/react-ui/src/components/SearchableModelSelect.jsx
+15 -2
View File
@@ -7,7 +7,10 @@ import { useModels } from '../hooks/useModels'
// query isn't treated as a chosen value. After a commit the field is cleared,
// matching the add-and-clear flow. Default false keeps the as-you-type
// behaviour single-value editors rely on.
export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false }) {
// hints: optional { [modelId]: string } shown as muted text beside an entry and
// searchable along with the name. Used to mark aliases with the model they
// point at, so a picker that lists both can tell them apart.
export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false, hints = {} }) {
const { models, loading } = useModels(capability)
const [query, setQuery] = useState('')
const [open, setOpen] = useState(false)
@@ -29,8 +32,10 @@ export default function SearchableModelSelect({ value, onChange, capability, pla
return () => document.removeEventListener('mousedown', handler)
}, [])
const needle = query.toLowerCase()
const filtered = models.filter(m =>
m.id.toLowerCase().includes(query.toLowerCase())
m.id.toLowerCase().includes(needle) ||
(hints[m.id] || '').toLowerCase().includes(needle)
)
// Which item Enter will select — matches SearchableSelect behavior
@@ -126,6 +131,11 @@ export default function SearchableModelSelect({ value, onChange, capability, pla
color: var(--color-primary);
font-weight: 600;
}
.sms-hint {
color: var(--color-text-muted);
font-size: 0.75rem;
flex-shrink: 0;
}
.sms-empty {
padding: 8px 10px;
font-size: 0.8125rem;
@@ -172,6 +182,9 @@ export default function SearchableModelSelect({ value, onChange, capability, pla
}}
>
<span style={{ flex: 1, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>{m.id}</span>
{hints[m.id] && (
<span className="sms-hint">{hints[m.id]}</span>
)}
{isEnterTarget && (
<span style={{ color: 'var(--color-text-muted)', fontSize: '0.75rem', flexShrink: 0 }}>↵</span>
)}
core/http/react-ui/src/components/nodes/KeyValueChips.jsx
+125 -38
View File
@@ -1,4 +1,6 @@
import { useState } from 'react'
import { useState, useRef, useEffect } from 'react'
import { suggestKeys, suggestValues } from '../../utils/nodeLabelSuggestions'
/**
* Controlled chip-builder for { key: value } maps. Replaces the prior
@@ -11,6 +13,14 @@ import { useState } from 'react'
* labels editor). The component just renders chips and a key/value input
* row.
*
* With `suggestions` it also completes what the user types against the
* vocabulary the cluster actually uses. That is where label discovery lives on
* the scheduling page: labels only matter while a selector is being written,
* so browsing them belongs in the field rather than in a card standing open
* above the rules. The suggestions are never a constraint - a key no node
* reports yet still commits as typed, which is the workflow of writing a rule
* before labelling the nodes for it.
*
* Props:
* pairs - current map of key -> value
* onAdd(k,v) - called when the user adds a pair (parent handles dedup
@@ -18,45 +28,119 @@ import { useState } from 'react'
* onRemove(k) - called when a chip's × is clicked
* placeholderKey, placeholderValue - input hints
* ariaLabel - accessible name for the section
* ariaLabelKey, ariaLabelValue - accessible names for the two inputs
* addLabel - accessible name for the commit button
* suggestions - label index from utils/nodeLabelSuggestions; omit for none
*/
export default function KeyValueChips({ pairs, onAdd, onRemove, placeholderKey = 'key', placeholderValue = 'value', ariaLabel }) {
export default function KeyValueChips({
pairs, onAdd, onRemove,
placeholderKey = 'key', placeholderValue = 'value',
ariaLabel, ariaLabelKey = 'Key', ariaLabelValue = 'Value',
addLabel = 'Add', suggestions,
}) {
const [k, setK] = useState('')
const [v, setV] = useState('')
// Which input owns the open list, and which of its options is armed for
// Enter. -1 means the user is typing free text and Enter should commit the
// pair rather than pick anything.
const [openField, setOpenField] = useState(null)
const [active, setActive] = useState(-1)
const rowRef = useRef(null)
const entries = pairs ? Object.entries(pairs) : []
const options = !suggestions || !openField
? []
: openField === 'key'
? suggestKeys(suggestions, k, entries.map(([key]) => key))
: suggestValues(suggestions, k.trim(), v)
// A click anywhere else is a dismissal. Without this the list survives the
// user moving on to the rest of the form and covers it.
useEffect(() => {
if (!openField) return undefined
const onDocumentPointerDown = (event) => {
if (!rowRef.current?.contains(event.target)) setOpenField(null)
}
document.addEventListener('mousedown', onDocumentPointerDown)
return () => document.removeEventListener('mousedown', onDocumentPointerDown)
}, [openField])
const add = () => {
const key = k.trim()
if (!key) return
onAdd(key, v.trim())
setK(''); setV('')
}
const onKeyDown = (e) => {
if (e.key === 'Enter') { e.preventDefault(); add() }
setOpenField(null); setActive(-1)
}
const entries = pairs ? Object.entries(pairs) : []
const pick = (field, option) => {
if (field === 'key') setK(option)
else setV(option)
setOpenField(null)
setActive(-1)
}
const onKeyDown = (field) => (e) => {
const open = openField === field && options.length > 0
if (e.key === 'ArrowDown' && open) {
e.preventDefault()
setActive(current => (current + 1) % options.length)
return
}
if (e.key === 'ArrowUp' && open) {
e.preventDefault()
setActive(current => (current <= 0 ? options.length - 1 : current - 1))
return
}
if (e.key === 'Escape' && openField) {
e.preventDefault()
setOpenField(null)
setActive(-1)
return
}
if (e.key === 'Enter') {
e.preventDefault()
// Enter completes the suggestion the user armed, and commits the pair
// otherwise. Committing a half-typed key because a list happened to be
// open is the error this ordering avoids.
if (open && active >= 0) pick(field, options[active])
else add()
}
}
const listId = 'kvchips-suggestions'
const inputProps = (field, value, setValue, placeholder, label) => ({
className: 'input flex-1',
type: 'text',
role: suggestions ? 'combobox' : undefined,
'aria-expanded': suggestions ? openField === field : undefined,
'aria-controls': suggestions && openField === field ? listId : undefined,
'aria-autocomplete': suggestions ? 'list' : undefined,
'aria-label': label,
placeholder,
value,
onChange: (e) => {
setValue(e.target.value)
if (suggestions) { setOpenField(field); setActive(-1) }
},
onFocus: () => { if (suggestions) { setOpenField(field); setActive(-1) } },
onKeyDown: onKeyDown(field),
})
return (
<div aria-label={ariaLabel}>
{entries.length > 0 && (
<div style={{ display: 'flex', flexWrap: 'wrap', gap: 4, marginBottom: 'var(--spacing-xs)' }}>
<div className="kvchips__chips">
{entries.map(([key, val]) => (
<span key={key} style={{
display: 'inline-flex', alignItems: 'center', gap: 4,
fontSize: '0.75rem', padding: '2px 8px',
borderRadius: 'var(--radius-sm)',
background: 'var(--color-bg-tertiary)',
border: '1px solid var(--color-border-subtle)',
fontFamily: 'var(--font-mono)',
}}>
<span key={key} className="kvchips__chip">
{key}={val}
<button
type="button"
onClick={(e) => { e.stopPropagation(); onRemove(key) }}
aria-label={`Remove ${key}`}
title="Remove"
style={{
background: 'none', border: 'none', cursor: 'pointer',
color: 'var(--color-text-muted)', fontSize: '0.625rem', padding: 0,
}}
className="kvchips__chip-remove"
>
<i className="fas fa-times" />
</button>
@@ -64,32 +148,35 @@ export default function KeyValueChips({ pairs, onAdd, onRemove, placeholderKey =
))}
</div>
)}
<div style={{ display: 'flex', gap: 'var(--spacing-xs)', alignItems: 'stretch' }}>
<input
className="input flex-1"
type="text"
placeholder={placeholderKey}
value={k}
onChange={e => setK(e.target.value)}
onKeyDown={onKeyDown}
/>
<input
className="input flex-1"
type="text"
placeholder={placeholderValue}
value={v}
onChange={e => setV(e.target.value)}
onKeyDown={onKeyDown}
/>
<div className="kvchips__row" ref={rowRef}>
<input {...inputProps('key', k, setK, placeholderKey, ariaLabelKey)} />
<input {...inputProps('value', v, setV, placeholderValue, ariaLabelValue)} />
<button
type="button"
className="btn btn-secondary btn-sm"
className="btn btn-secondary btn-sm kvchips__add"
onClick={add}
disabled={!k.trim()}
style={{ minHeight: 36 }}
aria-label={addLabel}
>
<i className="fas fa-plus" /> Add
</button>
{options.length > 0 && (
<ul className="kvchips__suggestions" id={listId} role="listbox" data-testid="label-suggestions">
{options.map((option, index) => (
<li key={option} role="option" aria-selected={index === active}>
<button
type="button"
className={`kvchips__suggestion${index === active ? ' kvchips__suggestion--active' : ''}`}
// mousedown, not click: the input's blur would otherwise
// close the list before the click ever lands.
onMouseDown={(e) => { e.preventDefault(); pick(openField, option) }}
>
{option}
</button>
</li>
))}
</ul>
)}
</div>
</div>
)
core/http/react-ui/src/hooks/useRecommendedModels.js
Vendored
+11 -2
View File
@@ -1,6 +1,7 @@
import { useState, useEffect } from 'react'
import { modelsApi } from '../utils/api'
import { useResources } from './useResources'
import { modelBudget } from '../utils/modelBudget'
// Data-driven "recommended for your hardware" model picks. The gallery exposes
// no popularity/download signal and the list response carries no size, so we:
@@ -21,13 +22,21 @@ const DEFAULT_CTX = 4096
export const isNvfp4Name = (name) => /nvfp4/i.test(name || '')
export function hasNvidiaGpu(resources) {
// A distributed controller has no GPUs of its own, so the question is
// whether any worker does. The registry reports the cluster's best node,
// and it is that node these picks have to run on.
if (resources?.cluster?.enabled) return !!resources.cluster.is_gpu
return Array.isArray(resources?.gpus) &&
resources.gpus.some(g => (g?.vendor || '').toLowerCase() === 'nvidia')
}
export function recommendTier(resources) {
const isGpu = resources?.type === 'gpu'
const vram = resources?.aggregate?.total_memory || 0
// Same reading the models page sizes against: the cluster's largest node in
// distributed mode, the local host otherwise. Ranked against the controller,
// a fleet of A100s was recommended the models a GPU-less pod could run.
const budget = modelBudget(resources)
const isGpu = budget.scope === 'cluster' ? budget.hasGpu : resources?.type === 'gpu'
const vram = budget.totalMemory
if (!isGpu || vram <= 0) return { id: 'cpu', vram: 0 }
if (vram < 8 * GB) return { id: 'gpu-small', vram }
if (vram < 24 * GB) return { id: 'gpu-mid', vram }
core/http/react-ui/src/pages/Chat.jsx
+26 -2
View File
@@ -21,6 +21,8 @@ import { useOperations } from '../hooks/useOperations'
import { relativeTime } from '../utils/format'
import { copyToClipboard } from '../utils/clipboard'
const FOCUS_MODE_KEY = 'localai_chat_focus_mode'
function getLastMessagePreview(chat) {
if (!chat.history || chat.history.length === 0) return ''
for (let i = chat.history.length - 1; i >= 0; i--) {
@@ -405,12 +407,20 @@ export default function Chat() {
// Focus mode: once a conversation has at least one message we slim the
// surrounding chrome (collapse the global app rail, fade non-essential
// header items). Esc gives the user back the full chrome for the rest of
// this session.
// this session. The settings drawer offers a persistent opt-out.
const isInConversation = (activeChat?.history?.length || 0) > 0
const [focusOverride, setFocusOverride] = useState(false)
const focusActive = isInConversation && !focusOverride
const [focusModeEnabled, setFocusModeEnabled] = useState(() => {
try { return localStorage.getItem(FOCUS_MODE_KEY) !== 'false' } catch (_) { return true }
})
const focusActive = focusModeEnabled && isInConversation && !focusOverride
const prevAppCollapseRef = useRef(null)
const toggleFocusMode = (next) => {
setFocusModeEnabled(next)
try { localStorage.setItem(FOCUS_MODE_KEY, String(next)) } catch (_) {}
}
const artifacts = useMemo(
() => canvasMode ? extractCodeArtifacts(activeChat?.history, 'role', 'assistant') : [],
[activeChat?.history, canvasMode]
@@ -1110,6 +1120,20 @@ export default function Chat() {
/>
</div>
)}
<div className="form-group chat-settings-toggle-row">
<div className="chat-settings-toggle-text">
<span className="chat-settings-toggle-title">
<i className="fas fa-compress" /> {t('settings.focusMode')}
</span>
<span className="chat-settings-toggle-desc">
{t('settings.focusModeDesc')}
</span>
</div>
<Toggle
checked={focusModeEnabled}
onChange={toggleFocusMode}
/>
</div>
<div className="form-group">
<label className="form-label">{t('settings.systemPrompt')}</label>
<textarea
core/http/react-ui/src/pages/Models.jsx
+25 -11
View File
@@ -7,6 +7,7 @@ import { safeHref } from '../utils/url'
import { useDebouncedCallback } from '../hooks/useDebounce'
import { useOperations } from '../hooks/useOperations'
import { useResources } from '../hooks/useResources'
import { modelBudget } from '../utils/modelBudget'
import SearchableSelect from '../components/SearchableSelect'
import PageHeader from '../components/PageHeader'
import GalleryLoader from '../components/GalleryLoader'
@@ -208,12 +209,13 @@ export default function Models() {
const [useCaseOpen, setUseCaseOpen] = useState(false)
// Rail groups the user has folded away.
const [collapsedGroups, setCollapsedGroups] = useState(() => new Set())
// Total GPU memory for "fits" check
const totalGpuMemory = resources?.aggregate?.total_memory || 0
// gpu_count is 0 and gpus is null on a CPU-only host, where total_memory is
// system RAM. The fits check has always used it either way; only the copy
// has to stop calling it VRAM.
const hasGpu = (resources?.aggregate?.gpu_count || 0) > 0 || (resources?.gpus?.length || 0) > 0
// What every "will it fit" verdict on this page is measured against. In
// distributed mode that is the cluster's largest node rather than the
// controller serving the page, which is usually a GPU-less pod (see
// modelBudget).
const budget = modelBudget(resources)
const totalGpuMemory = budget.totalMemory
const hasGpu = budget.hasGpu
const fetchModels = useCallback(async (params = {}) => {
try {
@@ -865,6 +867,7 @@ export default function Models() {
onPickContext={setContextSize}
totalGpuMemory={totalGpuMemory}
fitsGpu={fitsGpu}
budgetNode={budget.scope === 'cluster' ? budget.nodeName : ''}
installing={isInstalling(selectedName)}
progress={getOperationProgress(selectedName)}
onInstall={handleInstall}
@@ -894,9 +897,16 @@ export default function Models() {
the data did not support. */}
{totalGpuMemory <= 0
? t('shelves.heroNoGpu', { count: stats.total })
: hasGpu
? t('shelves.heroWithGpu', { vram: formatBytes(totalGpuMemory), count: stats.total })
: t('shelves.heroWithRam', { ram: formatBytes(totalGpuMemory), count: stats.total })}
: budget.scope === 'cluster'
// Naming the node is the point: a cluster figure with
// no owner reads as this machine's, which is the very
// confusion the cluster reading exists to end.
? t(budget.nodeCount > 1 ? 'shelves.heroWithCluster' : 'shelves.heroWithNode', {
vram: formatBytes(totalGpuMemory), node: budget.nodeName, nodes: budget.nodeCount, count: stats.total,
})
: hasGpu
? t('shelves.heroWithGpu', { vram: formatBytes(totalGpuMemory), count: stats.total })
: t('shelves.heroWithRam', { ram: formatBytes(totalGpuMemory), count: stats.total })}
</h2>
<p className="zero-pane__text">{t('shelves.heroHint')}</p>
</div>
@@ -1413,7 +1423,7 @@ function VramByContext({ estimate, contextSize, onPickContext, totalGpuMemory, t
// and hands the rest to ModelDetail, which already knows how to render an
// entry's fields and is shared with the per-variant panel.
function DiscoverDetail({
model, estimate, contextSize, onPickContext, totalGpuMemory, fitsGpu,
model, estimate, contextSize, onPickContext, totalGpuMemory, fitsGpu, budgetNode,
installing, progress, onInstall, installedProfile, onOpen, onManage, onBack,
expandedFiles, setExpandedFiles, variantData, variantDetails, onLoadVariantDetail, t,
}) {
@@ -1472,7 +1482,11 @@ function DiscoverDetail({
{ label: t('detail.size'), value: sizeDisplay && sizeDisplay !== '0 B' ? sizeDisplay : '—' },
{ label: t('detail.vramAt', { context: contextLabel }), value: vramBytes ? formatBytes(vramBytes) : '—' },
{
label: t('detail.headroom'),
// Headroom is headroom somewhere. On a distributed controller that
// somewhere is a worker, and an unqualified figure reads as this
// machine's, which is the confusion the cluster reading exists to
// end.
label: budgetNode ? t('detail.headroomOn', { node: budgetNode }) : t('detail.headroom'),
value: headroom === null ? '—' : (headroom < 0 ? '−' : '') + formatBytes(Math.abs(headroom)),
tone: headroom === null ? undefined : headroom < 0 ? 'bad' : 'ok',
},
core/http/react-ui/src/pages/Scheduling.jsx
+80 -133
View File
@@ -1,12 +1,13 @@
import { useState, useEffect, useCallback } from 'react'
import { useOutletContext } from 'react-router-dom'
import { useTranslation } from 'react-i18next'
import { nodesApi } from '../utils/api'
import { nodesApi, modelsApi } from '../utils/api'
import PageHeader from '../components/PageHeader'
import ConfirmDialog from '../components/ConfirmDialog'
import ResponsiveTable from '../components/ResponsiveTable'
import SearchableModelSelect from '../components/SearchableModelSelect'
import KeyValueChips from '../components/nodes/KeyValueChips'
import { labelIndex } from '../utils/nodeLabelSuggestions'
// Numeric input with quick-pick preset chips. Picked over a slider because
// replica counts are exact specs (operator math), not fuzzy estimates. The
@@ -65,7 +66,7 @@ function configMode(config) {
return 'placement'
}
function SchedulingForm({ initialConfig, onSave, onCancel }) {
function SchedulingForm({ initialConfig, onSave, onCancel, labels, aliases }) {
const [mode, setMode] = useState(() => configMode(initialConfig))
const [modelName, setModelName] = useState(initialConfig?.model_name || '')
// Selector is now a chip-builder map instead of a comma-separated string.
@@ -83,6 +84,10 @@ function SchedulingForm({ initialConfig, onSave, onCancel }) {
const [minPrefixMatch, setMinPrefixMatch] = useState(initialConfig?.min_prefix_match ?? 0)
const hasSelector = Object.keys(selector).length > 0
// Aliases are listed in the picker alongside models, tagged with the model
// they resolve to so the two are distinguishable in one flat list.
const aliasHints = Object.fromEntries(Object.entries(aliases || {}).map(([name, target]) => [name, `alias of ${target}`]))
const aliasTarget = (aliases || {})[modelName]
const isValid = () => {
if (!modelName) return false
@@ -158,9 +163,20 @@ function SchedulingForm({ initialConfig, onSave, onCancel }) {
<SearchableModelSelect
value={modelName}
onChange={setModelName}
placeholder="Type to search models, or paste a name..."
placeholder="Type to search models or aliases, or paste a name..."
hints={aliasHints}
/>
)}
{/* An alias is a stable name for whichever model currently serves it,
so a rule on one is a rule on a slot rather than on a model. Say
so at the point of choosing, because the consequence (repointing
the alias carries the rule along) is not visible anywhere else. */}
{aliasTarget && (
<span className="text-meta d-block mt-xs">
<i className="fas fa-link icon-before" aria-hidden="true" />
{modelName} is an alias for {aliasTarget}. This rule applies to whichever model the alias points at, and follows it if you repoint it.
</span>
)}
</div>
<div>
@@ -174,6 +190,10 @@ function SchedulingForm({ initialConfig, onSave, onCancel }) {
placeholderKey="key (e.g. gpu.vendor)"
placeholderValue="value (e.g. nvidia)"
ariaLabel="Node selector"
ariaLabelKey="Selector key"
ariaLabelValue="Selector value"
addLabel="Add selector"
suggestions={labels}
/>
<span className="text-meta d-block mt-xs">
{mode === 'placement'
@@ -285,139 +305,40 @@ function SchedulingForm({ initialConfig, onSave, onCancel }) {
)
}
const INITIAL_NODE_LIMIT = 5
const NODE_LIMIT_STEP = 20
function fuzzyIncludes(text, term) {
if (text.includes(term)) return true
let termIndex = 0
for (const character of text) {
if (character === term[termIndex]) termIndex++
if (termIndex === term.length) return true
}
return false
}
function matchesNode(node, query) {
const terms = query.toLocaleLowerCase().trim().split(/\s+/).filter(Boolean)
if (!terms.length) return true
const labels = Object.entries(node.labels || {})
const haystack = [
node.name,
node.id,
...labels.flatMap(([key, value]) => [key, String(value), `${key}=${value}`]),
].filter(Boolean).join(' ').toLocaleLowerCase()
return terms.every(term => fuzzyIncludes(haystack, term))
}
function NodeLabelReference() {
const [expanded, setExpanded] = useState(true)
const [nodes, setNodes] = useState([])
const [query, setQuery] = useState('')
const [visibleLimit, setVisibleLimit] = useState(INITIAL_NODE_LIMIT)
const [loading, setLoading] = useState(true)
const [error, setError] = useState(false)
const fetchNodes = useCallback(async () => {
setLoading(true)
setError(false)
try {
const data = await nodesApi.list()
setNodes(Array.isArray(data) ? data : [])
} catch {
setError(true)
} finally {
setLoading(false)
}
}, [])
useEffect(() => { fetchNodes() }, [fetchNodes])
const filtered = nodes.filter(node => matchesNode(node, query))
const visible = filtered.slice(0, visibleLimit)
const updateQuery = event => {
setQuery(event.target.value)
setVisibleLimit(INITIAL_NODE_LIMIT)
}
return (
<section className="card scheduling-node-reference" data-testid="node-label-reference">
<button
type="button"
className="scheduling-node-reference__toggle"
aria-expanded={expanded}
aria-controls="scheduling-node-label-content"
onClick={() => setExpanded(value => !value)}
>
<span><i className="fas fa-tags icon-before" aria-hidden="true" />Node labels</span>
<i className={`fas fa-chevron-${expanded ? 'up' : 'down'}`} aria-hidden="true" />
</button>
{expanded && (
<div id="scheduling-node-label-content" className="scheduling-node-reference__content">
<p className="text-note">Browse labels available for node selectors without leaving this page.</p>
{loading ? (
<p className="scheduling-node-message" role="status">Loading node labels…</p>
) : error ? (
<div className="scheduling-node-message" role="alert">
<span>Could not load node labels.</span>
<button type="button" className="btn btn-secondary btn-sm" aria-label="Retry loading node labels" onClick={fetchNodes}>Retry</button>
</div>
) : nodes.length === 0 ? (
<p className="scheduling-node-message">No nodes are available yet.</p>
) : (
<>
<div className="scheduling-node-toolbar">
<input
type="search"
className="input"
aria-label="Search node labels"
placeholder="Search node, label, or key=value…"
value={query}
onChange={updateQuery}
/>
<span className="text-meta" aria-live="polite">{Math.min(visibleLimit, filtered.length)} of {filtered.length} nodes</span>
</div>
{filtered.length === 0 ? (
<p className="scheduling-node-message">No nodes match your search.</p>
) : (
<div className="scheduling-node-grid">
{visible.map(node => {
const labels = Object.entries(node.labels || {})
return (
<article className="scheduling-node-card" key={node.id || node.name}>
<div className="scheduling-node-card__header">
<strong>{node.name || node.id}</strong>
<span className={`scheduling-node-status scheduling-node-status--${String(node.status || 'unknown').toLowerCase()}`}>{node.status || 'unknown'}</span>
</div>
{labels.length ? (
<div className="scheduling-node-labels">
{labels.map(([key, value]) => <span className="scheduling-node-label" key={key}>{key}={String(value)}</span>)}
</div>
) : <span className="text-note">No labels</span>}
</article>
)
})}
</div>
)}
{visibleLimit < filtered.length && (
<button type="button" className="btn btn-secondary btn-sm scheduling-show-more" aria-label="Show 20 more nodes" onClick={() => setVisibleLimit(limit => limit + NODE_LIMIT_STEP)}>
Show 20 more
</button>
)}
</>
)}
</div>
)}
</section>
)
}
export default function Scheduling() {
const { addToast } = useOutletContext()
const { t } = useTranslation('admin')
const [schedulingConfigs, setSchedulingConfigs] = useState([])
const [formState, setFormState] = useState(null)
const [confirmDelete, setConfirmDelete] = useState(null)
// The label vocabulary the selector field completes against. A roster that
// will not load costs the admin the hints and nothing else, so the failure
// is swallowed rather than surfaced: the field still commits whatever is
// typed into it.
const [labels, setLabels] = useState(() => labelIndex([]))
// name -> target for every configured alias. Feeds the picker so aliases are
// listed as schedulable names. Failing to load costs the annotation and
// nothing else: an alias typed by hand still resolves server-side.
const [aliases, setAliases] = useState({})
useEffect(() => {
let cancelled = false
modelsApi.listAliases()
.then(data => {
if (cancelled || !Array.isArray(data)) return
setAliases(Object.fromEntries(data.map(a => [a.name, a.target])))
})
.catch(() => {})
return () => { cancelled = true }
}, [])
useEffect(() => {
let cancelled = false
nodesApi.list()
.then(data => { if (!cancelled) setLabels(labelIndex(Array.isArray(data) ? data : [])) })
.catch(() => {})
return () => { cancelled = true }
}, [])
const fetchScheduling = useCallback(async () => {
try {
@@ -453,7 +374,6 @@ export default function Scheduling() {
supporting={t('scheduling.subtitle')}
/>
<div>
<NodeLabelReference />
<button className="btn btn-primary btn-sm mb-md"
onClick={() => setFormState(current => current?.kind === 'add' ? null : { kind: 'add' })}>
<i className="fas fa-plus icon-before" />
@@ -465,6 +385,8 @@ export default function Scheduling() {
initialConfig={formState.kind === 'edit' ? formState.config : undefined}
onSave={handleSave}
onCancel={() => setFormState(null)}
labels={labels}
aliases={aliases}
/>
)}
{schedulingConfigs.length === 0 && !formState ? (
@@ -496,9 +418,26 @@ export default function Scheduling() {
// of the model silently failing to scale.
const unsatisfiableUntil = cfg.unsatisfiable_until ? new Date(cfg.unsatisfiable_until) : null
const isUnsatisfiable = unsatisfiableUntil && unsatisfiableUntil.getTime() > Date.now()
// A rule keyed by an alias names a slot, so the model it
// currently governs is worth showing next to it.
const governs = cfg.target_model && cfg.target_model !== cfg.model_name ? cfg.target_model : null
const danglingAlias = cfg.model_is_alias && !governs
return (
<tr key={cfg.id || cfg.model_name}>
<td style={{ fontWeight: 600, fontSize: '0.875rem' }}>{cfg.model_name}</td>
<td style={{ fontWeight: 600, fontSize: '0.875rem' }}>
{cfg.model_name}
{governs && (
<div className="text-meta scheduling-rule-target">
<i className="fas fa-arrow-right icon-before" aria-hidden="true" />
{governs}
</div>
)}
{danglingAlias && (
<div className="text-meta scheduling-rule-target--broken">
alias points at nothing
</div>
)}
</td>
<td>
<span style={{
display: 'inline-block', fontSize: '0.75rem', padding: '2px 8px', borderRadius: "var(--radius-sm)",
@@ -545,7 +484,15 @@ export default function Scheduling() {
) : '-'}
</td>
<td>
{isUnsatisfiable ? (
{cfg.shadowed ? (
<span
className="scheduling-rule-shadowed"
title="Another rule already governs the same model, so this one has no effect. Placement decides where a single shared load runs, so only one rule per model can apply."
>
<i className="fas fa-eye-slash icon-before" />
Shadowed
</span>
) : isUnsatisfiable ? (
<span
title={`Reconciler couldn't satisfy this rule (capacity exhausted). Will retry by ${unsatisfiableUntil.toLocaleString()}, or sooner on a node lifecycle change.`}
style={{
core/http/react-ui/src/utils/modelBudget.js
Vendored
+41
View File
@@ -0,0 +1,41 @@
// modelBudget answers the one question every "will this model run here" verdict
// on the models page is built from: how much memory a model may occupy, and
// whose memory it is.
//
// In distributed mode that is NOT the host serving this page. The controller is
// usually a GPU-less pod while every model runs on a worker, so sizing against
// its own aggregate told admins that a cluster of A100s could only run the
// smallest CPU build. The server reports the cluster's best single node in an
// additional `cluster` block; the local aggregate stays untouched for the
// resource monitor, which is genuinely about this host.
//
// The best single node, not the fleet total: a model loads into one node, so a
// summed fleet of four 16GB cards would promise a 40GB model a home it does not
// have.
//
// Every missing or unusable field falls back to the local reading, so a
// controller that cannot reach its registry keeps behaving exactly as a
// single-node install does.
export function modelBudget(resources) {
const cluster = resources?.cluster
if (cluster?.enabled && cluster.total_memory > 0) {
return {
totalMemory: cluster.total_memory,
hasGpu: !!cluster.is_gpu,
nodeName: cluster.node_name || '',
nodeCount: cluster.node_count || 0,
scope: 'cluster',
}
}
return {
totalMemory: resources?.aggregate?.total_memory || 0,
// gpu_count is 0 and gpus is null on a CPU-only host, where total_memory is
// system RAM. The fits check has always used it either way; only the copy
// has to stop calling it VRAM.
hasGpu: (resources?.aggregate?.gpu_count || 0) > 0 || (resources?.gpus?.length || 0) > 0,
nodeName: '',
nodeCount: 0,
scope: 'local',
}
}
core/http/react-ui/src/utils/modelBudget.test.js
+56
View File
@@ -0,0 +1,56 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { modelBudget } from './modelBudget.js'
const GB = 1024 * 1024 * 1024
test('reports nothing to size against before the first reading arrives', () => {
assert.deepEqual(modelBudget(null), {
totalMemory: 0, hasGpu: false, nodeName: '', nodeCount: 0, scope: 'local',
})
})
test('reports the local aggregate on a single-node host', () => {
assert.deepEqual(
modelBudget({ aggregate: { total_memory: 12 * GB, gpu_count: 1 }, gpus: [{ index: 0 }] }),
{ totalMemory: 12 * GB, hasGpu: true, nodeName: '', nodeCount: 0, scope: 'local' },
)
})
test('treats a CPU-only host as having no GPU', () => {
const budget = modelBudget({ aggregate: { total_memory: 8 * GB, gpu_count: 0 }, gpus: [] })
assert.equal(budget.hasGpu, false)
assert.equal(budget.totalMemory, 8 * GB)
})
// The defect: the controller's own 8GB must not decide what a fleet of A100s
// can run.
test('prefers the cluster reading over the controller it is served from', () => {
assert.deepEqual(
modelBudget({
aggregate: { total_memory: 8 * GB, gpu_count: 0 },
gpus: [],
cluster: { enabled: true, node_name: 'dgx-01', total_memory: 80 * GB, is_gpu: true, node_count: 4 },
}),
{ totalMemory: 80 * GB, hasGpu: true, nodeName: 'dgx-01', nodeCount: 4, scope: 'cluster' },
)
})
test('falls back to the local reading when the cluster reports no usable memory', () => {
const budget = modelBudget({
aggregate: { total_memory: 8 * GB, gpu_count: 0 },
cluster: { enabled: true, node_name: 'dgx-01', total_memory: 0, is_gpu: false, node_count: 0 },
})
assert.equal(budget.scope, 'local')
assert.equal(budget.totalMemory, 8 * GB)
})
test('ignores a cluster block that says distributed mode is off', () => {
const budget = modelBudget({
aggregate: { total_memory: 8 * GB },
cluster: { enabled: false, total_memory: 80 * GB },
})
assert.equal(budget.scope, 'local')
assert.equal(budget.totalMemory, 8 * GB)
})
core/http/react-ui/src/utils/nodeLabelSuggestions.js
Vendored
+52
View File
@@ -0,0 +1,52 @@
// The scheduling page used to browse node labels in a card that stood open
// above the rules whether or not anyone was writing one. Labels are only ever
// needed while filling a rule's node selector, so discovery moved into that
// field: these helpers turn the node roster into what the field offers as the
// user types.
//
// The roster is already fetched for the page, so this costs no request.
// labelIndex reduces the node roster to the label vocabulary the cluster
// actually uses: every distinct key, and the values each key takes.
//
// Nodes carrying no labels are not an error, they simply contribute nothing.
export function labelIndex(nodes) {
const values = {}
for (const node of Array.isArray(nodes) ? nodes : []) {
for (const [key, value] of Object.entries(node?.labels || {})) {
const seen = values[key] || (values[key] = [])
const text = String(value)
if (!seen.includes(text)) seen.push(text)
}
}
for (const key of Object.keys(values)) values[key].sort()
return { keys: Object.keys(values).sort(), values }
}
// rank orders matches so what the user is most likely typing comes first: a
// prefix match beats a match buried in the middle of the string.
function rank(candidates, query) {
const needle = query.trim().toLowerCase()
if (!needle) return candidates
return candidates
.filter(candidate => candidate.toLowerCase().includes(needle))
.sort((a, b) => {
const ap = a.toLowerCase().startsWith(needle)
const bp = b.toLowerCase().startsWith(needle)
if (ap !== bp) return ap ? -1 : 1
return a.localeCompare(b)
})
}
// suggestKeys offers the label keys matching what has been typed, minus the
// ones this selector already carries: re-adding a key would silently overwrite
// the pair the user just built.
export function suggestKeys(index, query, exclude = []) {
return rank(index.keys.filter(key => !exclude.includes(key)), query)
}
// suggestValues offers only the values the key being filled actually takes, so
// a selector cannot be built out of a pair no node in the cluster matches.
export function suggestValues(index, key, query) {
return rank(index.values[key] || [], query)
}
core/http/react-ui/src/utils/nodeLabelSuggestions.test.js
+57
View File
@@ -0,0 +1,57 @@
import assert from 'node:assert/strict'
import test from 'node:test'
import { labelIndex, suggestKeys, suggestValues } from './nodeLabelSuggestions.js'
const NODES = [
{ id: 'n1', name: 'Falcon GPU', labels: { 'gpu.vendor': 'NVIDIA', zone: 'east' } },
{ id: 'n2', name: 'Worker 2', labels: { 'gpu.vendor': 'amd', zone: 'west' } },
{ id: 'n3', name: 'Worker 3', labels: {} },
{ id: 'n4', name: 'Worker 4' },
{ id: 'n5', name: 'Worker 5', labels: { 'gpu.vram': '24GB' } },
]
test('collects every distinct label key across the cluster', () => {
assert.deepEqual(labelIndex(NODES).keys, ['gpu.vendor', 'gpu.vram', 'zone'])
})
test('survives a node list that has not loaded yet', () => {
assert.deepEqual(labelIndex(null), { keys: [], values: {} })
assert.deepEqual(labelIndex([]), { keys: [], values: {} })
})
test('collects the values a key actually takes, deduplicated', () => {
const index = labelIndex([...NODES, { id: 'n6', labels: { zone: 'east' } }])
assert.deepEqual(index.values.zone, ['east', 'west'])
})
test('offers every key before the user has typed anything', () => {
assert.deepEqual(suggestKeys(labelIndex(NODES), ''), ['gpu.vendor', 'gpu.vram', 'zone'])
})
test('matches a key anywhere in the string, ignoring case', () => {
assert.deepEqual(suggestKeys(labelIndex(NODES), 'VEND'), ['gpu.vendor'])
})
test('ranks keys that start with the query above keys that merely contain it', () => {
const index = labelIndex([{ id: 'n1', labels: { 'node.zone': 'a', zone: 'b' } }])
assert.deepEqual(suggestKeys(index, 'zone'), ['zone', 'node.zone'])
})
// A key already in the selector is not a suggestion: adding it again would
// silently overwrite the pair the user just built.
test('drops keys the selector already carries', () => {
assert.deepEqual(suggestKeys(labelIndex(NODES), '', ['gpu.vendor']), ['gpu.vram', 'zone'])
})
test('offers only the values that belong to the key being filled', () => {
assert.deepEqual(suggestValues(labelIndex(NODES), 'gpu.vendor', ''), ['NVIDIA', 'amd'])
})
test('matches a value ignoring case, so the chip keeps the cluster spelling', () => {
assert.deepEqual(suggestValues(labelIndex(NODES), 'gpu.vendor', 'nvi'), ['NVIDIA'])
})
test('offers nothing for a key the cluster has never reported', () => {
assert.deepEqual(suggestValues(labelIndex(NODES), 'made.up', ''), [])
})
core/http/routes/cluster_memory.go
+99
View File
@@ -0,0 +1,99 @@
package routes
import (
"context"
"github.com/mudler/LocalAI/core/application"
"github.com/mudler/LocalAI/core/gallery"
"github.com/mudler/LocalAI/core/http/endpoints/localai"
"github.com/mudler/LocalAI/core/services/nodes"
"github.com/mudler/LocalAI/pkg/system"
"github.com/mudler/xlog"
)
// ClusterMemoryProvider reports the memory budget a model actually gets: the
// largest single healthy node. It is nil in single-node mode, where the local
// host is the only thing worth sizing against.
type ClusterMemoryProvider func(ctx context.Context) (*nodes.ClusterMemory, error)
// ClusterMemoryProviderFor returns the memory source backing every surface that
// answers "will this model fit", or nil in single-node mode.
//
// This is the sibling of ClusterCapabilityProviderFor and exists for the same
// reason. Backend discovery already unions worker capabilities because the
// controller is usually a GPU-less pod; the model gallery asks a second
// question about the same hardware, "how big a model can run here", and
// answering it from the controller's own RAM told admins that a cluster of
// A100s could only run the smallest CPU build.
func ClusterMemoryProviderFor(app *application.Application) ClusterMemoryProvider {
if app == nil || !app.IsDistributed() || app.Distributed().Registry == nil {
return nil
}
return app.Distributed().Registry.HealthyNodeMemory
}
// resolveClusterMemory reads the cluster's memory budget, degrading to no
// reading on error.
//
// Every caller treats a nil reading as "size against the local host exactly as
// before", so a registry hiccup narrows the catalog back to single-node
// behavior rather than marking every model as too large.
func resolveClusterMemory(ctx context.Context, provider ClusterMemoryProvider) *nodes.ClusterMemory {
if provider == nil {
return nil
}
memory, err := provider(ctx)
if err != nil {
xlog.Warn("Could not read cluster memory, sizing models against the local system only", "error", err)
return nil
}
return memory
}
// clusterResourceBlock renders a cluster reading for the API surfaces that
// carry one, or nil when there is nothing to report.
//
// It is an ADDITIONAL field rather than a rewrite of the local aggregate. The
// resource monitor shows the controller's genuine own usage and must keep
// doing so; only the model-sizing surfaces switch to this block, and a client
// that has never heard of it behaves exactly as it did before.
func clusterResourceBlock(memory *nodes.ClusterMemory) map[string]any {
if memory == nil {
return nil
}
return map[string]any{
"enabled": true,
"node_id": memory.NodeID,
"node_name": memory.NodeName,
"total_memory": memory.TotalMemory,
"is_gpu": memory.IsGPU,
"node_count": memory.NodeCount,
}
}
// hostModelEnv describes the local host to variant selection.
func hostModelEnv(ctx context.Context, systemState *system.SystemState) gallery.ResolveEnv {
return gallery.HostResolveEnv(ctx, systemState)
}
// clusterModelEnv describes whichever machine actually runs models to variant
// selection: the cluster's best node in distributed mode, the local host
// otherwise.
//
// The two providers are read independently on purpose. A cluster that can
// report its hardware but not a usable memory figure still deserves the
// hardware verdict, so each half falls back on its own.
func clusterModelEnv(ctx context.Context, systemState *system.SystemState, memory ClusterMemoryProvider, capabilities localai.ClusterCapabilityProvider) gallery.ResolveEnv {
reading := resolveClusterMemory(ctx, memory)
caps := localai.ResolveClusterCapabilities(ctx, capabilities)
if reading == nil && len(caps) == 0 {
return hostModelEnv(ctx, systemState)
}
var budget uint64
if reading != nil {
budget = reading.TotalMemory
}
return gallery.ClusterResolveEnv(ctx, systemState, budget, caps)
}
core/http/routes/cluster_memory_internal_test.go
+100
View File
@@ -0,0 +1,100 @@
package routes
import (
"context"
"errors"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/services/nodes"
"github.com/mudler/LocalAI/pkg/system"
)
// Every model-sizing surface on a distributed controller reads the cluster
// through these seams, and each one degrades to the controller's own hardware
// rather than to "nothing fits".
var _ = Describe("cluster memory resolution", func() {
gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 }
reading := &nodes.ClusterMemory{
NodeID: "n-1", NodeName: "dgx-01", TotalMemory: 80 * 1024 * 1024 * 1024,
IsGPU: true, NodeCount: 4,
}
Describe("resolveClusterMemory", func() {
It("reports nothing in single-node mode", func() {
Expect(resolveClusterMemory(context.Background(), nil)).To(BeNil())
})
It("reports the provider's reading", func() {
provider := func(context.Context) (*nodes.ClusterMemory, error) { return reading, nil }
Expect(resolveClusterMemory(context.Background(), provider)).To(Equal(reading))
})
// A registry hiccup must never mark the whole catalog as too large.
It("degrades to no reading when the registry errors", func() {
provider := func(context.Context) (*nodes.ClusterMemory, error) {
return nil, errors.New("connection refused")
}
Expect(resolveClusterMemory(context.Background(), provider)).To(BeNil())
})
})
Describe("clusterResourceBlock", func() {
It("reports nothing to serialize when there is no reading", func() {
Expect(clusterResourceBlock(nil)).To(BeNil())
})
// The node name travels with the number because "fits" is only ever
// meaningful somewhere, and the UI says where.
It("names the node the budget belongs to", func() {
block := clusterResourceBlock(reading)
Expect(block).To(HaveKeyWithValue("enabled", true))
Expect(block).To(HaveKeyWithValue("node_count", 4))
Expect(block).To(HaveKeyWithValue("total_memory", gib(80)))
Expect(block).To(HaveKeyWithValue("node_name", "dgx-01"))
Expect(block).To(HaveKeyWithValue("node_id", "n-1"))
Expect(block).To(HaveKeyWithValue("is_gpu", true))
})
})
Describe("clusterModelEnv", func() {
controller := system.NewCapabilityState("default")
It("describes the controller when no provider is wired", func() {
env := clusterModelEnv(context.Background(), controller, nil, nil)
Expect(env.AvailableMemory).To(Equal(hostModelEnv(context.Background(), controller).AvailableMemory))
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeFalse())
})
It("describes the cluster when both providers answer", func() {
memProvider := func(context.Context) (*nodes.ClusterMemory, error) { return reading, nil }
capProvider := func(context.Context) ([]string, error) {
return []string{"nvidia-cuda-13"}, nil
}
env := clusterModelEnv(context.Background(), controller, memProvider, capProvider)
Expect(env.AvailableMemory).To(Equal(gib(80)))
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
})
// Half an answer is still better than the controller's: a cluster that
// reports hardware but no usable memory keeps the hardware verdict.
It("uses what the cluster could answer when the memory reading is missing", func() {
capProvider := func(context.Context) ([]string, error) {
return []string{"nvidia-cuda-13"}, nil
}
env := clusterModelEnv(context.Background(), controller, nil, capProvider)
Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue())
Expect(env.AvailableMemory).To(Equal(hostModelEnv(context.Background(), controller).AvailableMemory))
})
})
})
core/http/routes/ui_api.go
+27 -3
View File
@@ -123,6 +123,14 @@ func getDirectorySize(path string) (int64, error) {
// RegisterUIAPIRoutes registers JSON API routes for the web UI
func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model.ModelLoader, appConfig *config.ApplicationConfig, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, applicationInstance *application.Application, adminMiddleware echo.MiddlewareFunc) {
// Both are nil in single-node mode, which leaves every surface below
// sizing models against the local host exactly as it always has. In
// distributed mode the models run on the workers, so "how big a model fits"
// and "which hardware can run it" are questions about them, not about this
// usually GPU-less controller.
clusterMemory := ClusterMemoryProviderFor(applicationInstance)
clusterCapabilities := ClusterCapabilityProviderFor(applicationInstance)
// Operations API - Get all current operations (models + backends)
app.GET("/api/operations", func(c echo.Context) error {
processingData, taskTypes := opcache.GetStatus()
@@ -772,7 +780,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
ramInfo, _ := xsysinfo.GetSystemRAMInfo()
return c.JSON(200, map[string]any{
listing := map[string]any{
"models": modelsJSON,
"repositories": appConfig.Galleries,
"allTags": tags,
@@ -788,7 +796,16 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"totalPages": totalPages,
"prevPage": prevPage,
"nextPage": nextPage,
})
}
// The ram* fields above stay the controller's own, so nothing that
// reads them changes meaning; a client sizing models reads this
// instead, and it is absent entirely in single-node mode.
if block := clusterResourceBlock(resolveClusterMemory(c.Request().Context(), clusterMemory)); block != nil {
listing["cluster"] = block
}
return c.JSON(200, listing)
}, adminMiddleware)
// Returns installed models with their capability flags for UI filtering
@@ -1000,7 +1017,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
// The full, unpaginated list: a variant references another gallery
// entry by name and that entry need not be anywhere near this one.
env := gallery.HostResolveEnv(c.Request().Context(), appConfig.SystemState)
env := clusterModelEnv(c.Request().Context(), appConfig.SystemState, clusterMemory, clusterCapabilities)
view, err := gallery.DescribeVariants(models, model, env)
if err != nil {
// A malformed variant list must not break the picker; the entry
@@ -1876,6 +1893,13 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model
"watchdog_interval": watchdogInterval,
}
// An additional field, never a rewrite of the local aggregate above:
// the resource monitor reports this controller's genuine own usage, and
// only the model-sizing surfaces read the cluster block.
if block := clusterResourceBlock(resolveClusterMemory(c.Request().Context(), clusterMemory)); block != nil {
response["cluster"] = block
}
return c.JSON(200, response)
}, adminMiddleware)
core/services/modeladmin/config.go
+11 -8
View File
@@ -177,13 +177,17 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[
if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil {
return fmt.Errorf("reload configs: %w", err)
}
loaded, ok := s.Loader.GetModelConfig(updated.Name)
if !ok {
if _, ok := s.Loader.GetModelConfig(updated.Name); !ok {
return fmt.Errorf("reload configs: model %q missing", updated.Name)
}
revision, err := config.ModelConfigRevision(&loaded)
// Resolve the revision the way an inference request does. Hashing the
// stored config instead publishes a value no request will ever carry,
// because SetDefaults runs again on the request path and is not
// idempotent for every model, and the edit would leave the model
// unroutable.
revision, err := s.Loader.RevisionFor(updated.Name, s.AppConfig)
if err != nil {
return fmt.Errorf("compute config revision: %w", err)
return err
}
_ = s.Loader.Preload(s.modelsPath())
pending, err := s.applyRevision(ctx, name, updated.Name, revision, updated.IsDisabled())
@@ -343,13 +347,12 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
if err := s.Loader.LoadModelConfigsFromPath(modelsPath, s.AppConfig.ToConfigLoaderOptions()...); err != nil {
return fmt.Errorf("reload configs: %w", err)
}
loaded, ok := s.Loader.GetModelConfig(req.Name)
if !ok {
if _, ok := s.Loader.GetModelConfig(req.Name); !ok {
return fmt.Errorf("reload configs: model %q missing", req.Name)
}
revision, err := config.ModelConfigRevision(&loaded)
revision, err := s.Loader.RevisionFor(req.Name, s.AppConfig)
if err != nil {
return fmt.Errorf("compute config revision: %w", err)
return err
}
if err := s.Loader.Preload(modelsPath); err != nil {
return fmt.Errorf("preload after edit: %w", err)
core/services/modeladmin/modeladmin_suite_test.go
+8
View File
@@ -2,6 +2,7 @@ package modeladmin
import (
"testing"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
@@ -9,5 +10,12 @@ import (
func TestModelAdmin(t *testing.T) {
RegisterFailHandler(Fail)
// Several specs in this suite coordinate goroutines through
// Eventually/Consistently on unbuffered-ish channels (e.g. the
// blockingRevisionLifecycle helper). Gomega's 1s default timeout can be
// too tight on slower or loaded CI runners (notably macOS runners),
// causing spurious "Timed out after 1.005s" failures even though the
// goroutines eventually make progress. Give them more headroom.
SetDefaultEventuallyTimeout(5 * time.Second)
RunSpecs(t, "modeladmin test suite")
}
core/services/modeladmin/remote_sync.go
+5 -11
View File
@@ -49,9 +49,9 @@ func applyRemoteChange(ctx context.Context, cl *config.ModelConfigLoader, models
disabled := true
if exists {
var err error
revision, err = config.ModelConfigRevision(&cfg)
revision, err = authoritative.RevisionForPath(name, modelsPath, opts...)
if err != nil {
return fmt.Errorf("compute authoritative model config revision for %q: %w", name, err)
return fmt.Errorf("resolve authoritative model config revision for %q: %w", name, err)
}
disabled = cfg.IsDisabled()
}
@@ -83,15 +83,9 @@ func changedConfigNames(current, snapshot map[string]config.ModelConfig, named s
changed[name] = struct{}{}
continue
}
previousRevision, err := config.ModelConfigRevision(&previous)
if err != nil {
return nil, fmt.Errorf("compute current model config revision for %q: %w", name, err)
}
revision, err := config.ModelConfigRevision(&cfg)
if err != nil {
return nil, fmt.Errorf("compute authoritative model config revision for %q: %w", name, err)
}
if previousRevision != revision {
// Both sides come from a loader, so both carry the revision stamped
// when their file was parsed. Comparing the stamps compares the files.
if previous.PersistedConfigRevision() != cfg.PersistedConfigRevision() {
changed[name] = struct{}{}
}
}
core/services/modeladmin/remote_sync_test.go
+7 -7
View File
@@ -58,9 +58,9 @@ var _ = Describe("ApplyRemoteChange", func() {
Expect(ApplyRemoteChange(context.Background(), loader, dir, evt, lifecycle)).To(Succeed())
Expect(ApplyRemoteChange(context.Background(), loader, dir, evt, lifecycle)).To(Succeed())
Expect(lifecycle.calls).To(HaveLen(2))
loaded, ok := loader.GetModelConfig("peer-alias")
_, ok := loader.GetModelConfig("peer-alias")
Expect(ok).To(BeTrue())
revision, err := config.ModelConfigRevision(&loaded)
revision, err := loader.RevisionForPath("peer-alias", dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.calls[0].revision).To(Equal(revision))
Expect(lifecycle.calls[1].revision).To(Equal(revision))
@@ -84,7 +84,7 @@ var _ = Describe("ApplyRemoteChange", func() {
loaded, ok := loader.GetModelConfig("peer-alias")
Expect(ok).To(BeTrue())
Expect(loaded.ContextSize).To(HaveValue(Equal(10000)))
revision, err := config.ModelConfigRevision(&loaded)
revision, err := loader.RevisionForPath(loaded.Name, dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.calls).To(HaveLen(3))
Expect(lifecycle.calls[1].revision).To(Equal(revision))
@@ -100,7 +100,7 @@ var _ = Describe("ApplyRemoteChange", func() {
loaded, ok := loader.GetModelConfig("reinstalled")
Expect(ok).To(BeTrue())
revision, err := config.ModelConfigRevision(&loaded)
revision, err := loader.RevisionForPath(loaded.Name, dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.calls).To(HaveLen(1))
Expect(lifecycle.calls[0].revision).To(Equal(revision))
@@ -172,7 +172,7 @@ var _ = Describe("ApplyRemoteChange", func() {
Expect(loaded.ContextSize).To(HaveValue(Equal(10000)))
_, ok = loader.GetModelConfig("deleted")
Expect(ok).To(BeFalse())
changedRevision, err := config.ModelConfigRevision(&loaded)
changedRevision, err := loader.RevisionForPath(loaded.Name, dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.calls).To(ConsistOf(
revisionLifecycleCall{oldName: "changed", newName: "changed", revision: changedRevision},
@@ -228,7 +228,7 @@ var _ = Describe("ApplyRemoteChange", func() {
loaded, ok := loader.GetModelConfig("ordered")
Expect(ok).To(BeTrue())
Expect(loaded.ContextSize).To(HaveValue(Equal(10000)))
revision, err := config.ModelConfigRevision(&loaded)
revision, err := loader.RevisionForPath(loaded.Name, dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.revisions()).To(HaveLen(2))
Expect(lifecycle.revisions()[1]).To(Equal(revision))
@@ -263,7 +263,7 @@ var _ = Describe("ApplyRemoteChange", func() {
Expect(ok).To(BeTrue())
Expect(loaded.ContextSize).To(HaveValue(Equal(10000)))
Expect(readMap(filepath.Join(dir, "ordered.yaml"))).To(HaveKeyWithValue("context_size", 10000))
revision, err := config.ModelConfigRevision(&loaded)
revision, err := loader.RevisionForPath(loaded.Name, dir)
Expect(err).ToNot(HaveOccurred())
Expect(lifecycle.revisions()).To(HaveLen(2))
Expect(lifecycle.revisions()[1]).To(Equal(revision))
core/services/modeladmin/revision_agreement_test.go
+102
View File
@@ -0,0 +1,102 @@
package modeladmin
import (
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/system"
)
// A model's revision is published by administration and checked against on
// every inference request. Those were computed by different code, and each time
// they drifted the model became unroutable until someone deleted the row by
// hand: the request path resolves through the loader, while publishers hashed
// whatever ModelConfig they were holding, which by then had SetDefaults applied.
//
// There is now one resolver, ModelConfigLoader.RevisionFor, and the raw hash is
// unexported so a new publisher cannot reintroduce the split. This pins the
// property that mattered: whatever a publisher writes is what a request brings.
var _ = Describe("Published and requested revisions agree", func() {
var (
dir string
appConfig *config.ApplicationConfig
loader *config.ModelConfigLoader
)
// Several shapes, because the divergence only ever showed up on configs
// rich enough for SetDefaults to change something: a model file to guess
// from, several derived usecases, explicit options.
models := map[string]string{
"plain": "name: plain\nbackend: llama-cpp\nparameters:\n model: plain.gguf\n",
"multimodal": "name: multimodal\nbackend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\nmmproj: mm/mmproj.gguf\noptions:\n - use_jinja:true\n - parallel:2\nparameters:\n model: mm/model.gguf\n",
"auto-ctx": "name: auto-ctx\nbackend: llama-cpp\ncontext_size: -1\nparameters:\n model: auto.gguf\n",
"no-backend": "name: no-backend\nparameters:\n model: bare.gguf\n",
"with-thread": "name: with-thread\nbackend: llama-cpp\nthreads: 3\nparameters:\n model: t.gguf\n",
}
BeforeEach(func() {
dir = GinkgoT().TempDir()
for name, body := range models {
Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed())
}
appConfig = config.NewApplicationConfig()
appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
appConfig.Threads = 8
loader = config.NewModelConfigLoader(dir)
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
})
// requestRevision mirrors what core/backend.ModelOptions forwards to the
// router: the stamp on the config the request pipeline resolved.
requestRevision := func(name string) string {
cfg, err := loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig)
Expect(err).ToNot(HaveOccurred())
return cfg.PersistedConfigRevision()
}
It("resolves the same revision a request will carry, for every model shape", func() {
for name := range models {
published, err := loader.RevisionFor(name, appConfig)
Expect(err).ToNot(HaveOccurred(), "model %s", name)
Expect(published).To(Equal(requestRevision(name)), "model %s: publisher and request disagree", name)
}
})
It("resolves the same revision through the path-based form", func() {
for name := range models {
byAppConfig, err := loader.RevisionFor(name, appConfig)
Expect(err).ToNot(HaveOccurred())
byPath, err := loader.RevisionForPath(name, dir, appConfig.ToConfigLoaderOptions()...)
Expect(err).ToNot(HaveOccurred())
Expect(byPath).To(Equal(byAppConfig), "model %s", name)
}
})
It("does not move when the app-level defaults change", func() {
before := map[string]string{}
for name := range models {
r, err := loader.RevisionFor(name, appConfig)
Expect(err).ToNot(HaveOccurred())
before[name] = r
}
other := config.NewApplicationConfig()
other.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
other.Threads = 1
other.F16 = true
other.ContextSize = 4096
fresh := config.NewModelConfigLoader(dir)
Expect(fresh.LoadModelConfigsFromPath(dir, other.ToConfigLoaderOptions()...)).To(Succeed())
for name := range models {
r, err := fresh.RevisionFor(name, other)
Expect(err).ToNot(HaveOccurred())
Expect(r).To(Equal(before[name]),
"model %s: changing an app-level setting must not make every model unroutable", name)
}
})
})
core/services/modeladmin/revision_resync.go
+133
View File
@@ -0,0 +1,133 @@
package modeladmin
import (
"context"
"errors"
"fmt"
"github.com/mudler/xlog"
"gorm.io/gorm"
"github.com/mudler/LocalAI/core/config"
)
// ErrNoStoredRevision reports that the controller holds no revision for a
// model, which is the normal state for one that has never been served.
var ErrNoStoredRevision = gorm.ErrRecordNotFound
// RevisionStore is the controller state this resync reads and corrects.
type RevisionStore interface {
GetModelConfigRevision(ctx context.Context, modelName string) (string, error)
ApplyConfigRevisions(ctx context.Context, transitions []ModelRevisionTransition) (int, error)
}
// RevisionReader is the read half, satisfied by the node registry.
type RevisionReader interface {
GetModelConfigRevision(ctx context.Context, modelName string) (string, error)
}
type revisionStore struct {
RevisionReader
lifecycle ModelRevisionLifecycle
}
func (s revisionStore) ApplyConfigRevisions(ctx context.Context, t []ModelRevisionTransition) (int, error) {
return s.lifecycle.ApplyConfigRevisions(ctx, t)
}
// NewRevisionStore pairs the registry that holds the stored revisions with the
// lifecycle that publishes new ones. Returns nil when either half is missing,
// which ResyncModelConfigRevisions treats as "nothing to reconcile".
func NewRevisionStore(reader RevisionReader, lifecycle ModelRevisionLifecycle) RevisionStore {
if reader == nil || lifecycle == nil {
return nil
}
return revisionStore{RevisionReader: reader, lifecycle: lifecycle}
}
// ResyncModelConfigRevisions makes the controller's stored revision for each
// model agree with what this build computes from the configuration on disk.
//
// The stored revision is what every inference request is checked against, but
// nothing ever re-derived it from the persisted configuration: it moved only on
// an edit, a gallery install, or a peer's change broadcast. Any other way for
// the two to diverge left the model permanently unroutable, because an
// inference request may only establish a revision, never replace one. A
// configuration edited while this frontend was down, or a change in what the
// revision is computed over, both landed there, and the only recovery was
// deleting the row by hand.
//
// Running this at startup makes that self-correcting. Only a model whose stored
// revision disagrees is republished, so replicas of models that did not drift
// keep serving: republishing is not free, it quarantines every replica loaded
// under the old revision.
//
// A model with no stored revision is left alone. It has never been served, and
// inventing controller state for it here would quarantine nothing and describe
// a model that may never be requested.
func ResyncModelConfigRevisions(ctx context.Context, loader *config.ModelConfigLoader, appConfig *config.ApplicationConfig, store RevisionStore) error {
if loader == nil || store == nil || appConfig == nil {
return nil
}
configs := loader.GetAllModelsConfigs()
if len(configs) == 0 {
// Reconciling nothing is indistinguishable from reconciling correctly,
// which is how a caller that ran this before the configs were loaded
// went unnoticed. Say so rather than report success.
xlog.Warn("Skipping model config revision resync: no model configurations are loaded")
return nil
}
var transitions []ModelRevisionTransition
for _, cfg := range configs {
// Resolve the revision the way an inference request does, through the
// loader, rather than hashing the stored config directly. SetDefaults
// is applied again on that path and is not idempotent for every model
// (it re-runs the GGUF guess and hardware defaults), so hashing the
// stored config yields a value no request will ever carry, and
// publishing it would wedge the model this resync exists to unwedge.
want, err := loader.RevisionFor(cfg.Name, appConfig)
if err != nil {
return err
}
stored, err := store.GetModelConfigRevision(ctx, cfg.Name)
if errors.Is(err, ErrNoStoredRevision) {
continue
}
if err != nil {
return fmt.Errorf("read stored config revision for %q: %w", cfg.Name, err)
}
if stored == want {
continue
}
xlog.Warn("Stored model config revision disagrees with the configuration on disk, republishing",
"model", cfg.Name, "stored", shortRevision(stored), "computed", shortRevision(want))
transitions = append(transitions, ModelRevisionTransition{
ModelName: cfg.Name, ConfigRevision: want, Disabled: cfg.IsDisabled(),
})
}
if len(transitions) == 0 {
return nil
}
if _, err := store.ApplyConfigRevisions(ctx, transitions); err != nil {
return fmt.Errorf("republish model config revisions: %w", err)
}
xlog.Info("Republished model config revisions to match the configuration on disk", "models", len(transitions))
return nil
}
// shortRevision trims a revision for log output; the leading bytes identify it
// well enough to tell two apart.
func shortRevision(revision string) string {
if revision == "" {
return "(none)"
}
if len(revision) > 12 {
return revision[:12]
}
return revision
}
core/services/modeladmin/revision_resync_test.go
+160
View File
@@ -0,0 +1,160 @@
package modeladmin
import (
"context"
"errors"
"os"
"path/filepath"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/system"
)
// stubRevisionStore stands in for the controller's stored revisions.
type stubRevisionStore struct {
stored map[string]string
getErr error
applied []ModelRevisionTransition
applyEr error
}
func (s *stubRevisionStore) GetModelConfigRevision(_ context.Context, name string) (string, error) {
if s.getErr != nil {
return "", s.getErr
}
rev, ok := s.stored[name]
if !ok {
return "", ErrNoStoredRevision
}
return rev, nil
}
func (s *stubRevisionStore) ApplyConfigRevisions(_ context.Context, t []ModelRevisionTransition) (int, error) {
s.applied = append(s.applied, t...)
return 0, s.applyEr
}
// The controller pins a model's replicas to a stored revision and rejects any
// request carrying a different one. Nothing ever re-derived that stored value
// from the configuration on disk: it only moved on an edit, a gallery install
// or a peer's change event. So whenever the stored value stopped matching what
// this build computes for an unchanged file, every request for that model was
// rejected until an operator deleted the row by hand.
var _ = Describe("ResyncModelConfigRevisions", func() {
var (
dir string
loader *config.ModelConfigLoader
store *stubRevisionStore
appConfig *config.ApplicationConfig
)
write := func(name, body string) {
Expect(os.WriteFile(filepath.Join(dir, name+".yaml"), []byte(body), 0o600)).To(Succeed())
}
// revisionOf resolves the revision the way an inference request does, which
// is the value the resync must publish.
revisionOf := func(name string) string {
cfg, err := loader.LoadModelConfigFileByNameDefaultOptions(name, appConfig)
Expect(err).ToNot(HaveOccurred())
Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty())
return cfg.PersistedConfigRevision()
}
BeforeEach(func() {
dir = GinkgoT().TempDir()
appConfig = config.NewApplicationConfig()
appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
loader = config.NewModelConfigLoader(dir)
store = &stubRevisionStore{stored: map[string]string{}}
})
load := func() {
Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed())
}
It("republishes the revision when the stored one no longer matches the config on disk", func() {
write("drifted", "name: drifted\nbackend: llama-cpp\ncontext_size: 4096\n")
load()
store.stored["drifted"] = "a-revision-from-an-earlier-build"
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed())
Expect(store.applied).To(HaveLen(1))
Expect(store.applied[0].ModelName).To(Equal("drifted"))
Expect(store.applied[0].ConfigRevision).To(Equal(revisionOf("drifted")))
})
It("leaves a model alone when the stored revision already matches", func() {
write("agreed", "name: agreed\nbackend: llama-cpp\n")
load()
store.stored["agreed"] = revisionOf("agreed")
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed())
Expect(store.applied).To(BeEmpty(), "republishing an unchanged revision would quarantine live replicas for nothing")
})
// A model nobody has served has no stored revision. Creating one here would
// invent controller state for a model that may never be requested; the first
// request establishes it.
It("does not create state for a model that has never been served", func() {
write("never-served", "name: never-served\nbackend: llama-cpp\n")
load()
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed())
Expect(store.applied).To(BeEmpty())
})
It("republishes only the models that actually drifted", func() {
write("drifted", "name: drifted\nbackend: llama-cpp\n")
write("agreed", "name: agreed\nbackend: llama-cpp\ncontext_size: 2048\n")
load()
store.stored["drifted"] = "stale"
store.stored["agreed"] = revisionOf("agreed")
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed())
Expect(store.applied).To(HaveLen(1))
Expect(store.applied[0].ModelName).To(Equal("drifted"))
})
It("reports a store failure instead of continuing silently", func() {
write("drifted", "name: drifted\nbackend: llama-cpp\n")
load()
store.stored["drifted"] = "stale"
store.applyEr = errors.New("database is down")
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed())
})
It("skips a model whose stored revision cannot be read rather than guessing", func() {
write("unreadable", "name: unreadable\nbackend: llama-cpp\n")
load()
store.getErr = errors.New("connection reset")
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).ToNot(Succeed())
Expect(store.applied).To(BeEmpty())
})
})
// Running the resync before the model configs are loaded reconciled nothing
// while reporting success, which is how a mis-ordered startup call went
// unnoticed. An empty loader is now called out instead of looking like a
// clean run.
var _ = Describe("ResyncModelConfigRevisions with nothing loaded", func() {
It("does not touch stored revisions when no configs are loaded", func() {
dir := GinkgoT().TempDir()
loader := config.NewModelConfigLoader(dir)
appConfig := config.NewApplicationConfig()
appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}}
store := &stubRevisionStore{stored: map[string]string{"served-before": "stale"}}
Expect(ResyncModelConfigRevisions(context.Background(), loader, appConfig, store)).To(Succeed())
Expect(store.applied).To(BeEmpty())
})
})
core/services/modeladmin/state.go
+8 -5
View File
@@ -7,7 +7,6 @@ import (
"gopkg.in/yaml.v3"
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/pkg/utils"
)
@@ -61,13 +60,17 @@ func (s *ConfigService) toggleState(ctx context.Context, name string, action Act
if err := s.Loader.LoadModelConfigsFromPath(s.modelsPath(), s.AppConfig.ToConfigLoaderOptions()...); err != nil {
return fmt.Errorf("reload configs: %w", err)
}
loaded, ok := s.Loader.GetModelConfig(name)
if !ok {
if _, ok := s.Loader.GetModelConfig(name); !ok {
return fmt.Errorf("reload configs: model %q missing", name)
}
revision, err := config.ModelConfigRevision(&loaded)
// Resolve the revision the way an inference request does. Hashing the
// stored config instead publishes a value no request will ever carry,
// because SetDefaults runs again on the request path and is not
// idempotent for every model, and the edit would leave the model
// unroutable.
revision, err := s.Loader.RevisionFor(name, s.AppConfig)
if err != nil {
return fmt.Errorf("compute config revision: %w", err)
return err
}
pending, err := s.applyRevision(ctx, name, name, revision, action == ActionDisable)
if err != nil {
core/services/nodes/alias_scheduling.go
+230
View File
@@ -0,0 +1,230 @@
package nodes
import (
"context"
"errors"
"fmt"
"github.com/mudler/xlog"
)
// AliasResolver maps a model name to the name of the model that actually
// serves it: an alias resolves to its target, anything else to itself. The
// second return reports whether the name was an alias.
//
// core/config.ModelConfigLoader implements this. It is an interface here so
// the registry stays testable without building a full config loader.
type AliasResolver interface {
ResolveAliasName(name string) (string, bool)
}
// SetAliasResolver installs the resolver used to map a scheduling rule's model
// name onto the model the rule actually governs. Called once at startup before
// serving. Leaving it unset makes every rule govern its own name, which is the
// behaviour from before rules could be keyed by an alias.
func (r *NodeRegistry) SetAliasResolver(resolver AliasResolver) {
r.aliasResolver.Store(&resolver)
}
// resolveAlias maps a name through the installed resolver, or returns it
// unchanged when no resolver is wired.
func (r *NodeRegistry) resolveAlias(name string) (string, bool) {
p := r.aliasResolver.Load()
if p == nil || *p == nil {
return name, false
}
return (*p).ResolveAliasName(name)
}
// applyTarget fills in the rule's derived TargetModel. Every read path runs a
// rule through this so callers can tell the rule's key (ModelName, the name
// the operator chose) apart from the model it governs (TargetModel).
func (r *NodeRegistry) applyTarget(cfg *ModelSchedulingConfig) {
if cfg == nil {
return
}
cfg.TargetModel, cfg.ModelIsAlias = r.resolveAlias(cfg.ModelName)
}
// GetGoverningScheduling returns the rule that governs a physical model: the
// rule keyed by the model's own name when one exists, otherwise the rule of an
// alias that resolves to it. Returns nil when no rule governs the model.
//
// The reverse lookup exists because rules stay keyed by the name the operator
// chose, so that an alias rule survives repointing the alias, while the router
// only ever sees the resolved model name (request middleware resolves the
// alias long before routing).
func (r *NodeRegistry) GetGoverningScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error) {
if direct, err := r.GetModelScheduling(ctx, modelName); err != nil || direct != nil {
return direct, err
}
rule, err := r.aliasRuleFor(ctx, modelName, "")
if err != nil {
return nil, err
}
return rule, nil
}
// aliasRuleFor returns the oldest alias-keyed rule resolving to targetModel,
// skipping the rule named by exclude. Returns nil when no alias rule resolves
// there.
//
// Several names can resolve to one model, but placement governs a single
// shared load, so exactly one rule can win. Ordering by creation time (then by
// name, so rules written in the same transaction still order the same way)
// makes every frontend and every reconciler tick pick the same rule.
//
// This scans the rule table rather than filtering in SQL, because the mapping
// from a rule's name to the model it governs lives in the config loader, not
// in the database. The scan is bounded by the number of scheduling rules an
// operator has written, not by the number of models in the cluster.
func (r *NodeRegistry) aliasRuleFor(ctx context.Context, targetModel, exclude string) (*ModelSchedulingConfig, error) {
if targetModel == "" {
return nil, nil
}
var configs []ModelSchedulingConfig
if err := r.db.WithContext(ctx).Order("created_at ASC, model_name ASC").Find(&configs).Error; err != nil {
return nil, err
}
for i := range configs {
cfg := &configs[i]
// A rule keyed by the target's own name is the direct rule, not an
// alias rule; callers handle that case with a precedence of its own.
if cfg.ModelName == targetModel || cfg.ModelName == exclude {
continue
}
r.applyTarget(cfg)
if cfg.Target() == targetModel {
return cfg, nil
}
}
return nil, nil
}
// SchedulingConflict reports the name of an existing rule that already governs
// targetModel, or "" when the target is free. exclude is the rule being
// created or edited, which never conflicts with itself.
//
// Placement governs one shared load, so two rules resolving to the same model
// would each claim to decide where it runs. Write paths use this to reject the
// second one instead of leaving the outcome to the tiebreak in
// GetGoverningScheduling.
func (r *NodeRegistry) SchedulingConflict(ctx context.Context, targetModel, exclude string) (string, error) {
if targetModel == "" {
return "", nil
}
if targetModel != exclude {
direct, err := r.GetModelScheduling(ctx, targetModel)
if err != nil {
return "", err
}
if direct != nil {
return direct.ModelName, nil
}
}
rule, err := r.aliasRuleFor(ctx, targetModel, exclude)
if err != nil || rule == nil {
return "", err
}
return rule.ModelName, nil
}
// ResolveRuleTarget returns the model a rule keyed by ruleName would govern,
// and whether ruleName is an alias. A name that is an alias but comes back
// unchanged is one that does not resolve.
func (r *NodeRegistry) ResolveRuleTarget(ruleName string) (string, bool) {
return r.resolveAlias(ruleName)
}
// markShadowed flags every rule that resolves to a model some other rule
// already governs. Placement decides where one shared load runs, so only one
// rule per target can take effect.
//
// The precedence matches GetGoverningScheduling: a rule keyed by the target's
// own name wins, otherwise the oldest rule does. configs is modified in place.
func markShadowed(configs []ModelSchedulingConfig) {
governing := make(map[string]int, len(configs))
for i := range configs {
target := configs[i].Target()
best, seen := governing[target]
if !seen {
governing[target] = i
continue
}
if rulePrecedes(configs[i], configs[best], target) {
governing[target] = i
}
}
for i := range configs {
configs[i].Shadowed = governing[configs[i].Target()] != i
}
}
// rulePrecedes reports whether rule a governs target instead of rule b.
func rulePrecedes(a, b ModelSchedulingConfig, target string) bool {
if (a.ModelName == target) != (b.ModelName == target) {
return a.ModelName == target
}
if !a.CreatedAt.Equal(b.CreatedAt) {
return a.CreatedAt.Before(b.CreatedAt)
}
return a.ModelName < b.ModelName
}
// ErrSchedulingConflict is returned when a rule would govern a model that
// another rule already governs. Callers map it onto a conflict status.
var ErrSchedulingConflict = errors.New("model already has a scheduling rule")
// ValidateSchedulingTarget checks that a rule keyed by ruleName can be written,
// and returns the model it will govern.
//
// It rejects two cases. An alias that does not resolve governs nothing
// loadable, so a rule on it would sit inert forever. And a model that another
// rule already governs cannot take a second one, because placement decides
// where a single shared load runs: two rules would each claim to decide, and
// only one could win.
func (r *NodeRegistry) ValidateSchedulingTarget(ctx context.Context, ruleName string) (string, error) {
target, isAlias := r.resolveAlias(ruleName)
if isAlias && target == ruleName {
return "", fmt.Errorf("%q is an alias that does not resolve to a model: point it at an existing model before giving it a scheduling rule", ruleName)
}
conflict, err := r.SchedulingConflict(ctx, target, ruleName)
if err != nil {
return "", err
}
if conflict != "" {
return "", fmt.Errorf("%w: rule %q already governs model %q, so edit or delete that rule instead of adding a second one", ErrSchedulingConflict, conflict, target)
}
return target, nil
}
// RefreshSchedulingTargets rewrites each rule's stored target_model to match
// the current alias mapping, and returns the number of rows it changed.
//
// Go callers resolve aliases live and never read the stored copy. It exists for
// the eviction guard, which matches rules to loaded replicas in raw SQL inside
// a locking transaction and so cannot resolve an alias itself. Repointing an
// alias therefore reaches that guard one reconciler tick later, which is early
// enough: until then the guard protects the previous target, and the reconciler
// is already reloading the new one.
func (r *NodeRegistry) RefreshSchedulingTargets(ctx context.Context) error {
var configs []ModelSchedulingConfig
if err := r.db.WithContext(ctx).Find(&configs).Error; err != nil {
return err
}
for i := range configs {
stored := configs[i].TargetModel
live, _ := r.resolveAlias(configs[i].ModelName)
if stored == live {
continue
}
if err := r.db.WithContext(ctx).Model(&ModelSchedulingConfig{}).
Where("id = ?", configs[i].ID).
Update("target_model", live).Error; err != nil {
return err
}
xlog.Info("Scheduling rule now governs a different model",
"rule", configs[i].ModelName, "was", stored, "now", live)
}
return nil
}
core/services/nodes/alias_scheduling_test.go
+196
View File
@@ -0,0 +1,196 @@
package nodes
import (
"context"
"runtime"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/services/testutil"
"gorm.io/gorm"
)
// fakeAliasResolver maps alias names to targets from a plain map, standing in
// for the config loader so these specs don't need a model directory.
type fakeAliasResolver struct{ aliases map[string]string }
func (f *fakeAliasResolver) ResolveAliasName(name string) (string, bool) {
target, ok := f.aliases[name]
if !ok {
return name, false
}
return target, true
}
var _ = Describe("Alias-keyed scheduling rules", func() {
var (
db *gorm.DB
registry *NodeRegistry
resolver *fakeAliasResolver
)
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("testcontainers requires Docker, not available on macOS CI")
}
db = testutil.SetupTestDB()
var err error
registry, err = NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
resolver = &fakeAliasResolver{aliases: map[string]string{"production": "qwen3"}}
registry.SetAliasResolver(resolver)
})
set := func(cfg *ModelSchedulingConfig) {
ExpectWithOffset(1, registry.SetModelScheduling(context.Background(), cfg)).To(Succeed())
}
Describe("resolving a rule to the model it governs", func() {
It("reports the alias target as a rule's target model", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
got, err := registry.GetModelScheduling(context.Background(), "production")
Expect(err).ToNot(HaveOccurred())
Expect(got).ToNot(BeNil())
// The rule keeps the operator's name; only what it governs resolves.
Expect(got.ModelName).To(Equal("production"))
Expect(got.Target()).To(Equal("qwen3"))
})
It("reports a plain model rule as governing itself", func() {
set(&ModelSchedulingConfig{ModelName: "qwen3", MinReplicas: 1})
got, err := registry.GetModelScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(got.Target()).To(Equal("qwen3"))
})
It("resolves targets when listing every rule", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
configs, err := registry.ListModelSchedulings(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(configs).To(HaveLen(1))
Expect(configs[0].Target()).To(Equal("qwen3"))
})
It("resolves targets when listing auto-scaling rules", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
configs, err := registry.ListAutoScalingConfigs(context.Background())
Expect(err).ToNot(HaveOccurred())
Expect(configs).To(HaveLen(1))
Expect(configs[0].Target()).To(Equal("qwen3"))
})
It("governs the new target after the alias is repointed", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
resolver.aliases["production"] = "llama4"
got, err := registry.GetModelScheduling(context.Background(), "production")
Expect(err).ToNot(HaveOccurred())
// The rule did not move: its settings now apply to llama4.
Expect(got.ModelName).To(Equal("production"))
Expect(got.Target()).To(Equal("llama4"))
Expect(got.MinReplicas).To(Equal(2))
})
It("governs its own name when no resolver is installed", func() {
plain, err := NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
Expect(plain.SetModelScheduling(context.Background(), &ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})).To(Succeed())
got, err := plain.GetModelScheduling(context.Background(), "production")
Expect(err).ToNot(HaveOccurred())
Expect(got.Target()).To(Equal("production"))
})
})
Describe("finding the rule that governs a loaded model", func() {
It("finds an alias rule from the model the alias resolves to", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2, NodeSelector: `{"tier":"gpu"}`})
got, err := registry.GetGoverningScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(got).ToNot(BeNil())
Expect(got.ModelName).To(Equal("production"))
Expect(got.NodeSelector).To(Equal(`{"tier":"gpu"}`))
})
It("returns nil when nothing governs the model", func() {
got, err := registry.GetGoverningScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(got).To(BeNil())
})
It("prefers a rule on the model itself over an alias rule", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
set(&ModelSchedulingConfig{ModelName: "qwen3", MinReplicas: 7})
got, err := registry.GetGoverningScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(got.ModelName).To(Equal("qwen3"))
Expect(got.MinReplicas).To(Equal(7))
})
// Two aliases onto one model is a conflict the write paths reject, but
// a config-file edit can still produce it. Whichever rule wins, it must
// be the same one on every frontend and every tick.
It("breaks a two-alias tie deterministically on the older rule", func() {
resolver.aliases["staging"] = "qwen3"
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
set(&ModelSchedulingConfig{ModelName: "staging", MinReplicas: 5})
got, err := registry.GetGoverningScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(got.ModelName).To(Equal("production"))
})
It("stops governing the old target once the alias is repointed", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
resolver.aliases["production"] = "llama4"
gone, err := registry.GetGoverningScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(gone).To(BeNil())
moved, err := registry.GetGoverningScheduling(context.Background(), "llama4")
Expect(err).ToNot(HaveOccurred())
Expect(moved).ToNot(BeNil())
Expect(moved.ModelName).To(Equal("production"))
})
It("ignores an alias rule whose target is a different model", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
got, err := registry.GetGoverningScheduling(context.Background(), "some-other-model")
Expect(err).ToNot(HaveOccurred())
Expect(got).To(BeNil())
})
})
Describe("rules that already resolve to a model", func() {
It("reports the rule already governing a target", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
conflict, err := registry.SchedulingConflict(context.Background(), "qwen3", "")
Expect(err).ToNot(HaveOccurred())
Expect(conflict).To(Equal("production"))
})
It("does not report the rule being edited as its own conflict", func() {
set(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})
conflict, err := registry.SchedulingConflict(context.Background(), "qwen3", "production")
Expect(err).ToNot(HaveOccurred())
Expect(conflict).To(BeEmpty())
})
It("reports no conflict when the target is free", func() {
conflict, err := registry.SchedulingConflict(context.Background(), "qwen3", "")
Expect(err).ToNot(HaveOccurred())
Expect(conflict).To(BeEmpty())
})
})
})
core/services/nodes/cluster_memory.go
+93
View File
@@ -0,0 +1,93 @@
package nodes
import (
"context"
"fmt"
)
// ClusterMemory reports the memory budget a model actually gets in a
// distributed deployment: that of the single largest healthy backend node.
//
// The largest node, not the fleet total. A model loads into one node, so
// summing a fleet of four 16GB cards into 64GB would tell an admin a 40GB
// model fits when no node can ever hold it. Naming the node is part of the
// answer for the same reason: "fits" is only meaningful somewhere.
type ClusterMemory struct {
NodeID string `json:"node_id"`
NodeName string `json:"node_name"`
TotalMemory uint64 `json:"total_memory"`
IsGPU bool `json:"is_gpu"`
NodeCount int `json:"node_count"`
}
// HealthyNodeMemory reports the largest model budget any single healthy backend
// node can offer, or nil when the cluster can answer nothing.
//
// A nil reading is not an error. It means the caller should size against
// whatever it sized against before, which keeps a registry hiccup or an
// empty cluster from marking the entire catalog as too large.
//
// Only healthy backend nodes count, the same predicate the scheduler places
// against, so a drained worker stops advertising hardware the cluster cannot
// currently use.
func (r *NodeRegistry) HealthyNodeMemory(ctx context.Context) (*ClusterMemory, error) {
var nodes []BackendNode
if err := r.db.WithContext(ctx).
Where("status = ? AND node_type = ?", StatusHealthy, NodeTypeBackend).
Find(&nodes).Error; err != nil {
return nil, fmt.Errorf("listing healthy backend node memory: %w", err)
}
var best *ClusterMemory
count := 0
for _, node := range nodes {
budget, isGPU := nodeModelBudget(node)
if budget == 0 {
continue
}
count++
if best == nil || betterBudget(budget, isGPU, best.TotalMemory, best.IsGPU) {
best = &ClusterMemory{
NodeID: node.ID,
NodeName: node.Name,
TotalMemory: budget,
IsGPU: isGPU,
}
}
}
if best == nil {
return nil, nil
}
best.NodeCount = count
return best, nil
}
// nodeModelBudget reports how much memory a model may occupy on one node, the
// per-node form of the same question core/gallery answers for a single host:
// VRAM when the node has a GPU, system RAM otherwise.
//
// An operator-set VRAM budget wins over raw VRAM. The scheduler already refuses
// a load above that ceiling, so sizing against the raw total would advertise a
// fit the cluster then rejects.
func nodeModelBudget(node BackendNode) (uint64, bool) {
if node.TotalVRAM > 0 {
if node.VRAMBudgetBytes > 0 && node.VRAMBudgetBytes < node.TotalVRAM {
return node.VRAMBudgetBytes, true
}
return node.TotalVRAM, true
}
return node.TotalRAM, false
}
// betterBudget ranks one node's budget against the incumbent's.
//
// A GPU node always beats a CPU node, however much system RAM the CPU node
// holds: a 512GB CPU box will serve a 70B model at a speed nobody would pick
// over a 24GB card, so reporting the CPU box as the cluster's capability would
// recommend models the cluster cannot usefully run.
func betterBudget(budget uint64, isGPU bool, bestBudget uint64, bestIsGPU bool) bool {
if isGPU != bestIsGPU {
return isGPU
}
return budget > bestBudget
}
core/services/nodes/interfaces.go
+2
View File
@@ -49,6 +49,7 @@ type ModelRouter interface {
FindLRUModel(ctx context.Context, nodeID string) (*NodeModel, error)
Get(ctx context.Context, nodeID string) (*BackendNode, error)
GetModelScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error)
GetGoverningScheduling(ctx context.Context, modelName string) (*ModelSchedulingConfig, error)
FindNodesBySelector(ctx context.Context, selector map[string]string) ([]BackendNode, error)
FindNodesWithFreeSlot(ctx context.Context, modelName string, candidateNodeIDs []string) ([]BackendNode, error)
NarrowByDiskHeadroom(ctx context.Context, candidateNodeIDs []string, required uint64) ([]string, error)
@@ -60,6 +61,7 @@ type ModelRouter interface {
GetNodeLabels(ctx context.Context, nodeID string) ([]NodeLabel, error)
FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error)
LoadedReplicaStats(ctx context.Context, modelName string, candidateNodeIDs []string) ([]ReplicaCandidate, error)
MarkUnhealthy(ctx context.Context, nodeID string) error
LoadJobStore
}
core/services/nodes/managers_agent_node_test.go
+84
View File
@@ -0,0 +1,84 @@
package nodes
import (
"context"
"runtime"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gorm.io/gorm"
"github.com/mudler/LocalAI/core/services/messaging"
"github.com/mudler/LocalAI/core/services/testutil"
)
// Agent workers do not subscribe to the backend.* subjects, so asking one to
// list its backends can only answer "no responders". ListBackends read that as
// a node that had gone away and marked it unhealthy; the node's next heartbeat
// marked it healthy again. Every poll of the backends view therefore flapped
// every agent node in the cluster, and while it was unhealthy the router would
// not schedule onto it.
var _ = Describe("Backend listing across mixed node types", func() {
var (
db *gorm.DB
registry *NodeRegistry
mc *scriptedMessagingClient
mgr *DistributedBackendManager
ctx context.Context
)
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("testcontainers requires Docker, not available on macOS CI")
}
db = testutil.SetupTestDB()
var err error
registry, err = NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
mc = newScriptedMessagingClient()
mgr = &DistributedBackendManager{
local: stubLocalBackendManager{},
adapter: NewRemoteUnloaderAdapter(nil, mc, 3*time.Minute, 15*time.Minute),
registry: registry,
}
ctx = context.Background()
})
register := func(name, nodeType string) *BackendNode {
node := &BackendNode{Name: name, NodeType: nodeType, Address: name + ":50051"}
Expect(registry.Register(ctx, node, true)).To(Succeed())
fetched, err := registry.GetByName(ctx, name)
Expect(err).ToNot(HaveOccurred())
Expect(fetched.Status).To(Equal(StatusHealthy))
return fetched
}
statusOf := func(id string) string {
n, err := registry.Get(ctx, id)
Expect(err).ToNot(HaveOccurred())
return n.Status
}
It("leaves an agent node healthy instead of flapping it", func() {
agent := register("agent-worker-1", NodeTypeAgent)
mc.scriptNoResponders(messaging.SubjectNodeBackendList(agent.ID))
_, err := mgr.ListBackends()
Expect(err).ToNot(HaveOccurred())
Expect(statusOf(agent.ID)).To(Equal(StatusHealthy),
"an agent node cannot answer backend.list and must not be judged on it")
})
It("still marks a backend node unhealthy when it does not answer", func() {
backendNode := register("worker-a", NodeTypeBackend)
mc.scriptNoResponders(messaging.SubjectNodeBackendList(backendNode.ID))
_, err := mgr.ListBackends()
Expect(err).ToNot(HaveOccurred())
Expect(statusOf(backendNode.ID)).To(Equal(StatusUnhealthy),
"a backend worker that does not answer is genuinely gone")
})
})
core/services/nodes/managers_distributed.go
+11 -2
View File
@@ -331,8 +331,9 @@ func (d *DistributedBackendManager) DeleteBackendDetailed(ctx context.Context, n
// populated from the first node seen so single-node-minded callers still work.
//
// Pending/offline/draining nodes are skipped because they aren't expected to
// answer NATS requests; unhealthy nodes are still queried — ErrNoResponders
// then marks them unhealthy and the loop continues.
// answer NATS requests, and so are non-backend workers, which do not subscribe
// to backend.list at all; unhealthy backend nodes are still queried —
// ErrNoResponders then marks them unhealthy and the loop continues.
func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, error) {
result := make(gallery.SystemBackends)
allNodes, err := d.registry.List(context.Background())
@@ -344,6 +345,14 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro
if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining {
continue
}
// Only backend workers subscribe to backend.list. Asking an agent
// worker can only answer "no responders", which the error handling
// below reads as a node that has gone away, so every poll of this view
// marked every agent node unhealthy and its next heartbeat marked it
// healthy again. The backend-op fan-out skips them for the same reason.
if node.NodeType != "" && node.NodeType != NodeTypeBackend {
continue
}
reply, err := d.adapter.ListBackends(node.ID)
if err != nil {
if errors.Is(err, nats.ErrNoResponders) {
core/services/nodes/model_router_test.go
+7
View File
@@ -128,6 +128,9 @@ func (f *fakeModelRouterForSmartRouter) Get(_ context.Context, nodeID string) (*
func (f *fakeModelRouterForSmartRouter) GetModelScheduling(_ context.Context, _ string) (*ModelSchedulingConfig, error) {
return nil, nil
}
func (f *fakeModelRouterForSmartRouter) GetGoverningScheduling(_ context.Context, _ string) (*ModelSchedulingConfig, error) {
return nil, nil
}
func (f *fakeModelRouterForSmartRouter) FindNodesBySelector(_ context.Context, _ map[string]string) ([]BackendNode, error) {
return nil, nil
}
@@ -250,3 +253,7 @@ var _ = Describe("ModelRouterAdapter", func() {
})
})
})
func (f *fakeModelRouterForSmartRouter) MarkUnhealthy(_ context.Context, _ string) error {
return nil
}
core/services/nodes/reconciler.go
+50 -21
View File
@@ -278,8 +278,9 @@ func (rc *ReplicaReconciler) reconcileOnce(ctx context.Context) {
// reconcileState runs the state-reconciliation passes: drain pending backend
// ops for freshly-healthy nodes, reconcile registry rows against what workers
// report they are running, then port-probe whatever is left. All passes are
// best-effort: a failure on one node doesn't stop the rest.
// report they are running, port-probe whatever is left, then reclaim replica
// slots held by loads nobody is driving. All passes are best-effort: a failure
// on one node doesn't stop the rest.
//
// Order matters. The worker pass runs first and refreshes updated_at for every
// model a worker vouches for, which takes those rows out of the port prober's
@@ -292,6 +293,9 @@ func (rc *ReplicaReconciler) reconcileState(ctx context.Context) {
rc.reconcileNodeProcesses(ctx)
rc.probeLoadedModels(ctx)
rc.sweepLeakedInFlight(ctx)
// Runs last: the passes above can move a row into a serving state, and a
// row that just became loaded is no longer this sweeper's business.
rc.reclaimAbandonedLoads(ctx)
}
// drainPendingBackendOps retries queued backend ops whose next_retry_at has
@@ -779,6 +783,12 @@ func (rc *ReplicaReconciler) pruneProbeFailures(seen map[string]struct{}) {
}
func (rc *ReplicaReconciler) reconcile(ctx context.Context) {
// Keep each rule's stored target in step with the alias mapping. Only the
// eviction guard reads that column, and it cannot resolve aliases itself.
if err := rc.registry.RefreshSchedulingTargets(ctx); err != nil {
xlog.Warn("Reconciler: failed to refresh scheduling targets", "error", err)
}
configs, err := rc.registry.ListAutoScalingConfigs(ctx)
if err != nil {
xlog.Warn("Reconciler: failed to list auto-scaling configs", "error", err)
@@ -830,7 +840,26 @@ func (rc *ReplicaReconciler) candidateNodeIDsForSelector(ctx context.Context, cf
return ids, true
}
// reconcileModel brings one scheduling rule's replica count in line with what
// the rule asks for.
//
// A rule is keyed by the name the operator chose, which may be an alias, while
// the model it governs is cfg.Target(). The two are used deliberately: anything
// that touches a loaded replica (counting, capacity, scheduling, eviction,
// cache pressure) goes through the target, and the rule's own bookkeeping
// columns (the unsatisfiable counter and cooldown) stay keyed by cfg.ModelName.
func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedulingConfig) {
// An alias that resolves to itself is one that no longer resolves at all:
// its target was removed, or it was pointed at another alias. Scheduling it
// would ask a worker to load a pure redirect, which has no backend and no
// model file behind it, so leave the rule alone until the alias is fixed.
if cfg.ModelIsAlias && cfg.Target() == cfg.ModelName {
xlog.Warn("Reconciler: scheduling rule is keyed by an alias that does not resolve; skipping",
"rule", cfg.ModelName)
return
}
target := cfg.Target()
// spread_all: derive a dynamic replica target equal to the number of nodes
// currently matching the selector (all healthy backend nodes when the
// selector is empty). Feeding it through Min==Max==target reuses every
@@ -860,9 +889,9 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
return
}
current, err := rc.registry.CountLoadedReplicas(ctx, cfg.ModelName)
current, err := rc.registry.CountLoadedReplicas(ctx, target)
if err != nil {
xlog.Warn("Reconciler: failed to count replicas", "model", cfg.ModelName, "error", err)
xlog.Warn("Reconciler: failed to count replicas", "model", target, "error", err)
return
}
@@ -873,14 +902,14 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
if cfg.MinReplicas > 0 && int(current) < cfg.MinReplicas {
candidateNodeIDs, selectorMatched := rc.candidateNodeIDsForSelector(ctx, cfg)
if !selectorMatched {
xlog.Warn("Reconciler: no nodes match selector", "model", cfg.ModelName, "selector", cfg.NodeSelector)
xlog.Warn("Reconciler: no nodes match selector", "model", target, "selector", cfg.NodeSelector)
rc.markCapacityProblem(ctx, cfg.ModelName, "no nodes match selector")
return
}
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, cfg.ModelName, candidateNodeIDs)
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, target, candidateNodeIDs)
if capErr != nil {
xlog.Warn("Reconciler: failed to compute cluster capacity", "model", cfg.ModelName, "error", capErr)
xlog.Warn("Reconciler: failed to compute cluster capacity", "model", target, "error", capErr)
return
}
@@ -894,11 +923,11 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
}
// Cap to actual capacity so we don't try harder than possible.
if needed > capacity {
xlog.Info("Reconciler: capping scale-up at cluster capacity", "model", cfg.ModelName,
xlog.Info("Reconciler: capping scale-up at cluster capacity", "model", target,
"need", needed, "capacity", capacity)
needed = capacity
}
xlog.Info("Reconciler: scaling up to meet minimum", "model", cfg.ModelName,
xlog.Info("Reconciler: scaling up to meet minimum", "model", target,
"current", current, "min", cfg.MinReplicas, "adding", needed)
if rc.scaleUp(ctx, cfg, needed) {
// A real (or partial) scale-up clears the hysteresis so a future
@@ -921,19 +950,19 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
// 2. Auto-scale up if all replicas are busy
if current > 0 && (cfg.MaxReplicas == 0 || int(current) < cfg.MaxReplicas) {
if rc.allReplicasBusy(ctx, cfg.ModelName) {
if rc.allReplicasBusy(ctx, target) {
candidateNodeIDs, selectorMatched := rc.candidateNodeIDsForSelector(ctx, cfg)
if !selectorMatched {
return
}
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, cfg.ModelName, candidateNodeIDs)
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, target, candidateNodeIDs)
if capErr != nil || capacity == 0 {
// All busy AND no slot available — burst load above capacity.
// Don't enter cooldown for this case (it's transient demand,
// not a misconfig); the next tick will retry naturally.
return
}
xlog.Info("Reconciler: all replicas busy, scaling up", "model", cfg.ModelName,
xlog.Info("Reconciler: all replicas busy, scaling up", "model", target,
"current", current)
// Only mark the tick as having scaled up if a replica was actually
// added. On a failed scaleUp, leave scaledUp false so the pressure
@@ -954,13 +983,13 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
// Skipped when the busy-burst path already scaled up this tick: at most
// one scaleUp(+1) per tick (see scaledUp above).
if !scaledUp && rc.pressure != nil && current > 0 && (cfg.MaxReplicas == 0 || int(current) < cfg.MaxReplicas) {
if pressureCount := rc.pressure.Count(cfg.ModelName, time.Now()); pressureCount >= rc.pressureThreshold {
if pressureCount := rc.pressure.Count(target, time.Now()); pressureCount >= rc.pressureThreshold {
candidateNodeIDs, selectorMatched := rc.candidateNodeIDsForSelector(ctx, cfg)
if selectorMatched {
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, cfg.ModelName, candidateNodeIDs)
capacity, capErr := rc.registry.ClusterCapacityForModel(ctx, target, candidateNodeIDs)
if capErr == nil && capacity > 0 {
xlog.Info("Reconciler: prefix-cache forced-disturb pressure, scaling up",
"model", cfg.ModelName, "current", current,
"model", target, "current", current,
"pressure", pressureCount,
"threshold", rc.pressureThreshold)
if rc.scaleUp(ctx, cfg, 1) {
@@ -975,7 +1004,7 @@ func (rc *ReplicaReconciler) reconcileModel(ctx context.Context, cfg ModelSchedu
// we preserve the signal so the next tick retries off
// the same accumulated pressure instead of having to
// re-accumulate a full window from scratch.
rc.pressure.Reset(cfg.ModelName)
rc.pressure.Reset(target)
}
}
// No capacity: transient demand, not a misconfig - let the next
@@ -1042,14 +1071,14 @@ func (rc *ReplicaReconciler) scaleUp(ctx context.Context, cfg ModelSchedulingCon
scheduled := 0
for i := 0; i < count; i++ {
node, err := rc.scheduler.ScheduleAndLoadModel(ctx, cfg.ModelName, candidateNodeIDs)
node, err := rc.scheduler.ScheduleAndLoadModel(ctx, cfg.Target(), candidateNodeIDs)
if err != nil {
xlog.Warn("Reconciler: failed to scale up replica", "model", cfg.ModelName,
xlog.Warn("Reconciler: failed to scale up replica", "model", cfg.Target(),
"attempt", i+1, "error", err)
break // stop trying on first failure
}
scheduled++
xlog.Info("Reconciler: scaled up replica", "model", cfg.ModelName, "node", node.Name)
xlog.Info("Reconciler: scaled up replica", "model", cfg.Target(), "node", node.Name)
}
return scheduled > 0
}
@@ -1069,7 +1098,7 @@ func (rc *ReplicaReconciler) scaleDownIdle(ctx context.Context, cfg ModelSchedul
var idleModels []NodeModel
currentModelRevision(rc.registry.db.WithContext(ctx)).
Where("node_models.model_name = ? AND node_models.state = ? AND node_models.in_flight = 0 AND node_models.last_used < ?",
cfg.ModelName, "loaded", cutoff).
cfg.Target(), "loaded", cutoff).
Order("replica_index DESC, last_used ASC").
Find(&idleModels)
@@ -1089,7 +1118,7 @@ func (rc *ReplicaReconciler) scaleDownIdle(ctx context.Context, cfg ModelSchedul
if err := rc.unloader.UnloadModelOnNode(nm.NodeID, nm.ModelName); err != nil {
xlog.Warn("Reconciler: unload failed (model already removed from registry)", "error", err)
}
xlog.Info("Reconciler: scaled down idle replica", "model", cfg.ModelName, "node", nm.NodeID, "replica", nm.ReplicaIndex)
xlog.Info("Reconciler: scaled down idle replica", "model", cfg.Target(), "node", nm.NodeID, "replica", nm.ReplicaIndex)
removed++
}
}
core/services/nodes/reconciler_abandoned_load.go
+113
View File
@@ -0,0 +1,113 @@
package nodes
import (
"context"
"errors"
"time"
"github.com/mudler/xlog"
"gorm.io/gorm"
)
const (
// abandonedLoadGrace is how long a replica row may sit in a pre-serving
// state before the sweeper will consider it at all.
//
// It exists to cover the window between creating the replica row and
// writing the load job that vouches for it. Without it a load could be
// reclaimed in the moment before its own job row exists. It is not the
// thing that protects a long transfer: the job heartbeat does that.
abandonedLoadGrace = 5 * time.Minute
)
// preServingStates are the replica states that hold a slot without being able
// to serve a request. NextFreeReplicaIndex counts every state except
// "unloading", so a row parked in one of these occupies capacity while
// answering nothing.
var preServingStates = []string{"loading", "staging"}
// reclaimAbandonedLoads removes replica rows whose load will never finish.
//
// The other reconciler passes and the router's eviction query all filter
// state = "loaded", and the per-model probe skips rows without an address, so
// nothing reclaimed a row that never got that far. On a node with one replica
// slot per model, a single interrupted transfer made the model unschedulable
// there until an operator intervened: scheduling saw no free slot, and eviction
// found nothing it was allowed to evict.
//
// A row is only reclaimed when something proves the load is not progressing:
// either a load job that has failed or stopped heartbeating, or, for a row with
// no job at all, a node that is no longer healthy.
//
// The no-job case has to be conservative. Only the request path creates load
// jobs; the reconciler's own scale-up loads a replica without one. Treating a
// missing job as proof of abandonment would let this sweeper delete a healthy
// reconciler-driven transfer the moment it ran past the grace period, which for
// a multi-gigabyte checkpoint is every time. A healthy node with no job is
// therefore left alone; when the node is gone, nothing can be progressing and
// the row is safe to reclaim.
func (rc *ReplicaReconciler) reclaimAbandonedLoads(ctx context.Context) {
if rc.db == nil {
return
}
cutoff := time.Now().Add(-abandonedLoadGrace)
var stuck []NodeModel
if err := rc.db.WithContext(ctx).
Where("state IN ? AND updated_at < ?", preServingStates, cutoff).
Find(&stuck).Error; err != nil {
xlog.Warn("Reconciler: failed to list replicas stuck before serving", "error", err)
return
}
now := time.Now()
for _, row := range stuck {
if !rc.loadAbandoned(ctx, row, now) {
continue
}
if err := rc.registry.RemoveNodeModel(ctx, row.NodeID, row.ModelName, row.ReplicaIndex); err != nil {
xlog.Warn("Reconciler: failed to reclaim abandoned load",
"node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex,
"state", row.State, "error", err)
continue
}
xlog.Warn("Reconciler: reclaimed a replica slot held by a load nobody is driving",
"node", row.NodeID, "model", row.ModelName, "replica", row.ReplicaIndex, "state", row.State)
}
}
// loadAbandoned reports whether this row's load has demonstrably stopped.
//
// Every uncertain case answers false. Leaving a slot held for another pass
// costs one scheduling opportunity; reclaiming a row out from under a live
// transfer restarts a multi-gigabyte load and, on a single-slot node, makes the
// model unschedulable there for as long as the retry loop runs.
func (rc *ReplicaReconciler) loadAbandoned(ctx context.Context, row NodeModel, now time.Time) bool {
job, err := rc.registry.GetLoadJob(ctx, row.ModelName)
switch {
case errors.Is(err, gorm.ErrRecordNotFound), err == nil && job == nil:
// No job: only the request path creates them, so this may be a healthy
// reconciler-driven load. Reclaim only once its node is gone.
return !rc.nodeHealthy(ctx, row.NodeID)
case err != nil:
xlog.Warn("Reconciler: cannot read load job, leaving the replica slot held",
"model", row.ModelName, "error", err)
return false
case job.State == LoadJobStateFailed:
return true
default:
return job.IsOrphaned(now)
}
}
// nodeHealthy reports whether the row's node is still healthy. An unreadable
// node counts as healthy so a database blip cannot trigger a reclaim.
func (rc *ReplicaReconciler) nodeHealthy(ctx context.Context, nodeID string) bool {
node, err := rc.registry.Get(ctx, nodeID)
if err != nil || node == nil {
xlog.Warn("Reconciler: cannot read node for a stuck replica, leaving the slot held",
"node", nodeID, "error", err)
return true
}
return node.Status == StatusHealthy
}
core/services/nodes/reconciler_abandoned_load_test.go
+148
View File
@@ -0,0 +1,148 @@
package nodes
import (
"context"
"runtime"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"gorm.io/gorm"
"github.com/mudler/LocalAI/core/services/testutil"
)
// A replica row in loading or staging holds its slot: NextFreeReplicaIndex
// counts every state except unloading. Nothing reclaimed such a row. Every
// reconciler sweep and the router's eviction query filter state = "loaded", and
// the per-model health probe skips rows with no address, which is exactly what a
// row that never finished loading has. So a worker that dropped out mid-transfer
// left a row that pinned the only replica slot on that node for that model, and
// the next request failed with "no replica slot ... all models busy".
//
// Elapsed time alone cannot decide this: staging a large checkpoint legitimately
// runs for tens of minutes. The load job's LastProgress heartbeat is the
// discriminator, the same signal job takeover already trusts.
var _ = Describe("ReplicaReconciler — abandoned load sweeper", func() {
var (
db *gorm.DB
registry *NodeRegistry
node *BackendNode
rc *ReplicaReconciler
)
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("testcontainers requires Docker, not available on macOS CI")
}
db = testutil.SetupTestDB()
var err error
registry, err = NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
node = &BackendNode{Name: "n1", NodeType: NodeTypeBackend, Address: "10.0.0.1:50051"}
Expect(registry.Register(context.Background(), node, true)).To(Succeed())
rc = NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, DB: db})
})
// seedReplica creates a replica row in the given state, aged so it is past
// the sweeper's grace period unless stated otherwise.
seedReplica := func(model, state string, age time.Duration) {
Expect(db.Create(&NodeModel{
ID: model + "-row",
NodeID: node.ID,
ModelName: model,
State: state,
UpdatedAt: time.Now().Add(-age),
}).Error).To(Succeed())
}
seedJob := func(model, state string, sinceProgress time.Duration) {
Expect(db.Create(&ModelLoadJob{
TrackingKey: model,
State: state,
OwnerReplica: "someone",
LastProgress: time.Now().Add(-sinceProgress),
CreatedAt: time.Now().Add(-sinceProgress),
UpdatedAt: time.Now().Add(-sinceProgress),
}).Error).To(Succeed())
}
rowExists := func(model string) bool {
var count int64
Expect(db.Model(&NodeModel{}).Where("model_name = ?", model).Count(&count).Error).To(Succeed())
return count > 0
}
It("reclaims a staging row whose load job has stopped heartbeating", func() {
seedReplica("abandoned", "staging", time.Hour)
seedJob("abandoned", LoadJobStateStaging, 30*time.Minute)
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("abandoned")).To(BeFalse())
})
It("reclaims a jobless row once its node is gone", func() {
seedReplica("orphan", "loading", time.Hour)
Expect(registry.MarkUnhealthy(context.Background(), node.ID)).To(Succeed())
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("orphan")).To(BeFalse())
})
// Only the request path creates load jobs. The reconciler's own scale-up
// loads a replica without one, so treating a missing job as abandonment
// deleted healthy transfers the moment they outran the grace period, which
// for a multi-gigabyte checkpoint is every time. That is what made a replica
// appear to hop between nodes instead of finishing anywhere.
It("keeps a jobless row while its node is still healthy", func() {
seedReplica("scaling-up", "staging", time.Hour)
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("scaling-up")).To(BeTrue(),
"a reconciler-driven load has no job row and must not be reclaimed for it")
})
It("keeps a long transfer whose job is still heartbeating", func() {
// The row itself is old, because staging does not touch it. Only the
// job proves the transfer is alive.
seedReplica("big-model", "staging", time.Hour)
seedJob("big-model", LoadJobStateStaging, time.Second)
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("big-model")).To(BeTrue(), "a live transfer must never be reclaimed")
})
It("leaves a freshly created row alone while its job row is still being written", func() {
seedReplica("just-started", "loading", time.Second)
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("just-started")).To(BeTrue())
})
It("does not touch loaded replicas, which the other sweeps own", func() {
seedReplica("serving", "loaded", time.Hour)
rc.reclaimAbandonedLoads(context.Background())
Expect(rowExists("serving")).To(BeTrue())
})
It("frees the slot so the model can be scheduled on that node again", func() {
seedReplica("wedged", "staging", time.Hour)
seedJob("wedged", LoadJobStateFailed, time.Minute)
_, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1)
Expect(err).To(MatchError(ErrNoFreeSlot), "precondition: the stuck row holds the only slot")
rc.reclaimAbandonedLoads(context.Background())
idx, err := registry.NextFreeReplicaIndex(context.Background(), node.ID, "wedged", 1)
Expect(err).ToNot(HaveOccurred())
Expect(idx).To(Equal(0))
})
})
core/services/nodes/reconciler_alias_test.go
+133
View File
@@ -0,0 +1,133 @@
package nodes
import (
"context"
"runtime"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/mudler/LocalAI/core/services/testutil"
"gorm.io/gorm"
)
var _ = Describe("ReplicaReconciler with alias-keyed rules", func() {
var (
db *gorm.DB
registry *NodeRegistry
resolver *fakeAliasResolver
)
BeforeEach(func() {
if runtime.GOOS == "darwin" {
Skip("testcontainers requires Docker, not available on macOS CI")
}
db = testutil.SetupTestDB()
var err error
registry, err = NewNodeRegistry(db)
Expect(err).ToNot(HaveOccurred())
resolver = &fakeAliasResolver{aliases: map[string]string{"production": "qwen3"}}
registry.SetAliasResolver(resolver)
})
registerNode := func(name, address string) *BackendNode {
node := &BackendNode{
Name: name,
NodeType: NodeTypeBackend,
Address: address,
MaxReplicasPerModel: 4,
}
Expect(registry.Register(context.Background(), node, true)).To(Succeed())
return node
}
setRule := func(cfg *ModelSchedulingConfig) ModelSchedulingConfig {
ExpectWithOffset(1, registry.SetModelScheduling(context.Background(), cfg)).To(Succeed())
return mustGetSched(registry, cfg.ModelName)
}
It("loads the model the alias points at, not the alias itself", func() {
node := registerNode("alias-n1", "10.9.0.1:50051")
rule := setRule(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 1, MaxReplicas: 2})
scheduler := &fakeScheduler{scheduleNode: node}
reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, Scheduler: scheduler, DB: db})
reconciler.reconcileModel(context.Background(), rule)
Expect(scheduler.scheduleCalls).To(HaveLen(1))
Expect(scheduler.scheduleCalls[0].modelName).To(Equal("qwen3"))
})
It("counts the target's replicas when deciding whether the floor is met", func() {
node := registerNode("alias-n2", "10.9.0.2:50051")
Expect(registry.SetNodeModel(context.Background(), node.ID, "qwen3", 0, "loaded", "", 0)).To(Succeed())
rule := setRule(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 1, MaxReplicas: 2})
scheduler := &fakeScheduler{scheduleNode: node}
reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, Scheduler: scheduler, DB: db})
reconciler.reconcileModel(context.Background(), rule)
// The floor is already met by the target's replica. Counting against
// the alias name instead would see zero and load a redundant replica.
Expect(scheduler.scheduleCalls).To(BeEmpty())
})
It("scales down idle replicas of the target", func() {
n1 := registerNode("alias-n3", "10.9.0.3:50051")
n2 := registerNode("alias-n4", "10.9.0.4:50051")
past := time.Now().Add(-10 * time.Minute)
for _, n := range []*BackendNode{n1, n2} {
Expect(registry.SetNodeModel(context.Background(), n.ID, "qwen3", 0, "loaded", "", 0)).To(Succeed())
db.Model(&NodeModel{}).Where("node_id = ? AND model_name = ?", n.ID, "qwen3").Update("last_used", past)
}
rule := setRule(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 1, MaxReplicas: 4})
unloader := &fakeUnloader{}
reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{
Registry: registry, Unloader: unloader, DB: db, ScaleDownDelay: time.Minute,
})
reconciler.reconcileModel(context.Background(), rule)
remaining, err := registry.CountLoadedReplicas(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(remaining).To(BeNumerically("==", 1))
})
It("skips a rule whose alias no longer resolves", func() {
registerNode("alias-n5", "10.9.0.5:50051")
resolver.aliases["orphan"] = "orphan" // target removed: resolves to itself
rule := setRule(&ModelSchedulingConfig{ModelName: "orphan", MinReplicas: 1})
scheduler := &fakeScheduler{}
reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, Scheduler: scheduler, DB: db})
reconciler.reconcileModel(context.Background(), rule)
// Loading the alias name would ask a worker to start a pure redirect
// that has no backend and no model file behind it.
Expect(scheduler.scheduleCalls).To(BeEmpty())
})
It("records unsatisfiable capacity against the rule, not the target", func() {
registerNode("alias-n6", "10.9.0.6:50051")
rule := setRule(&ModelSchedulingConfig{ModelName: "production", MinReplicas: 1, NodeSelector: `{"tier":"absent"}`})
reconciler := NewReplicaReconciler(ReplicaReconcilerOptions{Registry: registry, Scheduler: &fakeScheduler{}, DB: db})
for i := 0; i < unsatisfiableTickThreshold; i++ {
reconciler.reconcileModel(context.Background(), rule)
}
stored, err := registry.GetModelScheduling(context.Background(), "production")
Expect(err).ToNot(HaveOccurred())
Expect(stored.UnsatisfiableUntil).ToNot(BeNil())
// The bookkeeping belongs to the rule row; the target has no rule.
targetRule, err := registry.GetModelScheduling(context.Background(), "qwen3")
Expect(err).ToNot(HaveOccurred())
Expect(targetRule).To(BeNil())
})
})
core/services/nodes/registry.go
+61 -1
View File
@@ -225,6 +225,42 @@ type ModelSchedulingConfig struct {
UnsatisfiableTicks int `gorm:"column:unsatisfiable_ticks;default:0" json:"unsatisfiable_ticks"`
CreatedAt time.Time `json:"created_at"`
UpdatedAt time.Time `json:"updated_at"`
// TargetModel is the model this rule actually governs: ModelName itself,
// or, when ModelName is an alias, the model that alias points at. Callers
// must use Target() for anything that touches a loaded replica (counting,
// capacity, scheduling, eviction) and ModelName for anything that touches
// this rule's own row.
//
// Every read re-derives it from the live alias mapping, so Go callers never
// see a stale value. It is stored as well, purely so the eviction guard in
// evictLRUAndFreeNodeFrom can match a rule to a loaded replica inside its
// locking transaction: that check is raw SQL and cannot resolve an alias.
// RefreshSchedulingTargets rewrites the stored copy on every reconciler
// tick, so repointing an alias reaches the guard within a tick.
TargetModel string `gorm:"column:target_model;size:255" json:"target_model,omitempty"`
// ModelIsAlias reports whether ModelName is an alias rather than a model.
// Also derived on every read. An alias whose TargetModel equals ModelName
// is one that could not be resolved (its target is gone, or points at
// another alias): it governs nothing loadable.
ModelIsAlias bool `gorm:"-" json:"model_is_alias,omitempty"`
// Shadowed reports that another rule already governs this rule's target, so
// this one has no effect. Set only by ListModelSchedulings, which sees every
// rule at once. Write paths reject creating such a pair, but one can still
// arrive from a seed file or from repointing an alias onto a model that
// already has a rule, and an inert rule the operator cannot see is worse
// than one that is labelled.
Shadowed bool `gorm:"-" json:"shadowed,omitempty"`
}
// Target returns the model this rule governs. It falls back to ModelName when
// the rule was built by hand rather than read through the registry, so a rule
// that was never alias-resolved still governs itself.
func (c ModelSchedulingConfig) Target() string {
if c.TargetModel != "" {
return c.TargetModel
}
return c.ModelName
}
// NodeWithExtras extends BackendNode with computed fields for list views.
@@ -323,6 +359,13 @@ type NodeRegistry struct {
// Stored in an atomic.Pointer to an immutable slice so the startup wiring
// (append) and request / reconcile handling (fire) are race-free.
replicaRemovedHooks atomic.Pointer[[]func(modelName, nodeID string, replicaIndex int)]
// aliasResolver maps a scheduling rule's model name onto the model it
// governs, so a rule can be keyed by an alias. Installed once at startup
// (see SetAliasResolver); nil means every rule governs its own name.
// Held in an atomic.Pointer for the same reason as the hooks above: the
// startup wiring writes it while request handling reads it.
aliasResolver atomic.Pointer[AliasResolver]
}
// AddReplicaRemovedHook registers a callback invoked after a replica row for
@@ -404,6 +447,14 @@ func NewNodeRegistry(db *gorm.DB) (*NodeRegistry, error) {
return nil, fmt.Errorf("migrating node tables: %w", err)
}
// Rules written before scheduling rules could be keyed by an alias have no
// stored target. They are all direct rules, so their target is their own
// name, and the eviction guard needs the column filled in to match them.
_ = advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error {
return db.Exec(`UPDATE model_scheduling_configs SET target_model = model_name
WHERE target_model IS NULL OR target_model = ''`).Error
})
// One-shot cleanup of queue rows that can never drain: ops targeted at
// agent workers (wrong subscription set), at non-existent nodes, or with
// an empty backend name. The guard in enqueueAndDrainBackendOp prevents
@@ -1968,13 +2019,14 @@ func (r *NodeRegistry) SetModelScheduling(ctx context.Context, config *ModelSche
if config.ID == "" {
config.ID = uuid.New().String()
}
r.applyTarget(config)
return r.db.WithContext(ctx).
Clauses(clause.OnConflict{
Columns: []clause.Column{{Name: "model_name"}},
DoUpdates: clause.AssignmentColumns([]string{
"node_selector", "min_replicas", "max_replicas", "spread_all",
"route_policy", "balance_abs_threshold", "balance_rel_threshold", "min_prefix_match",
"updated_at",
"target_model", "updated_at",
}),
}).
Create(config).Error
@@ -2004,6 +2056,7 @@ func (r *NodeRegistry) GetModelScheduling(ctx context.Context, modelName string)
if err != nil {
return nil, err
}
r.applyTarget(&config)
return &config, nil
}
@@ -2011,6 +2064,10 @@ func (r *NodeRegistry) GetModelScheduling(ctx context.Context, modelName string)
func (r *NodeRegistry) ListModelSchedulings(ctx context.Context) ([]ModelSchedulingConfig, error) {
var configs []ModelSchedulingConfig
err := r.db.WithContext(ctx).Order("model_name ASC").Find(&configs).Error
for i := range configs {
r.applyTarget(&configs[i])
}
markShadowed(configs)
return configs, err
}
@@ -2018,6 +2075,9 @@ func (r *NodeRegistry) ListModelSchedulings(ctx context.Context) ([]ModelSchedul
func (r *NodeRegistry) ListAutoScalingConfigs(ctx context.Context) ([]ModelSchedulingConfig, error) {
var configs []ModelSchedulingConfig
err := r.db.WithContext(ctx).Where("min_replicas > 0 OR max_replicas > 0 OR spread_all = ?", true).Find(&configs).Error
for i := range configs {
r.applyTarget(&configs[i])
}
return configs, err
}
Loaded 100 of 131 files, more files were not shown because too many files have changed in this diff. Show more