diff --git a/.github/backend-matrix.yml b/.github/backend-matrix.yml index e7bada4b4..3d3b2b804 100644 --- a/.github/backend-matrix.yml +++ b/.github/backend-matrix.yml @@ -3754,6 +3754,19 @@ include: dockerfile: "./backend/Dockerfile.golang" context: "./" ubuntu-version: '2404' + - build-type: 'hipblas' + cuda-major-version: "" + cuda-minor-version: "" + platforms: 'linux/amd64' + tag-latest: 'auto' + tag-suffix: '-gpu-rocm-hipblas-stablediffusion-ggml' + runs-on: 'ubuntu-latest' + base-image: "rocm/dev-ubuntu-24.04:7.2.1" + skip-drivers: 'false' + backend: "stablediffusion-ggml" + dockerfile: "./backend/Dockerfile.golang" + context: "./" + ubuntu-version: '2404' - build-type: 'sycl_f16' cuda-major-version: "" cuda-minor-version: "" diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 17e85e101..cefd0cefd 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -29,10 +29,6 @@ updates: schedule: # Check for updates to GitHub Actions every weekday interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/bark" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/common/template" schedule: @@ -55,30 +51,10 @@ updates: ignore: - dependency-name: "torch" - dependency-name: "transformers" - - package-ecosystem: "pip" - directory: "/backend/python/exllama" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/exllama2" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/mamba" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/openvoice" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/rerankers" schedule: interval: "weekly" - - package-ecosystem: "pip" - directory: "/backend/python/sentencetransformers" - schedule: - interval: "weekly" - package-ecosystem: "pip" directory: "/backend/python/transformers" schedule: @@ -86,44 +62,4 @@ updates: - package-ecosystem: "pip" directory: "/backend/python/vllm" schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/chainlit" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/functions" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain/langchainpy-localai-example" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/langchain-chroma" - schedule: - interval: "weekly" - - package-ecosystem: "pip" - directory: "/examples/streamlit-bot" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/k8sgpt" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/kubernetes" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/langchain" - schedule: - interval: "weekly" - - package-ecosystem: "gomod" - directory: "/examples/semantic-todo" - schedule: - interval: "weekly" - - package-ecosystem: "docker" - directory: "/examples/telegram-bot" - schedule: - interval: "weekly" + interval: "weekly" \ No newline at end of file diff --git a/.github/workflows/notify-releases.yaml b/.github/workflows/notify-releases.yaml index eab8ce54f..8711c4745 100644 --- a/.github/workflows/notify-releases.yaml +++ b/.github/workflows/notify-releases.yaml @@ -31,13 +31,14 @@ jobs: messages: [ { role: "system", - content: "Write a discord message with a bullet point summary of the release notes." + content: "Write a Discord message with a bullet point summary of the release notes. Keep the complete message under 1800 characters." }, { role: "user", content: $input } - ] + ], + max_tokens: 450 }') # Send the request to LocalAI API @@ -46,7 +47,7 @@ jobs: -d "$json_payload") # Extract the summary from the response - summary=$(echo $response | jq -r '.choices[0].message.content') + summary=$(printf '%s' "$response" | jq -er '.choices[0].message.content | strings | .[0:1800]') # Print the summary # -H "Authorization: Bearer $API_KEY" \ diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index feb7ba0a0..837e84b7f 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=4d383be1bff107e823ffc19120dcb6c78d493c0f +AUDIO_CPP_VERSION?=89a0e9803380880305e9e1b83c93614f9df2c893 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 96171e6a9..522791d42 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=9ca265a57f85f2117942490f421f64a226dd9847 +BONSAI_VERSION?=312bb2a93ea2bf798333fa859614fbf913ecb9e2 LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= @@ -41,6 +41,7 @@ define bonsai-build # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:$(1)$(RESET)) @@ -79,6 +80,7 @@ bonsai-cpu-all: # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge + bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET)) diff --git a/backend/cpp/bonsai/patch-grpc-server.sh b/backend/cpp/bonsai/patch-grpc-server.sh new file mode 100644 index 000000000..aa9b23a50 --- /dev/null +++ b/backend/cpp/bonsai/patch-grpc-server.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai. + +set -euo pipefail + +if [[ $# -ne 1 ]]; then + echo "usage: $0 " >&2 + exit 2 +fi + +SRC=$1 +if [[ ! -f "$SRC" ]]; then + echo "grpc-server.cpp not found at $SRC" >&2 + exit 2 +fi + +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the Bonsai JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> Bonsai JSON exception patch OK" +else + echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping" +fi diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index bdf126122..b171fa391 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +# Upstream pin lives below as DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=84cc882352757baf628a1776badf7cc54d584e28 +DS4_VERSION?=8db89fe083ae4d17c9a2428ccd29803d3ae8f577 DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) diff --git a/backend/cpp/ik-llama-cpp/Makefile b/backend/cpp/ik-llama-cpp/Makefile index 847ddbb20..2ee3f30cd 100644 --- a/backend/cpp/ik-llama-cpp/Makefile +++ b/backend/cpp/ik-llama-cpp/Makefile @@ -1,5 +1,5 @@ -IK_LLAMA_VERSION?=8337e4cd3861406fc04e0854b1409cd1b027fbc9 +IK_LLAMA_VERSION?=15dddc60b3fc937a9e2a210359ecce392ccdf446 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 41861f5f2..b1f34d985 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=d59d455fd8ea09e5a2e87ce2a9d668267ffb5ccd +LLAMA_VERSION?=d7bd3bfcad3e29c7e49fd26f38c79ee3e9a3fd6b LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= diff --git a/backend/cpp/llama-cpp/grpc-server.cpp b/backend/cpp/llama-cpp/grpc-server.cpp index 171ae0483..95345aaf1 100644 --- a/backend/cpp/llama-cpp/grpc-server.cpp +++ b/backend/cpp/llama-cpp/grpc-server.cpp @@ -88,6 +88,12 @@ using grpc::ServerBuilder; using grpc::ServerContext; using grpc::Status; +#if LOCALAI_HAS_MTMD_INIT_OPT +#define LOCALAI_MTMD_INIT_OPT_ARG(value) , value +#else +#define LOCALAI_MTMD_INIT_OPT_ARG(value) +#endif + // gRPC bearer token auth for distributed mode. // Reads LOCALAI_GRPC_AUTH_TOKEN from the environment. When set, rejects // requests without a matching "authorization: Bearer " metadata header. @@ -294,7 +300,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const } else { SRV_WRN("[TOOLS DEBUG] parse_options: Parsed tools JSON is not an array: %s\n", tools_json.dump().c_str()); } - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_WRN("Failed to parse tools JSON from proto: %s\n", e.what()); SRV_WRN("[TOOLS DEBUG] parse_options: Tools string that failed to parse: %s\n", predict->tools().c_str()); } @@ -324,7 +330,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const SRV_DBG("[TOOLS DEBUG] Received tool_choice object from Go layer: %s\n", tool_choice_json.dump().c_str()); } SRV_INF("Extracted tool_choice from proto: %s\n", predict->toolchoice().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { // If parsing fails, treat as string data["tool_choice"] = predict->toolchoice(); SRV_INF("Extracted tool_choice as string: %s\n", predict->toolchoice().c_str()); @@ -353,7 +359,7 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const // Add to data - llama.cpp server expects it as an object (map) data["logit_bias"] = logit_bias_json; SRV_INF("Using logit_bias: %s\n", predict->logitbias().c_str()); - } catch (const json::parse_error& e) { + } catch (const common_json_error& e) { SRV_ERR("Failed to parse logit_bias JSON from proto: %s\n", e.what()); } } @@ -398,7 +404,10 @@ json parse_options(bool streaming, const backend::PredictOptions* predict, const }); } - data["stop"] = predict->stopprompts(); + data["stop"] = json::array(); + for (const auto & stop : predict->stopprompts()) { + data["stop"].push_back(stop); + } // data["n_probs"] = predict->nprobs(); //TODO: images, @@ -1116,14 +1125,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt try { int n = std::stoi(optval_str); if (n < 0) n = 0; - // Keep override-name storage alive for the lifetime of the params struct - // (mirrors upstream arg.cpp behavior with a function-local static). +#if LOCALAI_HAS_N_CPU_FFN_HELPER + llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.speculative.draft.tensor_buft_overrides); +#else static std::list buft_overrides_draft; for (int i = 0; i < n; ++i) { buft_overrides_draft.push_back(llm_ffn_exps_block_regex(i)); params.speculative.draft.tensor_buft_overrides.push_back( {buft_overrides_draft.back().c_str(), ggml_backend_cpu_buffer_type()}); } +#endif } catch (...) {} } @@ -1141,14 +1152,16 @@ static void params_parse(server_context& /*ctx_server*/, const backend::ModelOpt try { int n = std::stoi(optval_str); if (n < 0) n = 0; - // Keep override-name storage alive for the lifetime of the - // params struct (mirrors upstream arg.cpp's function-local static). +#if LOCALAI_HAS_N_CPU_FFN_HELPER + llm_add_n_cpu_ffn_overrides(n, LLM_FFN_EXPS_REGEX, params.tensor_buft_overrides); +#else static std::list buft_overrides_main; for (int i = 0; i < n; ++i) { buft_overrides_main.push_back(llm_ffn_exps_block_regex(i)); params.tensor_buft_overrides.push_back( {buft_overrides_main.back().c_str(), ggml_backend_cpu_buffer_type()}); } +#endif } catch (...) {} } @@ -1795,7 +1808,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -1988,7 +2001,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2074,30 +2087,27 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - for (const auto &img : *images_data) + for (const auto &img : data.at("image_data")) { auto decoded_data = base64_decode(img["data"].get()); files.push_back(decoded_data); } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -2111,10 +2121,10 @@ public: std::vector inputs; if (has_mtmd) { // multimodal - inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files)); + inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt))); } else { // Everything else, including multimodal completions. - inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true); + inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); } tasks.reserve(inputs.size()); @@ -2370,7 +2380,7 @@ public: for (int j = 0; j < request->audios_size(); j++) rin.audios.push_back(request->audios(j)); for (int j = 0; j < request->videos_size(); j++) rin.videos.push_back(request->videos(j)); } - messages_json.push_back(llama_grpc::build_reconstructed_message(rin)); + messages_json.push_back(json::parse(llama_grpc::build_reconstructed_message(rin).dump())); } // Final safety check: Ensure no message has null content (Jinja templates require strings) @@ -2563,7 +2573,7 @@ public: if (!body_json.contains("chat_template_kwargs")) { body_json["chat_template_kwargs"] = json::object(); } - for (auto& el : ctk.items()) { + for (auto el : ctk.items()) { body_json["chat_template_kwargs"][el.key()] = el.value(); } } @@ -2649,11 +2659,10 @@ public: // If not using chat templates, extract files from image_data/audio_data fields // (If using chat templates, files were already extracted by oaicompat_chat_params_parse) if (!request->usetokenizertemplate() || request->messages_size() == 0 || ctx_server.impl->chat_params.tmpls == nullptr) { - const auto &images_data = data.find("image_data"); - if (images_data != data.end() && images_data->is_array()) + if (data.contains("image_data") && data.at("image_data").is_array()) { - std::cout << "[PREDICT] Processing " << images_data->size() << " images" << std::endl; - for (const auto &img : *images_data) + std::cout << "[PREDICT] Processing " << data.at("image_data").size() << " images" << std::endl; + for (const auto &img : data.at("image_data")) { std::cout << "[PREDICT] Processing image" << std::endl; auto decoded_data = base64_decode(img["data"].get()); @@ -2661,20 +2670,18 @@ public: } } - const auto &audio_data = data.find("audio_data"); - if (audio_data != data.end() && audio_data->is_array()) + if (data.contains("audio_data") && data.at("audio_data").is_array()) { - for (const auto &audio : *audio_data) + for (const auto &audio : data.at("audio_data")) { auto decoded_data = base64_decode(audio["data"].get()); files.push_back(decoded_data); } } - const auto &video_data = data.find("video_data"); - if (video_data != data.end() && video_data->is_array()) + if (data.contains("video_data") && data.at("video_data").is_array()) { - for (const auto &video : *video_data) + for (const auto &video : data.at("video_data")) { auto decoded_data = base64_decode(video["data"].get()); files.push_back(decoded_data); @@ -2689,10 +2696,10 @@ public: std::vector inputs; if (has_mtmd) { // multimodal - inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files)); + inputs.push_back(process_mtmd_prompt(ctx_server.impl->mctx, prompt_str, files LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt))); } else { // Everything else, including multimodal completions. - inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true); + inputs = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt_str, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); } tasks.reserve(inputs.size()); @@ -2879,7 +2886,7 @@ public: json prompt = body.at("embeddings"); - auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true); + auto tokenized_prompts = tokenize_input_prompts(ctx_server.impl->vocab, ctx_server.impl->mctx, prompt, true, true LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); for (const auto & tokens : tokenized_prompts) { // this check is necessary for models that do not add BOS token to the input if (tokens.empty()) { @@ -2984,7 +2991,7 @@ public: tasks.reserve(documents.size()); for (size_t i = 0; i < documents.size(); i++) { - auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i]); + auto tmp = format_prompt_rerank(ctx_server.impl->model_tgt, ctx_server.impl->vocab, ctx_server.impl->mctx, request->query(), documents[i] LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); server_task task = server_task(SERVER_TASK_TYPE_RERANK); task.id = rd.queue_tasks.get_new_id(); task.index = i; @@ -3005,7 +3012,7 @@ public: } // Collect responses - json responses = json::array(); + std::vector responses; for (auto & res : all_results.results) { GGML_ASSERT(dynamic_cast(res.get()) != nullptr); responses.push_back(res->to_json()); @@ -3018,7 +3025,7 @@ public: // Crop results by request.top_n if specified int top_n = request->top_n(); if (top_n > 0 && top_n < static_cast(responses.size())) { - responses = json(responses.begin(), responses.begin() + top_n); + responses.resize(top_n); } // Set usage information backend::Usage* usage = rerankResult->mutable_usage(); @@ -3065,7 +3072,7 @@ public: return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, opts.error); } - auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false); + auto wrapper = mtmd_helper_bitmap_init_from_file(ctx_server.impl->mctx, opts.voice_path.c_str(), false LOCALAI_MTMD_INIT_OPT_ARG(ctx_server.impl->init_opt)); if (!wrapper.bitmap) { return grpc::Status(grpc::StatusCode::INVALID_ARGUMENT, "failed to read speaker reference audio: " + opts.voice_path); diff --git a/backend/cpp/llama-cpp/message_content.h b/backend/cpp/llama-cpp/message_content.h index 4c7317ecd..0b70c8b56 100644 --- a/backend/cpp/llama-cpp/message_content.h +++ b/backend/cpp/llama-cpp/message_content.h @@ -52,14 +52,15 @@ inline nlohmann::ordered_json normalize_message_content(const std::string& role, // (#7528). A multimodal user message legitimately carries a typed-part array // ({type:text}, {type:image_url}, ...), which must be left intact. Shared by the // streaming and non-streaming paths so this invariant cannot drift between them. -inline void normalize_template_message(nlohmann::ordered_json& msg) { +template +inline void normalize_template_message(Json& msg) { if (!msg.contains("content")) { msg["content"] = ""; // templates expect the field to exist return; } - nlohmann::ordered_json& content = msg["content"]; + auto& content = msg["content"]; const std::string role = (msg.contains("role") && msg["role"].is_string()) - ? msg["role"].get() + ? msg["role"].template get() : std::string(); if (content.is_null()) { content = ""; // #7324: null would crash content[:N] slicing diff --git a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch index f056d47ce..253e8da5f 100644 --- a/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch +++ b/backend/cpp/llama-cpp/patches/0001-add-server-task-type-score.patch @@ -6,10 +6,9 @@ Subject: [PATCH 1/2] score-patch --- common/common.cpp | 6 +- common/common.h | 3 + - tools/CMakeLists.txt | 1 + tools/server/server-context.cpp | 358 +++++++++++++++++++++++++++++++- tools/server/server-task.h | 47 +++++ - 5 files changed, 406 insertions(+), 9 deletions(-) + 4 files changed, 405 insertions(+), 9 deletions(-) diff --git a/common/common.cpp b/common/common.cpp index 2e3f14c..0cec0dc 100644 @@ -42,15 +41,6 @@ index 878534d..4001df2 100644 int32_t n_sequences = 1; // number of sequences to decode int32_t n_outputs_max = 0; // max outputs in a batch (0 = n_batch) int32_t n_outputs_max_per_seq = 1; // max outputs per sequence -diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt -index 780df32..1d2fe8f 100644 ---- a/tools/CMakeLists.txt -+++ b/tools/CMakeLists.txt -@@ -41,3 +41,4 @@ else() - add_subdirectory(fit-params) - add_subdirectory(results) - endif() -+add_subdirectory(grpc-server) diff --git a/tools/server/server-context.cpp b/tools/server/server-context.cpp index 3b5f6a1..d0e18e6 100644 --- a/tools/server/server-context.cpp diff --git a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch index 6681b04e8..6789ffc0d 100644 --- a/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch +++ b/backend/cpp/llama-cpp/patches/0002-add-server-task-type-tts.patch @@ -659,7 +659,7 @@ index 9069463fe..b7fa1e534 100644 + } + + if (speaker_ref_len > 0) { -+ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false); ++ auto wrapper = mtmd_helper_bitmap_init_from_buf(ctx_server.mctx, speaker_ref_data, speaker_ref_len, false, ctx_server.init_opt); + if (!wrapper.bitmap) { + res->error(format_error_response("failed to decode \"speaker_ref\"", ERROR_TYPE_INVALID_REQUEST)); + return res; diff --git a/backend/cpp/llama-cpp/prepare.sh b/backend/cpp/llama-cpp/prepare.sh index e658a940a..e00b139cb 100644 --- a/backend/cpp/llama-cpp/prepare.sh +++ b/backend/cpp/llama-cpp/prepare.sh @@ -61,11 +61,23 @@ if grep -q "server_metrics metrics;" llama.cpp/tools/server/server-task.h; then else HAS_SERVER_METRICS=0 fi +if grep -q "mtmd_helper_init_opt" llama.cpp/tools/mtmd/mtmd-helper.h; then + HAS_MTMD_INIT_OPT=1 +else + HAS_MTMD_INIT_OPT=0 +fi +if grep -q "llm_add_n_cpu_ffn_overrides" llama.cpp/common/common.h; then + HAS_N_CPU_FFN_HELPER=1 +else + HAS_N_CPU_FFN_HELPER=0 +fi cat > llama.cpp/tools/grpc-server/llama_compat.h < LOCALAI_TURBOQUANT_NO_CHECKPOINT_MIN_STEP define OK" fi +# 3. The shared source follows current upstream and catches common_json_error. +# TurboQuant still exposes nlohmann::json directly, so its equivalent parse +# failures use json::parse_error instead. +if grep -q 'common_json_error' "$SRC"; then + echo "==> patching $SRC to use the TurboQuant JSON exception type" + awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" + mv "$SRC.tmp" "$SRC" + echo "==> TurboQuant JSON exception patch OK" +else + echo "==> $SRC already uses a TurboQuant-compatible JSON exception type, skipping" +fi + echo "==> all patches applied" diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 9cf913762..b87b395a7 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=74bb374a8cc74284348d76a0a6e944180fbe6b07 +CRISPASR_VERSION?=ae4474dd8306384a0e697183d863dfc52e69a2fb SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF diff --git a/backend/go/depth-anything-cpp/Makefile b/backend/go/depth-anything-cpp/Makefile index 79a1598e2..14fa58ec9 100644 --- a/backend/go/depth-anything-cpp/Makefile +++ b/backend/go/depth-anything-cpp/Makefile @@ -14,7 +14,7 @@ JOBS?=$(shell nproc --ignore=1) # It is kept alive by the upstream tag da2-support (survives a squash-merge); # repoint to the master merge commit once mudler/depth-anything.cpp PR #1 lands. DEPTHANYTHING_REPO?=https://github.com/mudler/depth-anything.cpp.git -DEPTHANYTHING_VERSION?=54abd5c0abfd1f394e01cb3c38f2e3af4daedf85 +DEPTHANYTHING_VERSION?=739992d10bf9472c46dcd4622b14d2b20766c58d ifeq ($(NATIVE),false) CMAKE_ARGS+=-DGGML_NATIVE=OFF diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 1c1552074..fd82adbb0 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=97d2990807fe6d558e395f8764198d7c7e7b411c +STABLEDIFFUSION_GGML_VERSION?=be0e34480dada95f8ce9a021bbb95c5de85d67c7 CMAKE_ARGS+=-DGGML_MAX_NAME=128 @@ -38,8 +38,11 @@ else ifeq ($(BUILD_TYPE),hipblas) ROCM_PATH ?= /opt/rocm export CXX=$(ROCM_HOME)/llvm/bin/clang++ export CC=$(ROCM_HOME)/llvm/bin/clang - AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1200,gfx1201 - CMAKE_ARGS+=-DSD_HIPBLAS=ON -DGGML_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) + AMDGPU_TARGETS?=gfx908,gfx90a,gfx942,gfx950,gfx1030,gfx1100,gfx1101,gfx1102,gfx1151,gfx1200,gfx1201 + # SD_HIPBLAS turns on ggml's HIP backend itself; GGML_HIPBLAS is the name ggml + # used before it was renamed to GGML_HIP, so passing it here only produced an + # unused-variable warning. + CMAKE_ARGS+=-DSD_HIPBLAS=ON -DAMDGPU_TARGETS=$(AMDGPU_TARGETS) else ifeq ($(BUILD_TYPE),vulkan) CMAKE_ARGS+=-DSD_VULKAN=ON -DGGML_VULKAN=ON else ifeq ($(BUILD_TYPE),metal) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 754ce5560..1ab67324a 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=438305e1577768ec0f75729456a4c8b9f425e2ee +VLLM_CPP_VERSION?=150b37852c123f7855fb219b37347572ca9427e7 # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index b880105af..20e587a6b 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v21). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 21 +const abiVersion = 23 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -83,6 +83,7 @@ type cModelParams struct { LanguageModelOnly int32 // 0 = multimodal inputs enabled (ABI v19) _ [4]byte LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) + MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 681ca4d4e..55f6e211a 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v21) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(21)) + Expect(abiVersion).To(Equal(23)) }) It("cModelParams matches vllm_model_params", func() { @@ -51,7 +51,8 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(128))) + Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 214fba777..8b0f02d64 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=233fe1fc9b48a09e361d3594520838ca266537fe +WHISPER_CPP_VERSION?=978113305b2ead22249b881deafa131dc8884911 SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF diff --git a/backend/index.yaml b/backend/index.yaml index 3420d8549..6e3519f66 100644 --- a/backend/index.yaml +++ b/backend/index.yaml @@ -510,7 +510,7 @@ default: "cpu-stablediffusion-ggml" nvidia: "cuda12-stablediffusion-ggml" intel: "intel-sycl-f16-stablediffusion-ggml" - # amd: "rocm-stablediffusion-ggml" + amd: "rocm-stablediffusion-ggml" vulkan: "vulkan-stablediffusion-ggml" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml" metal: "metal-stablediffusion-ggml" @@ -2109,7 +2109,7 @@ default: "cpu-stablediffusion-ggml-development" nvidia: "cuda12-stablediffusion-ggml-development" intel: "intel-sycl-f16-stablediffusion-ggml-development" - # amd: "rocm-stablediffusion-ggml-development" + amd: "rocm-stablediffusion-ggml-development" vulkan: "vulkan-stablediffusion-ggml-development" nvidia-l4t: "nvidia-l4t-arm64-stablediffusion-ggml-development" metal: "metal-stablediffusion-ggml-development" @@ -3904,6 +3904,11 @@ uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:latest-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml" + uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:latest-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml" uri: "quay.io/go-skynet/local-ai-backends:latest-gpu-intel-sycl-f32-stablediffusion-ggml" @@ -3917,6 +3922,11 @@ uri: "quay.io/go-skynet/local-ai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml" mirrors: - localai/localai-backends:master-gpu-nvidia-cuda-12-stablediffusion-ggml +- !!merge <<: *stablediffusionggml + name: "rocm-stablediffusion-ggml-development" + uri: "quay.io/go-skynet/local-ai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml" + mirrors: + - localai/localai-backends:master-gpu-rocm-hipblas-stablediffusion-ggml - !!merge <<: *stablediffusionggml name: "intel-sycl-f32-stablediffusion-ggml-development" uri: "quay.io/go-skynet/local-ai-backends:master-gpu-intel-sycl-f32-stablediffusion-ggml" diff --git a/backend/python/sglang/backend.py b/backend/python/sglang/backend.py index c28b59a0e..76d99a726 100644 --- a/backend/python/sglang/backend.py +++ b/backend/python/sglang/backend.py @@ -323,7 +323,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): if not hasattr(request, proto_field): continue value = getattr(request, proto_field) - if value in (None, 0, 0.0, [], False, ""): + if proto_field != "Temperature" and value in (None, 0, 0.0, [], False, ""): continue # repeated fields come back as RepeatedScalarContainer — convert if hasattr(value, "__iter__") and not isinstance(value, (str, bytes)): @@ -363,8 +363,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): template_kwargs["tools"] = json.loads(request.Tools) except json.JSONDecodeError: pass - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: return self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) diff --git a/backend/python/sglang/test.py b/backend/python/sglang/test.py index 92688f444..c50ed577f 100644 --- a/backend/python/sglang/test.py +++ b/backend/python/sglang/test.py @@ -96,6 +96,67 @@ class TestSglangHelpers(unittest.TestCase): servicer._apply_engine_args({}, "[1,2,3]") self.assertIn("must be a JSON object", str(ctx.exception)) + def test_build_prompt_forwards_enable_thinking(self): + from types import SimpleNamespace + + class Tok: + def __init__(self): + self.kwargs = None + + def apply_chat_template(self, messages, **kwargs): + self.kwargs = kwargs + return "PROMPT" + + def kwargs_for(metadata): + servicer = self._servicer() + tok = Tok() + servicer.tokenizer = tok + msg = SimpleNamespace( + role="user", content="hi", name="", + tool_call_id="", reasoning_content="", tool_calls="", + ) + req = SimpleNamespace( + Prompt="", UseTokenizerTemplate=True, + Messages=[msg], Tools="", Metadata=metadata, + ) + self.assertEqual(servicer._build_prompt(req), "PROMPT") + return tok.kwargs + + self.assertIs(kwargs_for({"enable_thinking": "true"})["enable_thinking"], True) + # "false" used to be dropped, so Qwen3 kept thinking on + self.assertIs(kwargs_for({"enable_thinking": "false"})["enable_thinking"], False) + self.assertNotIn("enable_thinking", kwargs_for({})) + self.assertIs(kwargs_for({"enable_thinking": "FALSE"})["enable_thinking"], False) + + def test_explicit_zero_temperature_is_preserved(self): + """Temperature=0 is valid greedy decoding, not an unset value.""" + from types import SimpleNamespace + + servicer = self._servicer() + request = SimpleNamespace( + Temperature=0, + N=0, + PresencePenalty=0, + FrequencyPenalty=0, + RepetitionPenalty=0, + TopP=0, + TopK=0, + MinP=0, + Seed=0, + StopPrompts=[], + StopTokenIds=[], + IgnoreEOS=False, + Tokens=0, + MinTokens=0, + SkipSpecialTokens=False, + Grammar="", + ) + + params = servicer._build_sampling_params(request) + self.assertEqual(params["temperature"], 0) + # Other protobuf-default scalar fields must remain filtered. + self.assertNotIn("top_p", params) + if __name__ == "__main__": unittest.main() diff --git a/backend/python/vllm/backend.py b/backend/python/vllm/backend.py index 8fd3c2dc1..7235c8e07 100644 --- a/backend/python/vllm/backend.py +++ b/backend/python/vllm/backend.py @@ -523,9 +523,7 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): context.set_details(str(e)) return backend_pb2.ScoreResponse() - async def _predict(self, request, context, streaming=False): - # Build the sampling parameters - # NOTE: this must stay in sync with the vllm backend + def _build_sampling_params(self, request): request_to_sampling_params = { "N": "n", "PresencePenalty": "presence_penalty", @@ -555,9 +553,15 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): for request_field, param_field in request_to_sampling_params.items(): if hasattr(request, request_field): value = getattr(request, request_field) - if value not in (None, 0, [], False): + if request_field == "Temperature" or value not in (None, 0, [], False): setattr(sampling_params, param_field, value) + return sampling_params + + async def _predict(self, request, context, streaming=False): + # Build the sampling parameters + sampling_params = self._build_sampling_params(request) + # Structured-output decoding: use Grammar field to pass JSON schema or BNF if HAS_GUIDED_DECODING and request.Grammar: try: @@ -587,9 +591,9 @@ class BackendServicer(backend_pb2_grpc.BackendServicer): except json.JSONDecodeError: pass - # Enable thinking mode if requested - if request.Metadata.get("enable_thinking", "").lower() == "true": - template_kwargs["enable_thinking"] = True + _thinking = request.Metadata.get("enable_thinking", "").lower() + if _thinking in ("true", "false"): + template_kwargs["enable_thinking"] = (_thinking == "true") try: prompt = self.tokenizer.apply_chat_template(messages_dicts, **template_kwargs) diff --git a/backend/python/vllm/requirements-cublas13-after.txt b/backend/python/vllm/requirements-cublas13-after.txt index 34f4f50ff..b519b73fe 100644 --- a/backend/python/vllm/requirements-cublas13-after.txt +++ b/backend/python/vllm/requirements-cublas13-after.txt @@ -3,8 +3,8 @@ # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match # so uv consults this index alongside PyPI. ---extra-index-url https://wheels.vllm.ai/0.27.1/cu130 +--extra-index-url https://wheels.vllm.ai/0.28.0/cu130 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh), # which pins this exact vLLM version. Bumping vllm here means coordinating with a # vllm-metal release that supports the new version, or macOS/Metal builds break. -vllm==0.27.1 +vllm==0.28.0 diff --git a/backend/python/vllm/requirements-l4t13-after.txt b/backend/python/vllm/requirements-l4t13-after.txt index 14fb46634..68860db2b 100644 --- a/backend/python/vllm/requirements-l4t13-after.txt +++ b/backend/python/vllm/requirements-l4t13-after.txt @@ -9,4 +9,4 @@ # memory architecture crash deterministically with an empty "Engine core init # failed" set (mudler/LocalAI#10722). Leaving this unpinned let the L4T image # drift onto whatever wheel was latest at build time. -vllm==0.26.0 +vllm==0.28.0 diff --git a/backend/python/vllm/test.py b/backend/python/vllm/test.py index d00595f01..a0679d4ff 100644 --- a/backend/python/vllm/test.py +++ b/backend/python/vllm/test.py @@ -121,6 +121,19 @@ class TestBackendServicer(unittest.TestCase): finally: self.tearDown() + def test_explicit_zero_temperature_is_preserved(self): + """Temperature=0 is valid greedy decoding, not an unset value.""" + import sys, os + sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + from backend import BackendServicer + + servicer = BackendServicer() + request = backend_pb2.PredictOptions(Prompt="hello", Temperature=0) + sampling_params = servicer._build_sampling_params(request) + self.assertEqual(sampling_params.temperature, 0) + # Other protobuf-default scalar fields must remain filtered. + self.assertEqual(sampling_params.top_p, 0.9) + def test_messages_to_dicts(self): """ diff --git a/core/application/distributed.go b/core/application/distributed.go index 8389c5c9f..b7dc0bf91 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -162,6 +162,15 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } xlog.Info("Node registry initialized") + // Let scheduling rules be keyed by a model alias. The registry resolves a + // rule's name through the config loader to find the model it governs, so an + // operator can pin placement to a stable name like "production" and have it + // follow the alias when the alias is repointed. Wired before the seed below + // and before the reconciler starts, so the first tick already resolves. + if configLoader != nil { + registry.SetAliasResolver(configLoader) + } + // Seed declarative per-model scheduling config (LOCALAI_MODEL_SCHEDULING / // LOCALAI_MODEL_SCHEDULING_CONFIG). Authoritative: overwrites matching models // on every boot. Runs before the reconciler starts so the first tick already diff --git a/core/application/startup.go b/core/application/startup.go index 66a813162..abc2f4a17 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -267,6 +267,10 @@ func New(opts ...config.AppOption) (*Application, error) { } // Initialize distributed mode services (NATS, object storage, node registry) + // revisionStore is built inside the distributed block below but used after + // the model configs are loaded, so it is declared out here. + var revisionStore modeladmin.RevisionStore + distSvc, err := initDistributed(options, application.authDB, application.ModelConfigLoader()) if err != nil { return nil, fmt.Errorf("distributed mode initialization failed: %w", err) @@ -373,6 +377,9 @@ func New(opts ...config.AppOption) (*Application, error) { cfgLoaderOpts := options.ToConfigLoaderOptions() modelRevisionLifecycle := modeladmin.NewDistributedModelRevisionLifecycle(distSvc.Registry, distSvc.ModelCleanup) gs.SetModelRevisionLifecycle(modelRevisionLifecycle) + // Captured here, used after the model configs are loaded below: the + // resync reads the loader, which is still empty at this point. + revisionStore = modeladmin.NewRevisionStore(distSvc.Registry, modelRevisionLifecycle) gs.OnModelsChanged = func(evt messaging.CacheInvalidateEvent) { // ApplyRemoteChange honors the op: a "delete" prunes the element // (a reload-from-path is additive and cannot drop it), anything @@ -419,6 +426,18 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Error("error loading config files", "error", err) } + // Bring the controller's stored revisions back in line with the + // configuration just loaded. An inference request may only establish a + // revision, never replace one, so a model whose stored value has drifted + // stays unroutable until something republishes it. This has to run after + // the load above: the loader is empty until then, and a resync against an + // empty loader silently reconciles nothing. + if revisionStore != nil { + if err := modeladmin.ResyncModelConfigRevisions(options.Context, application.ModelConfigLoader(), options, revisionStore); err != nil { + xlog.Warn("Failed to resync model config revisions", "error", err) + } + } + if err := gallery.RegisterBackends(options.SystemState, application.ModelLoader()); err != nil { xlog.Error("error registering external backends", "error", err) } diff --git a/core/backend/options.go b/core/backend/options.go index 4f7c81483..93a6eadc0 100644 --- a/core/backend/options.go +++ b/core/backend/options.go @@ -202,18 +202,18 @@ func ModelOptions(c config.ModelConfig, so *config.ApplicationConfig, opts ...mo model.WithContext(so.Context), model.WithModelID(c.ModelID()), } - // Prefer the revision stamped when the configuration was loaded. c has since - // been merged with this request's prediction parameters (temperature, top_p, - // stop, ...), and hashing it here would produce a different revision for - // every distinct request body — which the controller reads as a config - // change and rejects as stale. Recomputing is the fallback for a config that - // never passed through the loader. + // Use the revision stamped when the configuration was parsed, and only + // that. By this point c has been merged with the request's prediction + // parameters and had SetDefaults applied, so hashing it here would produce + // a revision that depends on the request body and on whether the model file + // parsed, which the controller reads as a config change and rejects. Every + // config the loader hands out is stamped; an unstamped one was synthesized + // elsewhere and is routed without a revision rather than with a wrong one. if revision := c.PersistedConfigRevision(); revision != "" { defOpts = append(defOpts, model.WithConfigRevision(revision)) - } else if revision, err := config.ModelConfigRevision(&c); err == nil { - defOpts = append(defOpts, model.WithConfigRevision(revision)) } else { - xlog.Warn("Failed to compute model configuration revision", "model", c.ModelID(), "error", err) + xlog.Warn("Model configuration carries no revision stamp; routing without one", + "model", c.ModelID()) } managedPrimary := len(c.Artifacts) > 0 && c.Artifacts[0].Resolved != nil if managedPrimary { diff --git a/core/config/inference_defaults.json b/core/config/inference_defaults.json index 8fe888de1..42c463df8 100644 --- a/core/config/inference_defaults.json +++ b/core/config/inference_defaults.json @@ -1,6 +1,8 @@ { "_comment": "Auto-generated from unsloth inference_defaults.json. DO NOT EDIT. Run go generate ./core/config/ to update.", "families": { + "moss-tts-local-transformer-v1.5": {"min_p":0,"repeat_penalty":1,"temperature":1.7,"top_k":25,"top_p":0.8}, + "moss-tts-nano": {"min_p":0,"repeat_penalty":1,"temperature":1.7,"top_k":25,"top_p":0.8}, "qwen3.8": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8}, "qwen3.6": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8}, "qwen3.5": {"min_p":0,"presence_penalty":1.5,"repeat_penalty":1,"temperature":0.7,"top_k":20,"top_p":0.8}, @@ -60,5 +62,5 @@ "grok": {"min_p":0.01,"repeat_penalty":1,"temperature":1,"top_k":-1,"top_p":0.95}, "mimo": {"min_p":0.01,"repeat_penalty":1,"temperature":0.7,"top_k":-1,"top_p":0.95} }, - "patterns": ["qwen3.8","qwen3.6","qwen3.5","qwen3-coder","qwen3-next","qwen3-vl","qwen3","qwen2.5-coder","qwen2.5-vl","qwen2.5-omni","qwen2.5-math","qwen2.5","qwen2-vl","qwen2","qwq","gemma-4","gemma-3n","gemma-3","medgemma","gemma-2","muse-glimmer","llama-4","llama-3.3","llama-3.2","llama-3.1","llama-3","phi-4","phi-3","mistral-nemo","mistral-small","mistral-large","magistral","ministral","devstral","pixtral","deepseek-v4","deepseek-r1","deepseek-v3","deepseek-ocr","glm-5","glm-4","nemotron","minimax-m2.7","minimax-m2.5","minimax","gpt-oss","granite-4","kimi-k3","kimi-k2","kimi","lfm2","smollm","olmo","falcon","ernie","seed","grok","mimo"] + "patterns": ["moss-tts-local-transformer-v1.5","moss-tts-nano","qwen3.8","qwen3.6","qwen3.5","qwen3-coder","qwen3-next","qwen3-vl","qwen3","qwen2.5-coder","qwen2.5-vl","qwen2.5-omni","qwen2.5-math","qwen2.5","qwen2-vl","qwen2","qwq","gemma-4","gemma-3n","gemma-3","medgemma","gemma-2","muse-glimmer","llama-4","llama-3.3","llama-3.2","llama-3.1","llama-3","phi-4","phi-3","mistral-nemo","mistral-small","mistral-large","magistral","ministral","devstral","pixtral","deepseek-v4","deepseek-r1","deepseek-v3","deepseek-ocr","glm-5","glm-4","nemotron","minimax-m2.7","minimax-m2.5","minimax","gpt-oss","granite-4","kimi-k3","kimi-k2","kimi","lfm2","smollm","olmo","falcon","ernie","seed","grok","mimo"] } diff --git a/core/config/model_config.go b/core/config/model_config.go index c6121eb8c..600519c7f 100644 --- a/core/config/model_config.go +++ b/core/config/model_config.go @@ -1864,7 +1864,7 @@ func (c *ModelConfig) PersistedConfigRevision() string { // persisted. It is computed from the receiver as-is, so callers must invoke it // only on a configuration that has not been merged with request overrides. func (c *ModelConfig) StampPersistedConfigRevision() error { - revision, err := ModelConfigRevision(c) + revision, err := modelConfigRevision(c) if err != nil { return err } diff --git a/core/config/model_config_loader.go b/core/config/model_config_loader.go index 4c95a9665..9120cc268 100644 --- a/core/config/model_config_loader.go +++ b/core/config/model_config_loader.go @@ -168,6 +168,14 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model if err := yaml.Unmarshal(f, &configs); err == nil && len(configs) > 0 { for _, cc := range configs { cc.modelConfigFile = file + // Stamp before SetDefaults: the revision describes what is on disk. + // SetDefaults folds in the GGUF guess, hardware defaults and + // app-level options, none of which are persisted configuration, and + // the GGUF guess in particular depends on whether the model file + // parses at that moment. + if err := cc.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", cc.Name, err) + } cc.SetDefaults(opts...) cc.syncKnownUsecasesFromString() } @@ -182,6 +190,9 @@ func readModelConfigsFromFile(file string, opts ...ConfigLoaderOption) ([]*Model c.modelConfigFile = file c.syncKnownUsecasesFromString() + if err := c.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", c.Name, err) + } c.SetDefaults(opts...) return []*ModelConfig{c}, nil @@ -218,17 +229,18 @@ func (bcl *ModelConfigLoader) LoadModelConfigFileByName(modelName, modelPath str } } - cfg.SetDefaults(append(opts, ModelPath(modelPath))...) - - // Stamp the revision here, at the boundary between the persisted - // configuration and the request that is about to override parts of it. - // Everything downstream of this point (the request middleware) merges - // per-request prediction parameters into cfg, so a revision computed later - // would identify the request rather than the configuration. - if err := cfg.StampPersistedConfigRevision(); err != nil { - return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + // Stamp before SetDefaults, and only when this config did not come from + // disk already carrying one (a name with no config file on disk is + // synthesized above). Re-stamping a loaded config here would hash it after + // SetDefaults and reintroduce the dependency on the GGUF guess. + if cfg.PersistedConfigRevision() == "" { + if err := cfg.StampPersistedConfigRevision(); err != nil { + return nil, fmt.Errorf("stamping config revision for %q: %w", modelName, err) + } } + cfg.SetDefaults(append(opts, ModelPath(modelPath))...) + return cfg, nil } @@ -429,6 +441,26 @@ func (bcl *ModelConfigLoader) ResolveAlias(cfg *ModelConfig) (*ModelConfig, bool return &target, true, nil } +// ResolveAliasName maps a model name to the name of the model that actually +// serves it: an alias resolves to its target, anything else resolves to +// itself. The second return reports whether name was an alias. +// +// Unlike ResolveAlias this never errors. A name with no config (a rule may be +// authored before the model is installed), a dangling alias, and a chained +// alias all resolve to themselves, so callers keep a usable name that simply +// has no model behind it rather than silently governing a different model. +func (bcl *ModelConfigLoader) ResolveAliasName(name string) (string, bool) { + cfg, exists := bcl.GetModelConfig(name) + if !exists || !cfg.IsAlias() { + return name, false + } + target, exists := bcl.GetModelConfig(cfg.Alias) + if !exists || target.IsAlias() { + return name, true + } + return target.Name, true +} + // ValidateAliasTarget checks an alias config's target at create/swap time: // the target must exist, must not be an alias, and must not be disabled. // Returns nil for non-alias configs. @@ -953,3 +985,38 @@ func hasAnyMappingKey(mapping *yaml.Node, keys ...string) bool { func nonemptyScalar(node *yaml.Node) bool { return node != nil && node.Kind == yaml.ScalarNode && node.Tag == "!!str" && strings.TrimSpace(node.Value) != "" } + +// RevisionFor returns the config revision for modelName: the one an inference +// request for that model will carry. +// +// This is the only way to obtain a revision outside this package. Every +// publisher must use it, so that what is published and what is checked are +// the same value by construction rather than by two implementations happening +// to agree. Hashing a ModelConfig directly is not available to callers, because +// a config that has been through SetDefaults or the request middleware hashes +// to something no request will ever present. +func (bcl *ModelConfigLoader) RevisionFor(modelName string, appConfig *ApplicationConfig) (string, error) { + cfg, err := bcl.LoadModelConfigFileByNameDefaultOptions(modelName, appConfig) + if err != nil { + return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err) + } + return stampedRevision(cfg, modelName) +} + +// RevisionForPath is RevisionFor for callers that hold loader options and a +// models path rather than an ApplicationConfig. +func (bcl *ModelConfigLoader) RevisionForPath(modelName, modelPath string, opts ...ConfigLoaderOption) (string, error) { + cfg, err := bcl.LoadModelConfigFileByName(modelName, modelPath, opts...) + if err != nil { + return "", fmt.Errorf("resolving config revision for %q: %w", modelName, err) + } + return stampedRevision(cfg, modelName) +} + +func stampedRevision(cfg *ModelConfig, modelName string) (string, error) { + revision := cfg.PersistedConfigRevision() + if revision == "" { + return "", fmt.Errorf("no config revision stamped for %q", modelName) + } + return revision, nil +} diff --git a/core/config/model_config_loader_test.go b/core/config/model_config_loader_test.go index 87807deec..d654226ef 100644 --- a/core/config/model_config_loader_test.go +++ b/core/config/model_config_loader_test.go @@ -314,3 +314,57 @@ var _ = Describe("ModelConfigLoader alias resolution", func() { Expect(loader.ValidateAliasTarget(&bad)).To(MatchError(ContainSubstring("itself an alias"))) }) }) + +var _ = Describe("ModelConfigLoader ResolveAliasName", func() { + var loader *ModelConfigLoader + + BeforeEach(func() { + loader = NewModelConfigLoader("") + loader.configs["real"] = ModelConfig{Name: "real", Backend: "llama-cpp"} + loader.configs["production"] = ModelConfig{Name: "production", Alias: "real"} + loader.configs["chain"] = ModelConfig{Name: "chain", Alias: "production"} + loader.configs["dangling"] = ModelConfig{Name: "dangling", Alias: "nope"} + }) + + It("maps an alias name to the model that actually serves it", func() { + target, isAlias := loader.ResolveAliasName("production") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("real")) + }) + + It("maps a real model name to itself", func() { + target, isAlias := loader.ResolveAliasName("real") + Expect(isAlias).To(BeFalse()) + Expect(target).To(Equal("real")) + }) + + // A rule may be authored for a model that is not installed yet (pre-staging + // placement before standing up a node), so an unknown name must resolve to + // itself rather than to the empty string. + It("maps an unknown name to itself", func() { + target, isAlias := loader.ResolveAliasName("not-installed-yet") + Expect(isAlias).To(BeFalse()) + Expect(target).To(Equal("not-installed-yet")) + }) + + // A broken alias has no model behind it. Resolving to itself keeps the + // caller on a name that simply has no replicas, instead of silently + // governing some other model. + It("maps a dangling alias to itself", func() { + target, isAlias := loader.ResolveAliasName("dangling") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("dangling")) + }) + + It("maps a chained alias to itself rather than following the chain", func() { + target, isAlias := loader.ResolveAliasName("chain") + Expect(isAlias).To(BeTrue()) + Expect(target).To(Equal("chain")) + }) + + It("maps the empty name to itself", func() { + target, isAlias := loader.ResolveAliasName("") + Expect(isAlias).To(BeFalse()) + Expect(target).To(BeEmpty()) + }) +}) diff --git a/core/config/model_config_revision.go b/core/config/model_config_revision.go index e8dd1bee5..8cbad2d6b 100644 --- a/core/config/model_config_revision.go +++ b/core/config/model_config_revision.go @@ -10,10 +10,17 @@ import ( "google.golang.org/protobuf/proto" ) -// ModelConfigRevision returns a stable revision of the persisted semantic +// modelConfigRevision returns a stable revision of the persisted semantic // configuration. ModelConfig's JSON tags exclude runtime-derived state and // source bookkeeping, while encoding/json orders map keys deterministically. -func ModelConfigRevision(cfg *ModelConfig) (string, error) { +// +// Deliberately unexported. It must only ever be called on a configuration as +// parsed from disk, before SetDefaults folds in the GGUF guess, the hardware +// defaults and app-level options. Callers outside this package cannot tell +// which they hold, and every time one hashed a defaulted or request-merged +// config it published a revision no inference request would carry, which makes +// the model unroutable. Use ModelConfigLoader.RevisionFor instead. +func modelConfigRevision(cfg *ModelConfig) (string, error) { if cfg == nil { return "", errors.New("model config is nil") } diff --git a/core/config/model_config_revision_stability_test.go b/core/config/model_config_revision_stability_test.go index 19a0d8590..433b8e401 100644 --- a/core/config/model_config_revision_stability_test.go +++ b/core/config/model_config_revision_stability_test.go @@ -51,9 +51,8 @@ template: Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) cfg, ok := loader.GetModelConfig("example") Expect(ok).To(BeTrue()) - revision, err := config.ModelConfigRevision(&cfg) - Expect(err).ToNot(HaveOccurred()) - return revision + Expect(cfg.PersistedConfigRevision()).ToNot(BeEmpty()) + return cfg.PersistedConfigRevision() } It("does not change when the same file is loaded repeatedly", func() { @@ -73,18 +72,75 @@ template: }) // The request pipeline reloads the config through LoadModelConfigFileByName, - // which applies SetDefaults a second time. That must not move the revision - // away from the one model administration publishes from the loader map. + // which applies SetDefaults a second time. The stamp is taken before those + // defaults, so both the stored config and the one a request resolves carry + // the same revision. It("survives the extra SetDefaults the request path applies", func() { loader := config.NewModelConfigLoader(dir) Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) stored, ok := loader.GetModelConfig("example") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&stored) - Expect(err).ToNot(HaveOccurred()) + Expect(stored.PersistedConfigRevision()).ToNot(BeEmpty()) requestCfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) Expect(err).ToNot(HaveOccurred()) - Expect(requestCfg.PersistedConfigRevision()).To(Equal(adminRevision)) + Expect(requestCfg.PersistedConfigRevision()).To(Equal(stored.PersistedConfigRevision())) + }) +}) + +// The revision must describe the configuration as persisted, and nothing else. +// SetDefaults folds in values that are not persisted config: the GGUF guess +// (which reads the model file and can fail on slow or remote storage), the +// hardware defaults, and app-level options like threads. Hashing after that +// made the revision a function of whether a multi-gigabyte file happened to +// parse, so one unchanged YAML produced two different revisions depending on +// the moment, and the controller rejected every request carrying the other one. +var _ = Describe("Model config revision independence from runtime defaults", func() { + It("does not change when SetDefaults is applied", func() { + dir := GinkgoT().TempDir() + body := "backend: llama-cpp\ncontext_size: 50000\nknown_usecases:\n - chat\n" + + "mmproj: llama-cpp/mmproj/example/mmproj.gguf\nname: example\n" + + "parameters:\n model: llama-cpp/models/example/example.gguf\n" + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), []byte(body), 0o600)).To(Succeed()) + + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + + stored, ok := loader.GetModelConfig("example") + Expect(ok).To(BeTrue()) + before := stored.PersistedConfigRevision() + Expect(before).ToNot(BeEmpty()) + + // Applying defaults again is what the request path does. + stored.SetDefaults(appConfig.ToConfigLoaderOptions()...) + Expect(stored.PersistedConfigRevision()).To(Equal(before)) + + resolved, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + Expect(resolved.PersistedConfigRevision()).To(Equal(before), + "the request path must carry the same revision as the stored config") + }) + + It("does not change when app-level defaults differ", func() { + dir := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dir, "example.yaml"), + []byte("name: example\nbackend: llama-cpp\nparameters:\n model: m.gguf\n"), 0o600)).To(Succeed()) + + revWith := func(threads int, f16 bool) string { + appConfig := config.NewApplicationConfig() + appConfig.SystemState = &system.SystemState{Model: system.Model{ModelsPath: dir}} + appConfig.Threads = threads + appConfig.F16 = f16 + loader := config.NewModelConfigLoader(dir) + Expect(loader.LoadModelConfigsFromPath(dir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) + cfg, err := loader.LoadModelConfigFileByNameDefaultOptions("example", appConfig) + Expect(err).ToNot(HaveOccurred()) + return cfg.PersistedConfigRevision() + } + + Expect(revWith(8, false)).To(Equal(revWith(1, true)), + "an operator changing threads must not make every model unroutable") }) }) diff --git a/core/config/model_config_revision_test.go b/core/config/model_config_revision_test.go index 0b95d3549..b75937271 100644 --- a/core/config/model_config_revision_test.go +++ b/core/config/model_config_revision_test.go @@ -19,10 +19,11 @@ var _ = Describe("Model configuration revisions", func() { return cfg } + // The raw hash is unexported on purpose, so these specs exercise it the way + // every caller now must: by stamping the parsed config. revision := func(cfg *config.ModelConfig) string { - value, err := config.ModelConfigRevision(cfg) - Expect(err).NotTo(HaveOccurred()) - return value + Expect(cfg.StampPersistedConfigRevision()).To(Succeed()) + return cfg.PersistedConfigRevision() } It("is stable across equivalent YAML formatting and map order", func() { diff --git a/core/gallery/cluster_env_test.go b/core/gallery/cluster_env_test.go new file mode 100644 index 000000000..4a411277e --- /dev/null +++ b/core/gallery/cluster_env_test.go @@ -0,0 +1,94 @@ +package gallery_test + +import ( + "context" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/system" +) + +// On a distributed controller the GPUs live on the workers, so a variant +// picker sized against the controller tells admins a cluster of A100s can only +// run the smallest CPU build. +var _ = Describe("ClusterResolveEnv", func() { + gib := func(n uint64) uint64 { return n * 1024 * 1024 * 1024 } + + // The controller as Argus actually runs it: no GPU at all. + var controller *system.SystemState + + BeforeEach(func() { + controller = system.NewCapabilityState("default") + }) + + It("sizes models against the cluster reading rather than the controller", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"}) + + Expect(env.AvailableMemory).To(Equal(gib(80))) + }) + + It("accepts a CUDA backend that only the workers can run", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"}) + + Expect(env.BackendCompatible).ToNot(BeNil()) + // A name carrying the cuda token is what the controller rejects today; + // a bare engine name like "vllm" passes on any host and would prove + // nothing about the union. + Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue()) + Expect(env.BackendCompatible("llama-cpp")).To(BeTrue()) + }) + + // The union must stay a filter, not an open door: a Linux NVIDIA fleet + // still cannot run an Apple-only build. + It("still rejects a backend no node in the cluster can run", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"}) + + Expect(env.BackendCompatible("mlx")).To(BeFalse()) + }) + + It("accepts a backend that any one node in a mixed fleet can run", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13", "metal"}) + + Expect(env.BackendCompatible("mlx")).To(BeTrue()) + Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue()) + }) + + // Ranking has to follow the hardware too, or a cluster of NVIDIA workers + // gets offered the GGUF build over the vLLM one it should prefer. + It("ranks engines by the workers' hardware, not the controller's", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"}) + + Expect(env.EnginePreference).To(Equal(system.NewCapabilityState("nvidia-cuda-13").EnginePreferenceTokens())) + }) + + // Every degradation path lands here, so it must be indistinguishable from + // the single-node behavior that shipped before any of this existed. + It("falls back to the host description when the cluster reports nothing", func() { + host := gallery.HostResolveEnv(context.Background(), controller) + env := gallery.ClusterResolveEnv(context.Background(), controller, 0, nil) + + Expect(env.AvailableMemory).To(Equal(host.AvailableMemory)) + Expect(env.EnginePreference).To(Equal(host.EnginePreference)) + Expect(env.BackendCompatible("cuda-13-vllm")).To(Equal(host.BackendCompatible("cuda-13-vllm"))) + Expect(env.BackendCompatible("mlx")).To(Equal(host.BackendCompatible("mlx"))) + }) + + // A cluster that reports capabilities but no usable memory reading should + // still gain the hardware view; only the size question falls back. + It("keeps the host memory when only the memory reading is missing", func() { + host := gallery.HostResolveEnv(context.Background(), controller) + env := gallery.ClusterResolveEnv(context.Background(), controller, 0, []string{"nvidia-cuda-13"}) + + Expect(env.AvailableMemory).To(Equal(host.AvailableMemory)) + Expect(env.BackendCompatible("cuda-13-vllm")).To(BeTrue()) + }) + + It("keeps the probe wired so variant sizes are still measured", func() { + env := gallery.ClusterResolveEnv(context.Background(), controller, gib(80), []string{"nvidia-cuda-13"}) + + Expect(env.ProbeMemory).ToNot(BeNil()) + Expect(env.ServingFeaturePreference).To(Equal(system.ServingFeaturePreferenceTokens())) + }) +}) diff --git a/core/gallery/models.go b/core/gallery/models.go index f3184648b..7664b6e8c 100644 --- a/core/gallery/models.go +++ b/core/gallery/models.go @@ -938,3 +938,73 @@ func SafetyScanGalleryModel(galleryModel *GalleryModel) error { } return nil } + +// ClusterResolveEnv describes a CLUSTER to variant selection, where +// HostResolveEnv describes one machine. +// +// It exists because a distributed controller is the wrong machine to ask. The +// controller is typically a GPU-less pod while every model actually runs on a +// worker, so a picker sized against it reports that a fleet of A100s can only +// run the smallest CPU build, and auto-selection then installs exactly that. +// +// availableMemory is the largest single healthy node's budget, and capabilities +// are the capability strings present in the cluster. Either may be empty: a +// zero memory reading keeps the host's own figure and an empty capability list +// keeps the host's own hardware verdict, so every degradation path lands back +// on the single-node behavior rather than on a cluster described as having +// nothing. +func ClusterResolveEnv(ctx context.Context, systemState *system.SystemState, availableMemory uint64, capabilities []string) ResolveEnv { + env := HostResolveEnv(ctx, systemState) + + if availableMemory > 0 { + env.AvailableMemory = availableMemory + } + if len(capabilities) == 0 { + return env + } + + // One state pinned per capability, mirroring AvailableBackendsForCapabilities: + // the controller's own detection must not leak into a worker's verdict, and + // a forced capability on the controller image must not either. + nodeStates := make([]*system.SystemState, 0, len(capabilities)) + for _, capability := range capabilities { + nodeStates = append(nodeStates, system.NewCapabilityState(capability, + system.WithBackendPath(systemState.Backend.BackendsPath))) + } + + hostCompatible := env.BackendCompatible + // A union, because a variant only has to run SOMEWHERE. The controller + // stays in the union so a cluster whose workers all went offline still + // describes itself the way it did before distributed mode existed. + env.BackendCompatible = func(backend string) bool { + if hostCompatible != nil && hostCompatible(backend) { + return true + } + for _, nodeState := range nodeStates { + if nodeState.IsBackendCompatible(backend, "") { + return true + } + } + return false + } + + // Ranking follows the same hardware as the filter. Left on the controller's + // tokens, an NVIDIA fleet would be offered the GGUF build over the vLLM one + // even though nothing filtered the vLLM build out. + seen := make(map[string]struct{}) + preference := make([]string, 0, len(nodeStates)) + for _, nodeState := range nodeStates { + for _, token := range nodeState.EnginePreferenceTokens() { + if _, dup := seen[token]; dup { + continue + } + seen[token] = struct{}{} + preference = append(preference, token) + } + } + if len(preference) > 0 { + env.EnginePreference = preference + } + + return env +} diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index 4083333c9..d8a5612a1 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -356,7 +356,7 @@ func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc { // local system state is the only thing worth filtering against. type ClusterCapabilityProvider func(ctx context.Context) ([]string, error) -// resolveClusterCapabilities reads the capabilities present in the cluster, +// ResolveClusterCapabilities reads the capabilities present in the cluster, // degrading to the local-only listing on error. // // Every capability-filtered discovery endpoint shares this: on a distributed @@ -364,7 +364,7 @@ type ClusterCapabilityProvider func(ctx context.Context) ([]string, error) // (usually GPU-less) host hides GPU-only backends the cluster can actually // run. A registry hiccup must never blank the catalog, so a failure falls back // to the pre-existing local-only behavior rather than erroring the request. -func resolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string { +func ResolveClusterCapabilities(ctx context.Context, provider ClusterCapabilityProvider) []string { if provider == nil { return nil } @@ -423,7 +423,7 @@ func installedInCluster(backend *gallery.GalleryBackend, clusterInstalled map[st // @Router /backends/available [get] func (mgs *BackendEndpointService) ListAvailableBackendsEndpoint(systemState *system.SystemState, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { - capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) + capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities) backends, err := gallery.AvailableBackendsForCapabilities(mgs.galleries, systemState, capabilities) if err != nil { diff --git a/core/http/endpoints/localai/edit_model_test.go b/core/http/endpoints/localai/edit_model_test.go index 17f7c4a7c..223943e46 100644 --- a/core/http/endpoints/localai/edit_model_test.go +++ b/core/http/endpoints/localai/edit_model_test.go @@ -294,9 +294,9 @@ var _ = Describe("Edit Model test", func() { Expect(client.published[0]).To(Equal(messaging.CacheInvalidateEvent{ Element: "old", Op: "delete", ConfigRevision: modeladmin.DeletedModelConfigRevision("old"), })) - newConfig, ok := loader.GetModelConfig("new") + _, ok := loader.GetModelConfig("new") Expect(ok).To(BeTrue()) - newRevision, err := config.ModelConfigRevision(&newConfig) + newRevision, err := loader.RevisionForPath("new", tempDir) Expect(err).ToNot(HaveOccurred()) Expect(client.published[1]).To(Equal(messaging.CacheInvalidateEvent{ Element: "new", Op: "install", ConfigRevision: newRevision, @@ -313,9 +313,9 @@ var _ = Describe("Edit Model test", func() { } _, oldOnPeer := peerLoader.GetModelConfig("old") Expect(oldOnPeer).To(BeFalse()) - peerConfig, newOnPeer := peerLoader.GetModelConfig("new") + _, newOnPeer := peerLoader.GetModelConfig("new") Expect(newOnPeer).To(BeTrue()) - peerRevision, err := config.ModelConfigRevision(&peerConfig) + peerRevision, err := peerLoader.RevisionForPath("new", tempDir) Expect(err).ToNot(HaveOccurred()) Expect(peerRevision).To(Equal(newRevision)) Expect(peerLifecycle.batches).To(Equal([][]modeladmin.ModelRevisionTransition{ diff --git a/core/http/endpoints/localai/finetune.go b/core/http/endpoints/localai/finetune.go index 4948b65fc..2ddf1bcff 100644 --- a/core/http/endpoints/localai/finetune.go +++ b/core/http/endpoints/localai/finetune.go @@ -276,7 +276,7 @@ func DownloadExportedModelEndpoint(ftService *finetune.FineTuneService) echo.Han // ListFineTuneBackendsEndpoint returns installed backends tagged with "fine-tuning". func ListFineTuneBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { - capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) + capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities) installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled) backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities) if err != nil { diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index bc26baf49..bbae523b1 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -1218,6 +1218,20 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { return c.JSON(http.StatusBadRequest, nodeError(http.StatusBadRequest, err.Error())) } + // A rule may be keyed by an alias, in which case it governs whatever + // that alias currently points at. Reject an alias that resolves to + // nothing, and reject a second rule for a model some other rule already + // governs, so the operator hears about the clash instead of silently + // writing a rule that never takes effect. + target, err := registry.ValidateSchedulingTarget(ctx, req.ModelName) + if err != nil { + status := http.StatusBadRequest + if errors.Is(err, nodes.ErrSchedulingConflict) { + status = http.StatusConflict + } + return c.JSON(status, nodeError(status, err.Error())) + } + // Serialize node selector to JSON var selectorJSON string if len(req.NodeSelector) > 0 { @@ -1230,6 +1244,7 @@ func SetSchedulingEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { config := &nodes.ModelSchedulingConfig{ ModelName: req.ModelName, + TargetModel: target, NodeSelector: selectorJSON, MinReplicas: req.MinReplicas, MaxReplicas: req.MaxReplicas, diff --git a/core/http/endpoints/localai/nodes_backends_list_test.go b/core/http/endpoints/localai/nodes_backends_list_test.go index c625e8e95..636ab58b8 100644 --- a/core/http/endpoints/localai/nodes_backends_list_test.go +++ b/core/http/endpoints/localai/nodes_backends_list_test.go @@ -42,6 +42,8 @@ func (s *stubNodeCommandSender) StopBackend(_, _ string) error { return nil } func (s *stubNodeCommandSender) UnloadModelOnNode(_, _ string) error { return nil } +func (s *stubNodeCommandSender) PingNode(_ string) error { return nil } + var _ = Describe("ListBackendsOnNodeEndpoint", func() { var registry *nodes.NodeRegistry diff --git a/core/http/endpoints/localai/nodes_scheduling_alias_test.go b/core/http/endpoints/localai/nodes_scheduling_alias_test.go new file mode 100644 index 000000000..35065f614 --- /dev/null +++ b/core/http/endpoints/localai/nodes_scheduling_alias_test.go @@ -0,0 +1,125 @@ +package localai + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + + "github.com/labstack/echo/v4" + "github.com/mudler/LocalAI/core/services/nodes" + "github.com/mudler/LocalAI/core/services/testutil" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// aliasResolverStub maps alias names to targets in place of a config loader. +type aliasResolverStub struct{ aliases map[string]string } + +func (s *aliasResolverStub) ResolveAliasName(name string) (string, bool) { + target, ok := s.aliases[name] + if !ok { + return name, false + } + return target, true +} + +var _ = Describe("Scheduling endpoints with model aliases", func() { + var ( + registry *nodes.NodeRegistry + resolver *aliasResolverStub + ) + + BeforeEach(func() { + db := testutil.SetupTestDB() + var err error + registry, err = nodes.NewNodeRegistry(db) + Expect(err).ToNot(HaveOccurred()) + resolver = &aliasResolverStub{aliases: map[string]string{"production": "qwen3"}} + registry.SetAliasResolver(resolver) + }) + + post := func(body string) *httptest.ResponseRecorder { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + ExpectWithOffset(1, SetSchedulingEndpoint(registry)(c)).To(Succeed()) + return rec + } + + It("accepts a rule keyed by an alias and reports the model it governs", func() { + rec := post(`{"model_name":"production","min_replicas":2,"node_selector":{"tier":"gpu"}}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var resp map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) + Expect(resp["model_name"]).To(Equal("production")) + Expect(resp["target_model"]).To(Equal("qwen3")) + }) + + It("rejects a second rule for a model an alias rule already governs", func() { + Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"qwen3","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusConflict)) + Expect(rec.Body.String()).To(ContainSubstring("production")) + }) + + It("rejects an alias rule for a model that already has its own rule", func() { + Expect(post(`{"model_name":"qwen3","min_replicas":1}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"production","min_replicas":2}`) + Expect(rec.Code).To(Equal(http.StatusConflict)) + Expect(rec.Body.String()).To(ContainSubstring("qwen3")) + }) + + It("still allows editing a rule in place", func() { + Expect(post(`{"model_name":"production","min_replicas":2}`).Code).To(Equal(http.StatusOK)) + + rec := post(`{"model_name":"production","min_replicas":4}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + + stored, err := registry.GetModelScheduling(context.Background(), "production") + Expect(err).ToNot(HaveOccurred()) + Expect(stored.MinReplicas).To(Equal(4)) + }) + + It("rejects a rule keyed by an alias that does not resolve", func() { + resolver.aliases["orphan"] = "orphan" + + rec := post(`{"model_name":"orphan","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusBadRequest)) + Expect(rec.Body.String()).To(ContainSubstring("does not resolve")) + }) + + It("still accepts a rule for a model that is not installed yet", func() { + rec := post(`{"model_name":"not-installed-yet","min_replicas":1}`) + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + + It("labels a rule that another rule shadows when listing", func() { + // A seed file or a repointed alias can leave two rules on one model, + // which the write path above rejects but cannot retract. + Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "production", MinReplicas: 2})).To(Succeed()) + Expect(registry.SetModelScheduling(context.Background(), &nodes.ModelSchedulingConfig{ModelName: "qwen3", MinReplicas: 1})).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodGet, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + Expect(ListSchedulingEndpoint(registry)(c)).To(Succeed()) + + var listed []map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &listed)).To(Succeed()) + byName := map[string]map[string]any{} + for _, item := range listed { + byName[item["model_name"].(string)] = item + } + Expect(byName["qwen3"]["shadowed"]).To(BeNil()) + Expect(byName["production"]["shadowed"]).To(Equal(true)) + }) +}) diff --git a/core/http/endpoints/localai/quantization.go b/core/http/endpoints/localai/quantization.go index 175bf9740..ae6c91739 100644 --- a/core/http/endpoints/localai/quantization.go +++ b/core/http/endpoints/localai/quantization.go @@ -195,7 +195,7 @@ func DownloadQuantizedModelEndpoint(qService *quantization.QuantizationService) // ListQuantizationBackendsEndpoint returns installed backends tagged with "quantization". func ListQuantizationBackendsEndpoint(appConfig *config.ApplicationConfig, clusterCapabilities ClusterCapabilityProvider, clusterInstalled ClusterInstalledProvider) echo.HandlerFunc { return func(c echo.Context) error { - capabilities := resolveClusterCapabilities(c.Request().Context(), clusterCapabilities) + capabilities := ResolveClusterCapabilities(c.Request().Context(), clusterCapabilities) installed := resolveClusterInstalled(c.Request().Context(), clusterInstalled) backends, err := gallery.AvailableBackendsForCapabilities(appConfig.BackendGalleries, appConfig.SystemState, capabilities) if err != nil { diff --git a/core/http/endpoints/ollama/models_test.go b/core/http/endpoints/ollama/models_test.go index b13cf59a0..c4d0d6b5e 100644 --- a/core/http/endpoints/ollama/models_test.go +++ b/core/http/endpoints/ollama/models_test.go @@ -146,6 +146,20 @@ parameters: Expect(resp.Details.Format).To(Equal("gguf")) Expect(resp.Details.Families).ToNot(BeEmpty()) }) + + It("looks up the model when the Ollama :latest tag is included", func() { + writeConfig("chat", ` +name: chat +backend: llama-cpp +template: + chat: "{{ .Input }}" +parameters: + model: Llama-3-8B-Q4_K_M.gguf +`) + resp := callShow("chat:latest") + Expect(resp.Details.Format).To(Equal("gguf")) + Expect(resp.Capabilities).To(ContainElement("completion")) + }) }) Describe("ListModelsEndpoint", func() { diff --git a/core/http/endpoints/openai/realtime_webrtc.go b/core/http/endpoints/openai/realtime_webrtc.go index 4f13862c8..eca3aa2c6 100644 --- a/core/http/endpoints/openai/realtime_webrtc.go +++ b/core/http/endpoints/openai/realtime_webrtc.go @@ -1,6 +1,9 @@ package openai import ( + "encoding/json" + "io" + "mime" "net/http" "time" @@ -27,6 +30,61 @@ type RealtimeCallResponse struct { SessionID string `json:"session_id"` } +func decodeRealtimeCallRequest(c echo.Context) (RealtimeCallRequest, bool, error) { + var req RealtimeCallRequest + mediaType := "" + contentType := c.Request().Header.Get(echo.HeaderContentType) + if contentType != "" { + var err error + mediaType, _, err = mime.ParseMediaType(contentType) + if err != nil { + return req, false, err + } + } + + switch mediaType { + case echo.MIMEMultipartForm: + if err := c.Request().ParseMultipartForm(32 << 20); err != nil { + return req, true, err + } + req.SDP = c.FormValue("sdp") + var session struct { + Model string `json:"model"` + LocalAIAssistant bool `json:"localai_assistant,omitempty"` + } + if err := json.Unmarshal([]byte(c.FormValue("session")), &session); err != nil { + return req, true, err + } + req.Model = session.Model + req.LocalAIAssistant = session.LocalAIAssistant + return req, true, nil + case "application/sdp": + sdp, err := readRealtimeSDP(c.Request().Body) + req.SDP = sdp + req.Model = c.QueryParam("model") + return req, true, err + default: + err := c.Bind(&req) + return req, false, err + } +} + +func readRealtimeSDP(body io.Reader) (string, error) { + data, err := io.ReadAll(body) + return string(data), err +} + +func writeRealtimeCallResponse(c echo.Context, plainSDPResponse bool, sdp, sessionID string) error { + if plainSDPResponse { + return c.Blob(http.StatusCreated, "application/sdp", []byte(sdp)) + } + + return c.JSON(http.StatusCreated, RealtimeCallResponse{ + SDP: sdp, + SessionID: sessionID, + }) +} + // RealtimeCalls handles POST /v1/realtime/calls for WebRTC signaling. func RealtimeCalls(application *application.Application) echo.HandlerFunc { se, settingEngineErr := webRTCSettingEngine(application.ApplicationConfig()) @@ -38,8 +96,8 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { if settingEngineErr != nil { return c.JSON(http.StatusInternalServerError, map[string]string{"error": settingEngineErr.Error()}) } - var req RealtimeCallRequest - if err := c.Bind(&req); err != nil { + req, plainSDPResponse, err := decodeRealtimeCallRequest(c) + if err != nil { return c.JSON(http.StatusBadRequest, map[string]string{"error": "invalid request body"}) } if req.SDP == "" { @@ -189,10 +247,7 @@ func RealtimeCalls(application *application.Application) echo.HandlerFunc { runRealtimeSession(application, transport, req.Model, evaluator, opts) }() - return c.JSON(http.StatusCreated, RealtimeCallResponse{ - SDP: localDesc.SDP, - SessionID: sessionID, - }) + return writeRealtimeCallResponse(c, plainSDPResponse, localDesc.SDP, sessionID) } } diff --git a/core/http/endpoints/openai/realtime_webrtc_request_test.go b/core/http/endpoints/openai/realtime_webrtc_request_test.go new file mode 100644 index 000000000..187555329 --- /dev/null +++ b/core/http/endpoints/openai/realtime_webrtc_request_test.go @@ -0,0 +1,91 @@ +package openai + +import ( + "bytes" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/textproto" + + "github.com/labstack/echo/v4" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("decodeRealtimeCallRequest", func() { + It("decodes the legacy JSON request", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", bytes.NewBufferString(`{"sdp":"offer","model":"voice","localai_assistant":true}`)) + request.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeFalse()) + }) + + It("decodes the OpenAI multipart request", func() { + var body bytes.Buffer + writer := multipart.NewWriter(&body) + sdpHeader := make(textproto.MIMEHeader) + sdpHeader.Set("Content-Disposition", `form-data; name="sdp"`) + sdpHeader.Set("Content-Type", "application/sdp") + sdpPart, err := writer.CreatePart(sdpHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sdpPart.Write([]byte("offer")) + Expect(err).NotTo(HaveOccurred()) + sessionHeader := make(textproto.MIMEHeader) + sessionHeader.Set("Content-Disposition", `form-data; name="session"`) + sessionHeader.Set("Content-Type", echo.MIMEApplicationJSON) + sessionPart, err := writer.CreatePart(sessionHeader) + Expect(err).NotTo(HaveOccurred()) + _, err = sessionPart.Write([]byte(`{"type":"realtime","model":"voice","localai_assistant":true}`)) + Expect(err).NotTo(HaveOccurred()) + Expect(writer.Close()).To(Succeed()) + + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", &body) + request.Header.Set(echo.HeaderContentType, writer.FormDataContentType()) + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice", LocalAIAssistant: true})) + Expect(plainSDPResponse).To(BeTrue()) + }) + + It("decodes a raw SDP request with the model query parameter", func() { + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls?model=voice", bytes.NewBufferString("offer")) + request.Header.Set(echo.HeaderContentType, "application/sdp") + + req, plainSDPResponse, err := decodeRealtimeCallRequest(echo.New().NewContext(request, httptest.NewRecorder())) + + Expect(err).NotTo(HaveOccurred()) + Expect(req).To(Equal(RealtimeCallRequest{SDP: "offer", Model: "voice"})) + Expect(plainSDPResponse).To(BeTrue()) + }) +}) + +var _ = Describe("writeRealtimeCallResponse", func() { + It("writes the bare SDP answer for OpenAI request formats", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, true, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal("application/sdp")) + Expect(response.Body.String()).To(Equal("answer")) + }) + + It("preserves the JSON response for legacy requests", func() { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodPost, "/v1/realtime/calls", nil) + context := echo.New().NewContext(request, response) + + Expect(writeRealtimeCallResponse(context, false, "answer", "session-id")).To(Succeed()) + + Expect(response.Code).To(Equal(http.StatusCreated)) + Expect(response.Header().Get(echo.HeaderContentType)).To(Equal(echo.MIMEApplicationJSON)) + Expect(response.Body.String()).To(MatchJSON(`{"sdp":"answer","session_id":"session-id"}`)) + }) +}) diff --git a/core/http/middleware/request.go b/core/http/middleware/request.go index 1599ef05c..080a0b73c 100644 --- a/core/http/middleware/request.go +++ b/core/http/middleware/request.go @@ -141,6 +141,12 @@ func (re *RequestExtractor) SetModelAndConfig(initializer func() schema.LocalAIR } modelName := input.ModelName(nil) + // Ollama-compat /api/tags appends ":latest" to untagged names. + // Strip it for lookup so the listed name works on /api/chat, + // /v1/chat/completions, and the other model-bearing endpoints. + if strings.HasSuffix(modelName, ":latest") { + modelName = strings.TrimSuffix(modelName, ":latest") + } cfg, err := re.modelConfigLoader.LoadModelConfigFileByNameDefaultOptions(modelName, re.applicationConfig) if err != nil { diff --git a/core/http/middleware/request_config_revision_test.go b/core/http/middleware/request_config_revision_test.go index 419ae8e04..dec7bc04d 100644 --- a/core/http/middleware/request_config_revision_test.go +++ b/core/http/middleware/request_config_revision_test.go @@ -115,8 +115,8 @@ var _ = Describe("Model config revision seen by inference requests", func() { Expect(admin.LoadModelConfigsFromPath(modelDir, appConfig.ToConfigLoaderOptions()...)).To(Succeed()) loaded, ok := admin.GetModelConfig("test-model") Expect(ok).To(BeTrue()) - adminRevision, err := config.ModelConfigRevision(&loaded) - Expect(err).ToNot(HaveOccurred()) + adminRevision := loaded.PersistedConfigRevision() + Expect(adminRevision).ToNot(BeEmpty()) Expect(revisionFor(`{"model":"test-model","temperature":0.7,"messages":[{"role":"user","content":"hi"}]}`)). To(Equal(adminRevision)) diff --git a/core/http/middleware/request_test.go b/core/http/middleware/request_test.go index 1b00c7f02..afaf8d9c8 100644 --- a/core/http/middleware/request_test.go +++ b/core/http/middleware/request_test.go @@ -82,6 +82,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(resp.Error.Message).To(ContainSubstring("not found")) Expect(resp.Error.Type).To(Equal("invalid_request_error")) }) + + It("still 404s when :latest is appended to an unknown model", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"nonexistent-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusNotFound)) + }) }) Context("when the model exists as a config file", func() { @@ -97,6 +104,13 @@ var _ = Describe("SetModelAndConfig middleware", func() { Expect(rec.Code).To(Equal(http.StatusOK)) }) + + It("accepts the Ollama :latest tag that /api/tags appends", func() { + rec := postJSON(app, "/v1/chat/completions", + `{"model":"test-model:latest","messages":[{"role":"user","content":"hi"}]}`) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) }) Context("when the model exists as a pre-loaded config", func() { diff --git a/core/http/react-ui/e2e/models-cluster-fit.spec.js b/core/http/react-ui/e2e/models-cluster-fit.spec.js new file mode 100644 index 000000000..9c8916ca4 --- /dev/null +++ b/core/http/react-ui/e2e/models-cluster-fit.spec.js @@ -0,0 +1,120 @@ +import { test, expect } from "./coverage-fixtures.js"; + +// On a distributed controller the models run on the workers, so every "will +// this fit" answer on this page is about their hardware. The controller is +// usually a GPU-less pod: sized against it, a cluster of A100s is told it can +// only run the smallest CPU build. + +const GB = 1024 * 1024 * 1024; + +const MODELS = [ + { name: "big-gpu-model", description: "Needs a real GPU", backend: "vllm", installed: false, tags: ["chat"] }, +]; + +// 40GB: far past the controller's 8GB of RAM, comfortably inside one 80GB card. +const ESTIMATES = { + "big-gpu-model": { + sizeBytes: 40 * GB, + sizeDisplay: "40.0 GB", + estimates: { 8192: { vramBytes: 40 * GB, vramDisplay: "40.0 GB" } }, + }, +}; + +// The controller as Argus actually runs it: 8GB of system RAM, no GPU. +const CONTROLLER_ONLY = { + type: "ram", + available: true, + gpus: [], + aggregate: { total_memory: 8 * GB, used_memory: 2 * GB, free_memory: 6 * GB, gpu_count: 0 }, +}; + +const WITH_CLUSTER = { + ...CONTROLLER_ONLY, + cluster: { + enabled: true, + node_id: "n-1", + node_name: "dgx-01", + total_memory: 80 * GB, + is_gpu: true, + node_count: 4, + }, +}; + +async function mockModels(page, resources) { + await page.route("**/api/models*", (route) => + route.fulfill({ + contentType: "application/json", + body: JSON.stringify({ + models: MODELS, + allBackends: ["vllm"], + allTags: ["chat"], + availableModels: MODELS.length, + installedModels: 3, + totalPages: 1, + currentPage: 1, + }), + }), + ); + await page.route("**/api/models/estimate/*", (route) => { + const name = decodeURIComponent(new URL(route.request().url()).pathname.split("/").pop()); + return route.fulfill({ contentType: "application/json", body: JSON.stringify(ESTIMATES[name] || {}) }); + }); + await page.route("**/api/resources", (route) => + route.fulfill({ contentType: "application/json", body: JSON.stringify(resources) }), + ); +} + +const railItems = (page) => page.locator('[data-testid="discover-rail-item"]'); +const railItem = (page, name) => page.locator(`[data-entity="${name}"]`); +const railReady = (page) => expect(railItems(page).first()).toBeVisible({ timeout: 20_000 }); +const PANE = '[data-testid="discover-pane"]'; + +test.describe("Models gallery - cluster-aware fit", () => { + test("a model that only a worker can hold is not called too large", async ({ page }) => { + await mockModels(page, WITH_CLUSTER); + await page.goto("/app/models"); + + await railReady(page); + + // The whole defect in one assertion: 40GB against a 4-node cluster whose + // largest card holds 80GB. + await expect(railItem(page, "big-gpu-model")).toContainText("fits", { timeout: 20_000 }); + await expect(railItem(page, "big-gpu-model")).not.toContainText("too large"); + }); + + test("the fit verdict names the node it belongs to", async ({ page }) => { + await mockModels(page, WITH_CLUSTER); + await page.goto("/app/models"); + + await railReady(page); + await railItem(page, "big-gpu-model").click(); + // Wait for the detail itself: until it renders, the pane still holds the + // zero-state hero, which names the node for its own reasons. + await expect(page.locator(PANE).getByText("40.0 GB")).toBeVisible({ timeout: 20_000 }); + + // The headroom this model has is headroom SOMEWHERE, and the stat says + // where rather than leaving it to read as this machine's. + await expect(page.locator(PANE)).toContainText(/headroom on dgx-01/i); + }); + + test("the host summary describes the cluster, not the controller", async ({ page }) => { + await mockModels(page, WITH_CLUSTER); + await page.goto("/app/models"); + + await railReady(page); + // 80 GB is the cluster's best node; 8 GB is this pod's own RAM and must + // not be what the page advertises. + await expect(page.locator(".zero-pane__title")).toContainText("80 GB"); + await expect(page.locator(".zero-pane__title")).not.toContainText("8.00 GB"); + }); + + // Single-node behavior is the fallback every degradation path lands on, so + // it has to stay exactly as it was. + test("without a cluster the verdict is still the local host's", async ({ page }) => { + await mockModels(page, CONTROLLER_ONLY); + await page.goto("/app/models"); + + await railReady(page); + await expect(railItem(page, "big-gpu-model")).toContainText("too large", { timeout: 20_000 }); + }); +}); diff --git a/core/http/react-ui/e2e/scheduling.spec.js b/core/http/react-ui/e2e/scheduling.spec.js index 4d11d8b26..79d781a3f 100644 --- a/core/http/react-ui/e2e/scheduling.spec.js +++ b/core/http/react-ui/e2e/scheduling.spec.js @@ -36,35 +36,80 @@ async function mockScheduling(page, { rules = [rule], nodeList = nodes } = {}) { } test.describe('Scheduling page', () => { - test('groups node labels, collapses the reference, filters forgivingly, and expands results', async ({ page }) => { + // Node labels are only ever needed while writing a rule's node selector, so + // they live in that field rather than in a card standing open above the + // rules whether or not anyone is writing one. + test('keeps no standing label browser on the page', async ({ page }) => { await mockScheduling(page) await page.goto('/app/scheduling') + await expect(page.getByText('llama-3.3')).toBeVisible() - const reference = page.getByTestId('node-label-reference') - await expect(reference.getByText('Falcon GPU')).toBeVisible() - await expect(reference.getByText('No labels')).toBeVisible() - await expect(reference.locator('.scheduling-node-card')).toHaveCount(5) - await expect(reference.getByText('5 of 27 nodes')).toBeVisible() + await expect(page.getByTestId('node-label-reference')).toHaveCount(0) + await expect(page.getByRole('button', { name: /node labels/i })).toHaveCount(0) + await expect(page.locator('.scheduling-node-card')).toHaveCount(0) + // Falcon GPU is a node name, and nothing on this page has a reason to + // enumerate node names until a selector is being filled. + await expect(page.getByText('Falcon GPU')).toHaveCount(0) + }) - const toggle = page.getByRole('button', { name: /node labels/i }) - await expect(toggle).toHaveAttribute('aria-expanded', 'true') - await toggle.click() - await expect(toggle).toHaveAttribute('aria-expanded', 'false') - await expect(reference.getByRole('searchbox')).toBeHidden() - await toggle.click() + test('suggests the cluster\'s own label keys and values as the selector is typed', async ({ page }) => { + await mockScheduling(page) + await page.goto('/app/scheduling') + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() - await reference.getByRole('searchbox').fill('GPU.VENDOR=nvi') - await expect(reference.locator('.scheduling-node-card')).toHaveCount(1) - await expect(reference.getByText('Falcon GPU')).toBeVisible() + const keyInput = page.getByRole('combobox', { name: 'Selector key' }) + await keyInput.click() + const suggestions = page.getByTestId('label-suggestions') + // Every key the cluster reports, before a single character is typed. + await expect(suggestions.getByRole('option', { name: 'gpu.vendor' })).toBeVisible() + await expect(suggestions.getByRole('option', { name: 'zone' })).toBeVisible() - await reference.getByRole('searchbox').fill('flcn') - await expect(reference.locator('.scheduling-node-card')).toHaveCount(1) - await expect(reference.getByText('Falcon GPU')).toBeVisible() + await keyInput.fill('vend') + await expect(suggestions.getByRole('option')).toHaveCount(1) + await suggestions.getByRole('option', { name: 'gpu.vendor' }).click() + await expect(keyInput).toHaveValue('gpu.vendor') - await reference.getByRole('searchbox').fill('') - await reference.getByRole('button', { name: 'Show 20 more nodes' }).click() - await expect(reference.locator('.scheduling-node-card')).toHaveCount(25) - await expect(reference.getByText('25 of 27 nodes')).toBeVisible() + // Values are scoped to the key being filled, so a selector cannot be built + // out of a pair no node matches. + const valueInput = page.getByRole('combobox', { name: 'Selector value' }) + await valueInput.click() + await expect(suggestions.getByRole('option', { name: 'NVIDIA' })).toBeVisible() + await expect(suggestions.getByRole('option', { name: 'amd' })).toBeVisible() + await expect(suggestions.getByRole('option', { name: 'east' })).toHaveCount(0) + + await valueInput.fill('nvi') + await suggestions.getByRole('option', { name: 'NVIDIA' }).click() + await expect(valueInput).toHaveValue('NVIDIA') + }) + + test('picks a suggestion from the keyboard', async ({ page }) => { + await mockScheduling(page) + await page.goto('/app/scheduling') + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() + + const keyInput = page.getByRole('combobox', { name: 'Selector key' }) + await keyInput.fill('zon') + await keyInput.press('ArrowDown') + await keyInput.press('Enter') + await expect(keyInput).toHaveValue('zone') + // Enter picked the suggestion rather than committing the chip, so the + // half-built pair is still in the inputs. + await expect(page.getByLabel('Node selector').getByText('zone=', { exact: true })).toHaveCount(0) + }) + + // The cluster's vocabulary is a suggestion, never a constraint: an admin + // labelling nodes for a rule they are about to write must still be able to + // type a key no node reports yet. + test('still accepts a label the cluster has never reported', async ({ page }) => { + await mockScheduling(page) + await page.goto('/app/scheduling') + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() + + await page.getByRole('combobox', { name: 'Selector key' }).fill('tenant') + await page.getByRole('combobox', { name: 'Selector value' }).fill('acme') + await page.getByRole('button', { name: 'Add selector' }).click() + + await expect(page.getByLabel('Node selector').getByText('tenant=acme', { exact: true })).toBeVisible() }) test('edits all fields with a locked model and preserves values after a failed save', async ({ page }) => { @@ -113,42 +158,98 @@ test.describe('Scheduling page', () => { await expect(page.getByRole('combobox', { name: '' }).first()).toBeEnabled() }) - test('shows node loading, empty, no-match, and retry states independently from rules', async ({ page }) => { - let attempts = 0 + // The roster feeds suggestions and nothing else now, so failing to load it + // must cost the admin nothing but the hints. + test('leaves the selector fully usable when the node roster fails to load', async ({ page }) => { await page.route('**/api/nodes/scheduling', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([rule]) })) - await page.route('**/api/nodes', async route => { - attempts++ - if (attempts === 1) { - await new Promise(resolve => setTimeout(resolve, 250)) - await route.fulfill({ status: 500, body: 'failed' }) - } else { - await route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }) - } - }) + await page.route('**/api/nodes', route => route.fulfill({ status: 500, body: 'failed' })) await page.goto('/app/scheduling') - await expect(page.getByText('Loading node labels…')).toBeVisible() - await expect(page.getByText('llama-3.3')).toBeVisible() - await expect(page.getByText('Could not load node labels.')).toBeVisible() - await page.getByRole('button', { name: 'Retry loading node labels' }).click() - await expect(page.getByText('No nodes are available yet.')).toBeVisible() - await page.unroute('**/api/nodes') - await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) })) - await page.reload() - await page.getByRole('searchbox', { name: 'Search node labels' }).fill('not-a-real-label') - await expect(page.getByText('No nodes match your search.')).toBeVisible() + // The rules still render: the roster is not on their path. + await expect(page.getByText('llama-3.3')).toBeVisible() + + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() + await page.getByRole('combobox', { name: 'Selector key' }).fill('gpu.vendor') + await page.getByRole('combobox', { name: 'Selector value' }).fill('nvidia') + await page.getByRole('button', { name: 'Add selector' }).click() + + await expect(page.getByLabel('Node selector').getByText('gpu.vendor=nvidia', { exact: true })).toBeVisible() }) - test('uses one node column and accessible rule actions on a narrow viewport', async ({ page }) => { + // A rule may be keyed by an alias, in which case it governs whichever model + // the alias points at. The page has to say which model that is, because the + // rule's own name no longer tells you. + test.describe('rules keyed by a model alias', () => { + const aliasRule = { + model_name: 'production', + target_model: 'llama-3.3', + model_is_alias: true, + node_selector: { tier: 'gpu' }, + min_replicas: 2, + max_replicas: 4, + } + + async function mockAliases(page, aliases = [{ name: 'production', target: 'llama-3.3' }]) { + await page.route('**/api/aliases', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify(aliases), + })) + await page.route('**/api/models/capabilities', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ object: 'list', data: [{ id: 'llama-3.3' }, { id: 'production' }] }), + })) + } + + test('names the model an alias rule governs', async ({ page }) => { + await mockScheduling(page, { rules: [aliasRule] }) + await mockAliases(page) + await page.goto('/app/scheduling') + + await expect(page.getByText('production')).toBeVisible() + await expect(page.locator('.scheduling-rule-target')).toHaveText(/llama-3\.3/) + }) + + test('marks a rule another rule already governs as shadowed', async ({ page }) => { + await mockScheduling(page, { rules: [{ ...aliasRule, shadowed: true }, rule] }) + await mockAliases(page) + await page.goto('/app/scheduling') + + await expect(page.locator('.scheduling-rule-shadowed')).toHaveCount(1) + await expect(page.locator('.scheduling-rule-shadowed')).toContainText('Shadowed') + }) + + test('flags an alias rule that no longer resolves', async ({ page }) => { + await mockScheduling(page, { + rules: [{ model_name: 'orphan', target_model: 'orphan', model_is_alias: true, min_replicas: 1 }], + }) + await mockAliases(page, []) + await page.goto('/app/scheduling') + + await expect(page.locator('.scheduling-rule-target--broken')).toBeVisible() + }) + + test('offers aliases in the model picker, tagged with their target', async ({ page }) => { + await mockScheduling(page) + await mockAliases(page) + await page.goto('/app/scheduling') + await page.getByRole('button', { name: 'Add Scheduling Rule' }).click() + + const picker = page.locator('.searchable-model-select input') + await picker.click() + await expect(page.locator('.sms-hint')).toHaveText('alias of llama-3.3') + + await page.getByRole('option', { name: /production/ }).click() + await expect(page.getByText(/production is an alias for llama-3\.3/)).toBeVisible() + }) + }) + + test('keeps rule actions reachable on a narrow viewport', async ({ page }) => { await page.setViewportSize({ width: 390, height: 844 }) await mockScheduling(page, { nodeList: nodes.slice(0, 2) }) await page.goto('/app/scheduling') - const cards = page.locator('.scheduling-node-card') - const first = await cards.nth(0).boundingBox() - const second = await cards.nth(1).boundingBox() - expect(second.y).toBeGreaterThan(first.y + first.height - 1) - const actions = page.locator('.scheduling-rule-actions') await expect(actions.getByRole('button', { name: 'Edit llama-3.3' })).toBeVisible() await expect(actions.getByRole('button', { name: 'Delete llama-3.3' })).toBeVisible() diff --git a/core/http/react-ui/inline-style-baseline.txt b/core/http/react-ui/inline-style-baseline.txt index 08f851b6e..a08796291 100644 --- a/core/http/react-ui/inline-style-baseline.txt +++ b/core/http/react-ui/inline-style-baseline.txt @@ -1 +1 @@ -519 +514 diff --git a/core/http/react-ui/public/locales/de/models.json b/core/http/react-ui/public/locales/de/models.json index 21e29f782..779e0e7f6 100644 --- a/core/http/react-ui/public/locales/de/models.json +++ b/core/http/react-ui/public/locales/de/models.json @@ -142,7 +142,8 @@ "sha256": "SHA256", "backToAll": "Alle Modelle", "vramAt": "VRAM bei {{context}}", - "headroom": "Spielraum" + "headroom": "Spielraum", + "headroomOn": "Spielraum auf {{node}}" }, "empty": { "title": "Keine Modelle gefunden", @@ -216,6 +217,8 @@ "browsing": "Durchsuchen", "pickHint": "Wähle ein Modell, um die Details zu sehen.", "heroWithRam": "{{ram}} Systemspeicher, {{count}} Modelle in der Galerie.", + "heroWithCluster": "{{vram}} auf {{node}}, dem größten von {{nodes}} Knoten, {{count}} Modelle in der Galerie.", + "heroWithNode": "{{vram}} auf {{node}}, {{count}} Modelle in der Galerie.", "byUseCase": "Oder mit einem Anwendungsfall starten", "pickText": "Chat, Reasoning, Embeddings", "pickVision": "Bilder und Dokumente lesen", diff --git a/core/http/react-ui/public/locales/en/chat.json b/core/http/react-ui/public/locales/en/chat.json index f004cee56..f1c058790 100644 --- a/core/http/react-ui/public/locales/en/chat.json +++ b/core/http/react-ui/public/locales/en/chat.json @@ -33,6 +33,8 @@ "title": "Chat Settings", "manageMode": "Manage mode", "manageModeDesc": "Let this chat install models, switch backends, and edit configs by talking to LocalAI.", + "focusMode": "Focus mode", + "focusModeDesc": "Collapse the sidebar and slim the header while a conversation is active. Esc restores them temporarily; turn this off to keep the full layout.", "systemPrompt": "System Prompt", "systemPromptPlaceholder": "You are a helpful assistant...", "temperature": "Temperature", diff --git a/core/http/react-ui/public/locales/en/models.json b/core/http/react-ui/public/locales/en/models.json index fdc2e397f..ebadd3b3d 100644 --- a/core/http/react-ui/public/locales/en/models.json +++ b/core/http/react-ui/public/locales/en/models.json @@ -152,7 +152,8 @@ "sha256": "SHA256", "backToAll": "All models", "vramAt": "VRAM at {{context}}", - "headroom": "Headroom" + "headroom": "Headroom", + "headroomOn": "Headroom on {{node}}" }, "empty": { "title": "No models found", @@ -232,6 +233,8 @@ "browsing": "Browsing", "pickHint": "Select a model to see its detail.", "heroWithRam": "{{ram}} of system memory, {{count}} models in the gallery.", + "heroWithCluster": "{{vram}} on {{node}}, the largest of {{nodes}} nodes, {{count}} models in the gallery.", + "heroWithNode": "{{vram}} on {{node}}, {{count}} models in the gallery.", "byUseCase": "Or start with a use case", "pickText": "Chat, reasoning, embeddings", "pickVision": "Read images and documents", diff --git a/core/http/react-ui/public/locales/es/models.json b/core/http/react-ui/public/locales/es/models.json index 3d5cdd8df..d833fba55 100644 --- a/core/http/react-ui/public/locales/es/models.json +++ b/core/http/react-ui/public/locales/es/models.json @@ -142,7 +142,8 @@ "sha256": "SHA256", "backToAll": "Todos los modelos", "vramAt": "VRAM a {{context}}", - "headroom": "Margen" + "headroom": "Margen", + "headroomOn": "Margen en {{node}}" }, "empty": { "title": "No se encontraron modelos", @@ -216,6 +217,8 @@ "browsing": "Explorando", "pickHint": "Selecciona un modelo para ver su detalle.", "heroWithRam": "{{ram}} de memoria del sistema, {{count}} modelos en la galería.", + "heroWithCluster": "{{vram}} en {{node}}, el mayor de {{nodes}} nodos, {{count}} modelos en la galería.", + "heroWithNode": "{{vram}} en {{node}}, {{count}} modelos en la galería.", "byUseCase": "O empieza por un caso de uso", "pickText": "Chat, razonamiento, embeddings", "pickVision": "Leer imágenes y documentos", diff --git a/core/http/react-ui/public/locales/id/models.json b/core/http/react-ui/public/locales/id/models.json index 27a42af64..9088647b9 100644 --- a/core/http/react-ui/public/locales/id/models.json +++ b/core/http/react-ui/public/locales/id/models.json @@ -149,7 +149,8 @@ "sha256": "SHA256", "backToAll": "Semua model", "vramAt": "VRAM pada {{context}}", - "headroom": "Sisa ruang" + "headroom": "Sisa ruang", + "headroomOn": "Sisa ruang di {{node}}" }, "empty": { "title": "Model tidak ditemukan", @@ -229,6 +230,8 @@ "browsing": "Menjelajah", "pickHint": "Pilih model untuk melihat detailnya.", "heroWithRam": "Memori sistem {{ram}}, {{count}} model di galeri.", + "heroWithCluster": "{{vram}} di {{node}}, terbesar dari {{nodes}} node, {{count}} model di galeri.", + "heroWithNode": "{{vram}} di {{node}}, {{count}} model di galeri.", "byUseCase": "Atau mulai dari kasus penggunaan", "pickText": "Obrolan, penalaran, embedding", "pickVision": "Membaca gambar dan dokumen", diff --git a/core/http/react-ui/public/locales/it/models.json b/core/http/react-ui/public/locales/it/models.json index 80bd16284..cbc06d22c 100644 --- a/core/http/react-ui/public/locales/it/models.json +++ b/core/http/react-ui/public/locales/it/models.json @@ -142,7 +142,8 @@ "sha256": "SHA256", "backToAll": "Tutti i modelli", "vramAt": "VRAM a {{context}}", - "headroom": "Margine" + "headroom": "Margine", + "headroomOn": "Margine su {{node}}" }, "empty": { "title": "Nessun modello trovato", @@ -216,6 +217,8 @@ "browsing": "Esplorazione", "pickHint": "Seleziona un modello per vederne i dettagli.", "heroWithRam": "{{ram}} di memoria di sistema, {{count}} modelli nella galleria.", + "heroWithCluster": "{{vram}} su {{node}}, il più grande di {{nodes}} nodi, {{count}} modelli nella galleria.", + "heroWithNode": "{{vram}} su {{node}}, {{count}} modelli nella galleria.", "byUseCase": "Oppure parti da un caso d’uso", "pickText": "Chat, ragionamento, embedding", "pickVision": "Leggere immagini e documenti", diff --git a/core/http/react-ui/public/locales/ko/models.json b/core/http/react-ui/public/locales/ko/models.json index b4f1a4656..8ed38bf0f 100644 --- a/core/http/react-ui/public/locales/ko/models.json +++ b/core/http/react-ui/public/locales/ko/models.json @@ -148,7 +148,8 @@ "sha256": "SHA256", "backToAll": "모든 모델", "vramAt": "{{context}}에서의 VRAM", - "headroom": "여유 공간" + "headroom": "여유 공간", + "headroomOn": "{{node}}의 여유 공간" }, "empty": { "title": "모델을 찾을 수 없습니다", @@ -200,6 +201,8 @@ "browsing": "둘러보기", "pickHint": "모델을 선택하면 상세 정보가 표시됩니다.", "heroWithRam": "시스템 메모리 {{ram}}, 갤러리에 모델 {{count}}개.", + "heroWithCluster": "{{nodes}}개 노드 중 가장 큰 {{node}}에 {{vram}}, 갤러리에 모델 {{count}}개.", + "heroWithNode": "{{node}}에 {{vram}}, 갤러리에 모델 {{count}}개.", "byUseCase": "또는 용도로 시작하기", "pickText": "채팅, 추론, 임베딩", "pickVision": "이미지와 문서 읽기", diff --git a/core/http/react-ui/public/locales/pt-BR/models.json b/core/http/react-ui/public/locales/pt-BR/models.json index 9bcbdcffd..c354e89ae 100644 --- a/core/http/react-ui/public/locales/pt-BR/models.json +++ b/core/http/react-ui/public/locales/pt-BR/models.json @@ -152,7 +152,8 @@ "sha256": "SHA256", "backToAll": "Todos os modelos", "vramAt": "VRAM em {{context}}", - "headroom": "Margem de sobra" + "headroom": "Margem de sobra", + "headroomOn": "Folga em {{node}}" }, "empty": { "title": "Nenhum modelo encontrado", @@ -232,6 +233,8 @@ "browsing": "Explorando", "pickHint": "Selecione um modelo para ver seus detalhes.", "heroWithRam": "{{ram}} de memória do sistema, {{count}} modelos na galeria.", + "heroWithCluster": "{{vram}} em {{node}}, o maior de {{nodes}} nós, {{count}} modelos na galeria.", + "heroWithNode": "{{vram}} em {{node}}, {{count}} modelos na galeria.", "byUseCase": "Ou comece por um caso de uso", "pickText": "Chat, raciocínio, embeddings", "pickVision": "Leia imagens e documentos", diff --git a/core/http/react-ui/public/locales/zh-CN/models.json b/core/http/react-ui/public/locales/zh-CN/models.json index 181467b6a..667009901 100644 --- a/core/http/react-ui/public/locales/zh-CN/models.json +++ b/core/http/react-ui/public/locales/zh-CN/models.json @@ -142,7 +142,8 @@ "sha256": "SHA256", "backToAll": "全部模型", "vramAt": "{{context}} 时显存", - "headroom": "剩余显存" + "headroom": "剩余显存", + "headroomOn": "{{node}} 上的余量" }, "empty": { "title": "未找到模型", @@ -216,6 +217,8 @@ "browsing": "浏览中", "pickHint": "选择一个模型以查看详情。", "heroWithRam": "{{ram}} 系统内存，图库中有 {{count}} 个模型。", + "heroWithCluster": "{{node}} 上 {{vram}}，为 {{nodes}} 个节点中最大，图库中有 {{count}} 个模型。", + "heroWithNode": "{{node}} 上 {{vram}}，图库中有 {{count}} 个模型。", "byUseCase": "或从用途开始", "pickText": "对话、推理、向量", "pickVision": "读取图像与文档", diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index c7a2553db..679e16522 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -2697,116 +2697,82 @@ select.input { cursor: not-allowed; } -.scheduling-node-reference { - margin-bottom: var(--spacing-md); - overflow: hidden; +/* Key-value chip builder (node selectors, node labels) */ +.kvchips__chips { + display: flex; + flex-wrap: wrap; + gap: 4px; + margin-bottom: var(--spacing-xs); } -.scheduling-node-reference__toggle { +.kvchips__chip { align-items: center; - background: transparent; - border: 0; + background: var(--color-bg-tertiary); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-sm); + display: inline-flex; + font-family: var(--font-mono); + font-size: 0.75rem; + gap: 4px; + padding: 2px 8px; +} + +.kvchips__chip-remove { + background: none; + border: none; + color: var(--color-text-muted); + cursor: pointer; + font-size: 0.625rem; + padding: 0; +} + +.kvchips__row { + align-items: stretch; + display: flex; + gap: var(--spacing-xs); + position: relative; +} + +.kvchips__add { + min-height: 36px; +} + +/* Anchored to the input row so the list covers what follows the field rather + than pushing the rest of the form down as the user types. */ +.kvchips__suggestions { + background: var(--color-bg-secondary); + border: 1px solid var(--color-border); + border-radius: var(--radius-sm); + box-shadow: var(--shadow-md); + left: 0; + list-style: none; + margin: 4px 0 0; + max-height: 220px; + overflow-y: auto; + padding: 4px; + position: absolute; + right: 0; + top: 100%; + z-index: 20; +} + +.kvchips__suggestion { + background: none; + border: none; + border-radius: var(--radius-sm); color: var(--color-text-primary); cursor: pointer; - display: flex; - font: inherit; - font-weight: var(--font-weight-semibold); - justify-content: space-between; - padding: var(--spacing-md); + display: block; + font-family: var(--font-mono); + font-size: 0.8125rem; + padding: var(--spacing-xs) var(--spacing-sm); text-align: left; width: 100%; } -.scheduling-node-reference__toggle:focus-visible { - outline: 2px solid var(--color-primary); - outline-offset: -2px; -} - -.scheduling-node-reference__content { - border-top: 1px solid var(--color-border-subtle); - padding: var(--spacing-md); -} - -.scheduling-node-reference__content > .text-note { - margin: 0 0 var(--spacing-sm); -} - -.scheduling-node-toolbar { - align-items: center; - display: flex; - gap: var(--spacing-md); - margin-bottom: var(--spacing-md); -} - -.scheduling-node-toolbar .input { - flex: 1; -} - -.scheduling-node-toolbar .text-meta { - flex: none; -} - -.scheduling-node-grid { - display: grid; - gap: var(--spacing-sm); - grid-template-columns: repeat(auto-fit, minmax(240px, 1fr)); -} - -.scheduling-node-card { +.kvchips__suggestion:hover, +.kvchips__suggestion--active { background: var(--color-bg-tertiary); - border: 1px solid var(--color-border-subtle); - border-radius: var(--radius-md); - min-width: 0; - padding: var(--spacing-sm); -} - -.scheduling-node-card__header { - align-items: center; - display: flex; - gap: var(--spacing-sm); - justify-content: space-between; - margin-bottom: var(--spacing-xs); -} - -.scheduling-node-status { - color: var(--color-text-muted); - font-size: 0.75rem; - text-transform: capitalize; -} - -.scheduling-node-status--online, -.scheduling-node-status--ready, -.scheduling-node-status--healthy { - color: var(--color-success); -} - -.scheduling-node-labels { - display: flex; - flex-wrap: wrap; - gap: 4px; -} - -.scheduling-node-label { - border: 1px solid var(--color-border-subtle); - border-radius: var(--radius-sm); - font-family: var(--font-mono); - font-size: 0.75rem; - overflow-wrap: anywhere; - padding: 2px 6px; -} - -.scheduling-node-message { - align-items: center; - color: var(--color-text-muted); - display: flex; - gap: var(--spacing-sm); - justify-content: center; - margin: var(--spacing-md) 0; - text-align: center; -} - -.scheduling-show-more { - margin-top: var(--spacing-md); } .scheduling-rule-actions { @@ -2815,17 +2781,32 @@ select.input { justify-content: flex-end; } +/* Second line of a rule's Model cell: the model an alias-keyed rule currently + governs. The cell itself is bold, so the weight is reset here rather than + inherited. */ +.scheduling-rule-target { + font-weight: 400; +} + +.scheduling-rule-target--broken { + font-weight: 400; + color: var(--color-warning); +} + +/* Status pill for a rule another rule already governs, so it has no effect. + Mirrors the unsatisfiable pill's shape. */ +.scheduling-rule-shadowed { + display: inline-block; + font-size: var(--text-xs); + padding: 2px 8px; + border-radius: var(--radius-sm); + font-weight: 600; + background: var(--color-bg-tertiary); + border: 1px solid var(--color-warning); + color: var(--color-warning); +} + @media (max-width: 640px) { - .scheduling-node-grid { - grid-template-columns: minmax(0, 1fr); - } - - .scheduling-node-toolbar { - align-items: stretch; - flex-direction: column; - gap: var(--spacing-xs); - } - .scheduling-rule-actions { width: 100%; } diff --git a/core/http/react-ui/src/components/SearchableModelSelect.jsx b/core/http/react-ui/src/components/SearchableModelSelect.jsx index 3d920fa4d..f63902956 100644 --- a/core/http/react-ui/src/components/SearchableModelSelect.jsx +++ b/core/http/react-ui/src/components/SearchableModelSelect.jsx @@ -7,7 +7,10 @@ import { useModels } from '../hooks/useModels' // query isn't treated as a chosen value. After a commit the field is cleared, // matching the add-and-clear flow. Default false keeps the as-you-type // behaviour single-value editors rely on. -export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false }) { +// hints: optional { [modelId]: string } shown as muted text beside an entry and +// searchable along with the name. Used to mark aliases with the model they +// point at, so a picker that lists both can tell them apart. +export default function SearchableModelSelect({ value, onChange, capability, placeholder = 'Type or select a model...', style, commitOnly = false, hints = {} }) { const { models, loading } = useModels(capability) const [query, setQuery] = useState('') const [open, setOpen] = useState(false) @@ -29,8 +32,10 @@ export default function SearchableModelSelect({ value, onChange, capability, pla return () => document.removeEventListener('mousedown', handler) }, []) + const needle = query.toLowerCase() const filtered = models.filter(m => - m.id.toLowerCase().includes(query.toLowerCase()) + m.id.toLowerCase().includes(needle) || + (hints[m.id] || '').toLowerCase().includes(needle) ) // Which item Enter will select — matches SearchableSelect behavior @@ -126,6 +131,11 @@ export default function SearchableModelSelect({ value, onChange, capability, pla color: var(--color-primary); font-weight: 600; } + .sms-hint { + color: var(--color-text-muted); + font-size: 0.75rem; + flex-shrink: 0; + } .sms-empty { padding: 8px 10px; font-size: 0.8125rem; @@ -172,6 +182,9 @@ export default function SearchableModelSelect({ value, onChange, capability, pla }} > {m.id} + {hints[m.id] && ( + {hints[m.id]} + )} {isEnterTarget && ( ↵ )} diff --git a/core/http/react-ui/src/components/nodes/KeyValueChips.jsx b/core/http/react-ui/src/components/nodes/KeyValueChips.jsx index 8dcc926c2..1d6baba45 100644 --- a/core/http/react-ui/src/components/nodes/KeyValueChips.jsx +++ b/core/http/react-ui/src/components/nodes/KeyValueChips.jsx @@ -1,4 +1,6 @@ -import { useState } from 'react' +import { useState, useRef, useEffect } from 'react' + +import { suggestKeys, suggestValues } from '../../utils/nodeLabelSuggestions' /** * Controlled chip-builder for { key: value } maps. Replaces the prior @@ -11,6 +13,14 @@ import { useState } from 'react' * labels editor). The component just renders chips and a key/value input * row. * + * With `suggestions` it also completes what the user types against the + * vocabulary the cluster actually uses. That is where label discovery lives on + * the scheduling page: labels only matter while a selector is being written, + * so browsing them belongs in the field rather than in a card standing open + * above the rules. The suggestions are never a constraint - a key no node + * reports yet still commits as typed, which is the workflow of writing a rule + * before labelling the nodes for it. + * * Props: * pairs - current map of key -> value * onAdd(k,v) - called when the user adds a pair (parent handles dedup @@ -18,45 +28,119 @@ import { useState } from 'react' * onRemove(k) - called when a chip's × is clicked * placeholderKey, placeholderValue - input hints * ariaLabel - accessible name for the section + * ariaLabelKey, ariaLabelValue - accessible names for the two inputs + * addLabel - accessible name for the commit button + * suggestions - label index from utils/nodeLabelSuggestions; omit for none */ -export default function KeyValueChips({ pairs, onAdd, onRemove, placeholderKey = 'key', placeholderValue = 'value', ariaLabel }) { +export default function KeyValueChips({ + pairs, onAdd, onRemove, + placeholderKey = 'key', placeholderValue = 'value', + ariaLabel, ariaLabelKey = 'Key', ariaLabelValue = 'Value', + addLabel = 'Add', suggestions, +}) { const [k, setK] = useState('') const [v, setV] = useState('') + // Which input owns the open list, and which of its options is armed for + // Enter. -1 means the user is typing free text and Enter should commit the + // pair rather than pick anything. + const [openField, setOpenField] = useState(null) + const [active, setActive] = useState(-1) + const rowRef = useRef(null) + + const entries = pairs ? Object.entries(pairs) : [] + + const options = !suggestions || !openField + ? [] + : openField === 'key' + ? suggestKeys(suggestions, k, entries.map(([key]) => key)) + : suggestValues(suggestions, k.trim(), v) + + // A click anywhere else is a dismissal. Without this the list survives the + // user moving on to the rest of the form and covers it. + useEffect(() => { + if (!openField) return undefined + const onDocumentPointerDown = (event) => { + if (!rowRef.current?.contains(event.target)) setOpenField(null) + } + document.addEventListener('mousedown', onDocumentPointerDown) + return () => document.removeEventListener('mousedown', onDocumentPointerDown) + }, [openField]) const add = () => { const key = k.trim() if (!key) return onAdd(key, v.trim()) setK(''); setV('') - } - const onKeyDown = (e) => { - if (e.key === 'Enter') { e.preventDefault(); add() } + setOpenField(null); setActive(-1) } - const entries = pairs ? Object.entries(pairs) : [] + const pick = (field, option) => { + if (field === 'key') setK(option) + else setV(option) + setOpenField(null) + setActive(-1) + } + + const onKeyDown = (field) => (e) => { + const open = openField === field && options.length > 0 + if (e.key === 'ArrowDown' && open) { + e.preventDefault() + setActive(current => (current + 1) % options.length) + return + } + if (e.key === 'ArrowUp' && open) { + e.preventDefault() + setActive(current => (current <= 0 ? options.length - 1 : current - 1)) + return + } + if (e.key === 'Escape' && openField) { + e.preventDefault() + setOpenField(null) + setActive(-1) + return + } + if (e.key === 'Enter') { + e.preventDefault() + // Enter completes the suggestion the user armed, and commits the pair + // otherwise. Committing a half-typed key because a list happened to be + // open is the error this ordering avoids. + if (open && active >= 0) pick(field, options[active]) + else add() + } + } + + const listId = 'kvchips-suggestions' + const inputProps = (field, value, setValue, placeholder, label) => ({ + className: 'input flex-1', + type: 'text', + role: suggestions ? 'combobox' : undefined, + 'aria-expanded': suggestions ? openField === field : undefined, + 'aria-controls': suggestions && openField === field ? listId : undefined, + 'aria-autocomplete': suggestions ? 'list' : undefined, + 'aria-label': label, + placeholder, + value, + onChange: (e) => { + setValue(e.target.value) + if (suggestions) { setOpenField(field); setActive(-1) } + }, + onFocus: () => { if (suggestions) { setOpenField(field); setActive(-1) } }, + onKeyDown: onKeyDown(field), + }) + return (
{entries.length > 0 && ( -
+
{entries.map(([key, val]) => ( - + {key}={val} @@ -64,32 +148,35 @@ export default function KeyValueChips({ pairs, onAdd, onRemove, placeholderKey = ))}
)} -
- setK(e.target.value)} - onKeyDown={onKeyDown} - /> - setV(e.target.value)} - onKeyDown={onKeyDown} - /> +
+ + + {options.length > 0 && ( +
    + {options.map((option, index) => ( +
  • + +
    • + ))} +
+ )}
) diff --git a/core/http/react-ui/src/hooks/useRecommendedModels.js b/core/http/react-ui/src/hooks/useRecommendedModels.js index ca6090177..c970c7625 100644 --- a/core/http/react-ui/src/hooks/useRecommendedModels.js +++ b/core/http/react-ui/src/hooks/useRecommendedModels.js @@ -1,6 +1,7 @@ import { useState, useEffect } from 'react' import { modelsApi } from '../utils/api' import { useResources } from './useResources' +import { modelBudget } from '../utils/modelBudget' // Data-driven "recommended for your hardware" model picks. The gallery exposes // no popularity/download signal and the list response carries no size, so we: @@ -21,13 +22,21 @@ const DEFAULT_CTX = 4096 export const isNvfp4Name = (name) => /nvfp4/i.test(name || '') export function hasNvidiaGpu(resources) { + // A distributed controller has no GPUs of its own, so the question is + // whether any worker does. The registry reports the cluster's best node, + // and it is that node these picks have to run on. + if (resources?.cluster?.enabled) return !!resources.cluster.is_gpu return Array.isArray(resources?.gpus) && resources.gpus.some(g => (g?.vendor || '').toLowerCase() === 'nvidia') } export function recommendTier(resources) { - const isGpu = resources?.type === 'gpu' - const vram = resources?.aggregate?.total_memory || 0 + // Same reading the models page sizes against: the cluster's largest node in + // distributed mode, the local host otherwise. Ranked against the controller, + // a fleet of A100s was recommended the models a GPU-less pod could run. + const budget = modelBudget(resources) + const isGpu = budget.scope === 'cluster' ? budget.hasGpu : resources?.type === 'gpu' + const vram = budget.totalMemory if (!isGpu || vram <= 0) return { id: 'cpu', vram: 0 } if (vram < 8 * GB) return { id: 'gpu-small', vram } if (vram < 24 * GB) return { id: 'gpu-mid', vram } diff --git a/core/http/react-ui/src/pages/Chat.jsx b/core/http/react-ui/src/pages/Chat.jsx index 5866a2514..d799efba0 100644 --- a/core/http/react-ui/src/pages/Chat.jsx +++ b/core/http/react-ui/src/pages/Chat.jsx @@ -21,6 +21,8 @@ import { useOperations } from '../hooks/useOperations' import { relativeTime } from '../utils/format' import { copyToClipboard } from '../utils/clipboard' +const FOCUS_MODE_KEY = 'localai_chat_focus_mode' + function getLastMessagePreview(chat) { if (!chat.history || chat.history.length === 0) return '' for (let i = chat.history.length - 1; i >= 0; i--) { @@ -405,12 +407,20 @@ export default function Chat() { // Focus mode: once a conversation has at least one message we slim the // surrounding chrome (collapse the global app rail, fade non-essential // header items). Esc gives the user back the full chrome for the rest of - // this session. + // this session. The settings drawer offers a persistent opt-out. const isInConversation = (activeChat?.history?.length || 0) > 0 const [focusOverride, setFocusOverride] = useState(false) - const focusActive = isInConversation && !focusOverride + const [focusModeEnabled, setFocusModeEnabled] = useState(() => { + try { return localStorage.getItem(FOCUS_MODE_KEY) !== 'false' } catch (_) { return true } + }) + const focusActive = focusModeEnabled && isInConversation && !focusOverride const prevAppCollapseRef = useRef(null) + const toggleFocusMode = (next) => { + setFocusModeEnabled(next) + try { localStorage.setItem(FOCUS_MODE_KEY, String(next)) } catch (_) {} + } + const artifacts = useMemo( () => canvasMode ? extractCodeArtifacts(activeChat?.history, 'role', 'assistant') : [], [activeChat?.history, canvasMode] @@ -1110,6 +1120,20 @@ export default function Chat() { />
)} +
+
+ + {t('settings.focusMode')} + + + {t('settings.focusModeDesc')} + +
+ +