From 0cd6a564dfe1c20244a8440232834dc14791e6db Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:11:57 +0200 Subject: [PATCH 01/86] chore: :arrow_up: Update leejet/stable-diffusion.cpp to `7f410a3793c5bba8eb198e962ce7a3d6095f9d89` (#12012) :arrow_up: Update leejet/stable-diffusion.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/stablediffusion-ggml/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index 8a5ea0b46..cca0daca6 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=b68d58624d227682eb4b95ef8bcf569cd1311eb5 +STABLEDIFFUSION_GGML_VERSION?=7f410a3793c5bba8eb198e962ce7a3d6095f9d89 CMAKE_ARGS+=-DGGML_MAX_NAME=128 From 4f30cf3d0526a3de6adc4cb55732fbb3d0b11fc6 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:12:09 +0200 Subject: [PATCH 02/86] chore: :arrow_up: Update CrispStrobe/CrispASR to `ab87fba65a4ade1ef30ccaa881458a0914aae557` (#12011) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 3a3538346..521578eb8 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=301acd87b036764973b8bfba71e0a21818036d33 +CRISPASR_VERSION?=ab87fba65a4ade1ef30ccaa881458a0914aae557 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From 1842aff2c5ec9760e4d2b9421a893ded6e72394b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:15:47 +0200 Subject: [PATCH 03/86] chore: :arrow_up: Update 0xShug0/audio.cpp to `5ba81ac54fb071b835680973f8868546b4db372b` (#12010) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 953adba96..87e9cb732 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=efb04233dab73aeee4b2912042a90e7b36329061 +AUDIO_CPP_VERSION?=5ba81ac54fb071b835680973f8868546b4db372b AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From 9704270be49b1cd062a28a9afe3e4610758d2a72 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:15:58 +0200 Subject: [PATCH 04/86] chore: :arrow_up: Update ggml-org/whisper.cpp to `1da4dc82fa7996d4edda05890dca65aeceaafd6d` (#12009) :arrow_up: Update ggml-org/whisper.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/whisper/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/whisper/Makefile b/backend/go/whisper/Makefile index 49c47b8ae..f8ee2bb16 100644 --- a/backend/go/whisper/Makefile +++ b/backend/go/whisper/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # whisper.cpp version WHISPER_REPO?=https://github.com/ggml-org/whisper.cpp -WHISPER_CPP_VERSION?=c44b60b8053bbf2a5c1e014f11323fb3f2485177 +WHISPER_CPP_VERSION?=1da4dc82fa7996d4edda05890dca65aeceaafd6d SO_TARGET?=libgowhisper.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From e11f363a50c90428d0ce38af10bf2f4381d9f3c2 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:16:26 +0200 Subject: [PATCH 05/86] chore: :arrow_up: Update OpenNMT/CTranslate2 ROCm wheel to `v4.8.2` (#12004) :arrow_up: Update OpenNMT/CTranslate2 ROCm wheel Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/python/faster-whisper/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/faster-whisper/install.sh b/backend/python/faster-whisper/install.sh index d5350f700..3e5d97ad8 100755 --- a/backend/python/faster-whisper/install.sh +++ b/backend/python/faster-whisper/install.sh @@ -26,7 +26,7 @@ if [ "x${BUILD_PROFILE}" == "xl4t12" ]; then USE_PIP=true fi -CTRANSLATE2_VERSION=${CTRANSLATE2_VERSION:-v4.7.1} +CTRANSLATE2_VERSION=${CTRANSLATE2_VERSION:-v4.8.2} CTRANSLATE2_ROCM_WHEEL_OS=${CTRANSLATE2_ROCM_WHEEL_OS:-Linux} CTRANSLATE2_ROCM_WHEEL_ARCHIVE="rocm-python-wheels-${CTRANSLATE2_ROCM_WHEEL_OS}.zip" From 3613f29e5c41dcb75915d889e8f68e1d4c823036 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:36:51 +0200 Subject: [PATCH 06/86] chore: :arrow_up: Update ServeurpersoCom/omnivoice.cpp to `2ebbdcf281ce4b632e326011974da5a8b35c6b27` (#12006) :arrow_up: Update ServeurpersoCom/omnivoice.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/omnivoice-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/omnivoice-cpp/Makefile b/backend/go/omnivoice-cpp/Makefile index 92810a5ba..5a66eb208 100644 --- a/backend/go/omnivoice-cpp/Makefile +++ b/backend/go/omnivoice-cpp/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # omnivoice.cpp version OMNIVOICE_REPO?=https://github.com/ServeurpersoCom/omnivoice.cpp -OMNIVOICE_VERSION?=040c8b344d8c670ce1475194751d119b5ef82c78 +OMNIVOICE_VERSION?=2ebbdcf281ce4b632e326011974da5a8b35c6b27 SO_TARGET?=libgomnivoicecpp.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From d463316caf089e37d224e5a4356cdd3afb1bdf52 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 09:37:17 +0200 Subject: [PATCH 07/86] chore: :arrow_up: Update vllm-metal (darwin) to `v0.29.0` (#12007) :arrow_up: Update vllm-project/vllm-metal (darwin) Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/python/vllm/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/python/vllm/install.sh b/backend/python/vllm/install.sh index a4124977c..2079a3b87 100755 --- a/backend/python/vllm/install.sh +++ b/backend/python/vllm/install.sh @@ -119,7 +119,7 @@ if [ "$(uname -s)" = "Darwin" ]; then # can rewrite it. Darwin therefore follows vllm-metal and can lag the Linux # vllm pin (requirements-cublas13-after.txt, bumped independently against # vllm/vllm) until vllm-metal supports a newer vLLM. - VLLM_METAL_VERSION="v0.28.0" + VLLM_METAL_VERSION="v0.29.0" # The coupled vLLM source version is whatever this vllm-metal release builds # against. Derive it from the PINNED tag rather than hardcoding a second value From 9abe2aead5a93bafc2ca3e934d74a626d7d245f7 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 19:05:33 +0200 Subject: [PATCH 08/86] fix(gallery): use native Ministral tool parsing (#12022) The Ministral 3 14B Reasoning entry inherits a Mistral 0.3 prompt and JSON parser. Its name-first tool calls can therefore reach clients as plain text. Use the embedded template and llama.cpp's native tool parser. Document migration for installed configurations, which gallery updates do not rewrite. Assisted-by: Codex:gpt-6 Signed-off-by: Ettore Di Giacinto Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- docs/content/features/openai-functions.md | 30 ++++++++++++++++++++++- gallery/index.yaml | 13 +++++++++- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/docs/content/features/openai-functions.md b/docs/content/features/openai-functions.md index 0ed724daa..5e7a384ca 100644 --- a/docs/content/features/openai-functions.md +++ b/docs/content/features/openai-functions.md @@ -36,6 +36,34 @@ Reasoning content (`...` blocks from DeepSeek R1, Qwen3, Gemma 4, No configuration required - the autoparser detects the tool call format for any `ggml`/`gguf` model that was trained with tool support. +#### Migrating older Ministral 3 configurations + +The `mistralai_ministral-3-14b-reasoning-2512-multimodal` gallery entry uses the model's embedded template and llama.cpp's native tool parser. +Older installations inherit the Mistral 0.3 prompt and JSON parser, which can return tool calls such as `pick_tool{...}` as text. + +For an existing installation, replace the model YAML's `template` and `function` sections with: + +```yaml +template: + use_tokenizer_template: true +function: + disable_no_action: true + automatic_tool_parsing_fallback: true + grammar: + disable: true +``` + +Set `use_jinja:true` in the existing `options` list: + +```yaml +options: + - use_jinja:true +``` + +Remove the inherited `stopwords` list so llama.cpp controls the model's end-of-turn markers. +Keep your model path, `mmproj`, sampling settings, and MCP configuration. Reload the model after saving the YAML. +Gallery changes do not rewrite installed model configurations. + ### vLLM / vLLM Omni The parser must be specified explicitly because vLLM itself doesn't auto-detect one. Pass it via the model `options`: @@ -312,4 +340,4 @@ Grammars and function tools can be used as well in conjunction with vision APIs: ## 💡 Examples -A full e2e example with `docker-compose` is available [here](https://github.com/mudler/LocalAI-examples/tree/main/functions). \ No newline at end of file +A full e2e example with `docker-compose` is available [here](https://github.com/mudler/LocalAI-examples/tree/main/functions). diff --git a/gallery/index.yaml b/gallery/index.yaml index 74610a5b1..104827cc3 100644 --- a/gallery/index.yaml +++ b/gallery/index.yaml @@ -41311,7 +41311,7 @@ sha256: 2740ba9e9b30b09be4282a9a9f617ec43dc47b89aed416cb09b5f698f90783b5 uri: huggingface://unsloth/Ministral-3-14B-Instruct-2512-GGUF/mmproj-F32.gguf - name: mistralai_ministral-3-14b-reasoning-2512-multimodal - url: github:mudler/LocalAI/gallery/mistral-0.3.yaml@master + url: github:mudler/LocalAI/gallery/virtual.yaml@master urls: - https://huggingface.co/mistralai/Ministral-3-14B-Reasoning-2512 - https://huggingface.co/unsloth/Ministral-3-14B-Reasoning-2512-GGUF @@ -41359,8 +41359,19 @@ - vision last_checked: "2026-05-04" overrides: + backend: llama-cpp context_size: 32768 + mmap: true mmproj: llama-cpp/mmproj/mmproj-mistralai_Ministral-3-14B-Reasoning-2512-f32.gguf + options: + - use_jinja:true + template: + use_tokenizer_template: true + function: + disable_no_action: true + automatic_tool_parsing_fallback: true + grammar: + disable: true parameters: model: llama-cpp/models/mistralai_Ministral-3-14B-Reasoning-2512-Q4_K_M.gguf temperature: 0.7 From 35819d9e0de06f90fb3ff3e6616f0b7da13ffa67 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 19:06:43 +0200 Subject: [PATCH 09/86] feat(cli): benchmark configured text models (#12020) * feat(cli): benchmark configured text models Compare model aliases through a running LocalAI server. Report full request latency and server-reported throughput with raw JSON samples. Keep warmups separate and fail before writing results on request errors. Assisted-by: Codex:GPT-6 * fix(cli): satisfy benchmark error checks Explicitly discard errors from buffered report writes, HTTP response cleanup, and test server writes to pass errcheck without changing behavior. Assisted-by: Codex:gpt-6 golangci-lint --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- core/cli/benchmark/benchmark.go | 257 ++++++++++++++++++++++++ core/cli/benchmark/benchmark_test.go | 242 ++++++++++++++++++++++ core/cli/cli.go | 2 + docs/content/features/benchmark.md | 84 ++++++++ docs/content/reference/cli-reference.md | 2 + 5 files changed, 587 insertions(+) create mode 100644 core/cli/benchmark/benchmark.go create mode 100644 core/cli/benchmark/benchmark_test.go create mode 100644 docs/content/features/benchmark.md diff --git a/core/cli/benchmark/benchmark.go b/core/cli/benchmark/benchmark.go new file mode 100644 index 000000000..0ceef79f8 --- /dev/null +++ b/core/cli/benchmark/benchmark.go @@ -0,0 +1,257 @@ +// SPDX-License-Identifier: MIT +// Package benchmark measures text inference through a running LocalAI server. +package benchmark + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "os/signal" + "strings" + "syscall" + "text/tabwriter" + "time" + + "github.com/mudler/LocalAI/pkg/httpclient" +) + +type Command struct { + Models []string `arg:"" required:"" help:"Configured text model names to benchmark sequentially."` + Endpoint string `default:"http://127.0.0.1:8080" help:"LocalAI server URL, optionally ending in /v1."` + APIKey string `name:"api-key" env:"LOCALAI_API_KEY,API_KEY" help:"API key for the server."` + Prompt string `default:"Explain why the sky is blue." help:"User prompt sent with every request."` + MaxTokens int `default:"128" help:"Maximum completion tokens per request."` + Runs int `default:"3" help:"Measured requests per model."` + Warmup int `default:"1" help:"Unmeasured requests before each model's measured runs."` + Timeout time.Duration `default:"5m" help:"Timeout for each request."` + JSON bool `name:"json" help:"Write settings and raw samples as JSON."` +} + +type settings struct { + Endpoint string `json:"endpoint"` + Prompt string `json:"prompt"` + MaxTokens int `json:"max_tokens"` + Runs int `json:"runs"` + Warmup int `json:"warmup"` + Timeout string `json:"timeout"` + Temperature float64 `json:"temperature"` + Stream bool `json:"stream"` +} +type sample struct { + LatencySeconds float64 `json:"latency_seconds"` + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` +} +type modelResult struct { + Model string `json:"model"` + Samples []sample `json:"samples"` + MinSeconds float64 `json:"min_seconds"` + MeanSeconds float64 `json:"mean_seconds"` + MaxSeconds float64 `json:"max_seconds"` + CompletionTokensPerSecond *float64 `json:"completion_tokens_per_second"` +} +type report struct { + Settings settings `json:"settings"` + Results []modelResult `json:"results"` +} + +func (c *Command) Run() error { + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + return c.run(ctx, os.Stdout) +} + +func completionURL(endpoint string) (string, error) { + u, err := url.Parse(endpoint) + if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || strings.Contains(endpoint, "#") { + return "", errors.New("endpoint must be an HTTP(S) URL without credentials, query, or fragment") + } + path := strings.TrimRight(u.Path, "/") + if !strings.HasSuffix(path, "/v1") { + path += "/v1" + } + u.Path = path + "/chat/completions" + u.RawPath = "" + return u.String(), nil +} + +func (c *Command) run(ctx context.Context, out io.Writer) error { + endpoint, err := completionURL(c.Endpoint) + if err != nil { + return err + } + if c.Runs <= 0 || c.Warmup < 0 || c.MaxTokens <= 0 || c.Timeout <= 0 { + return errors.New("runs, max-tokens, and timeout must be positive; warmup must be nonnegative") + } + if strings.TrimSpace(c.Prompt) == "" { + return errors.New("prompt must not be blank") + } + if len(c.Models) == 0 { + return errors.New("at least one model is required") + } + for _, model := range c.Models { + if strings.TrimSpace(model) == "" { + return errors.New("model names must not be blank") + } + } + client := httpclient.NewWithTimeout(c.Timeout) + defer client.CloseIdleConnections() + result := report{Settings: settings{Endpoint: endpoint, Prompt: c.Prompt, MaxTokens: c.MaxTokens, Runs: c.Runs, Warmup: c.Warmup, Timeout: c.Timeout.String()}} + for _, model := range c.Models { + measured := modelResult{Model: model} + for i := 0; i < c.Warmup; i++ { + if _, err := c.request(ctx, client, endpoint, model); err != nil { + return fmt.Errorf("model %q warmup %d: %w", model, i+1, err) + } + } + for i := 0; i < c.Runs; i++ { + s, err := c.request(ctx, client, endpoint, model) + if err != nil { + return fmt.Errorf("model %q run %d: %w", model, i+1, err) + } + measured.Samples = append(measured.Samples, s) + } + measured.summarize() + result.Results = append(result.Results, measured) + } + if err := ctx.Err(); err != nil { + return err + } + // Buffer the complete report so a failed model never leaves partial results. + var buffer bytes.Buffer + if c.JSON { + encoder := json.NewEncoder(&buffer) + encoder.SetIndent("", " ") + if err := encoder.Encode(result); err != nil { + return err + } + } else { + table := tabwriter.NewWriter(&buffer, 0, 4, 2, ' ', 0) + _, _ = fmt.Fprintln(table, "MODEL\tRUNS\tMIN (s)\tMEAN (s)\tMAX (s)\tEND-TO-END TOKENS/s") + for _, r := range result.Results { + throughput := "N/A" + if r.CompletionTokensPerSecond != nil { + throughput = fmt.Sprintf("%.2f", *r.CompletionTokensPerSecond) + } + _, _ = fmt.Fprintf(table, "%s\t%d\t%.4f\t%.4f\t%.4f\t%s\n", r.Model, len(r.Samples), r.MinSeconds, r.MeanSeconds, r.MaxSeconds, throughput) + } + if err := table.Flush(); err != nil { + return err + } + } + _, err = io.Copy(out, &buffer) + return err +} + +func (c *Command) request(ctx context.Context, client *http.Client, endpoint, model string) (sample, error) { + var s sample + body, err := json.Marshal(struct { + Model string `json:"model"` + Messages []map[string]string `json:"messages"` + MaxTokens int `json:"max_tokens"` + Temperature float64 `json:"temperature"` + Stream bool `json:"stream"` + }{Model: model, Messages: []map[string]string{{"role": "user", "content": c.Prompt}}, MaxTokens: c.MaxTokens}) + if err != nil { + return s, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(body)) + if err != nil { + return s, errors.New("cannot create benchmark request") + } + req.Header.Set("Content-Type", "application/json") + if c.APIKey != "" { + req.Header.Set("Authorization", "Bearer "+c.APIKey) + } + start := time.Now() + resp, err := client.Do(req) + if err != nil { + if ctx.Err() != nil { + return s, ctx.Err() + } + if errors.Is(err, context.DeadlineExceeded) { + return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded) + } + if errors.Is(err, httpclient.ErrRedirectBlocked) { + return s, httpclient.ErrRedirectBlocked + } + // Transport errors and server responses can echo credentials. + return s, errors.New("HTTP request failed") + } + defer func() { _ = resp.Body.Close() }() + if resp.StatusCode < 200 || resp.StatusCode >= 300 { + return s, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + var response struct { + Choices []json.RawMessage `json:"choices"` + Usage struct { + PromptTokens *int `json:"prompt_tokens"` + CompletionTokens *int `json:"completion_tokens"` + } `json:"usage"` + Error json.RawMessage `json:"error"` + } + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&response); err != nil { + if ctx.Err() != nil { + return s, ctx.Err() + } + if errors.Is(err, context.DeadlineExceeded) { + return s, fmt.Errorf("request timed out: %w", context.DeadlineExceeded) + } + return s, errors.New("invalid JSON response") + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return s, errors.New("invalid trailing response data") + } + if len(response.Error) > 0 && string(response.Error) != "null" { + var detail struct { + Message string `json:"message"` + } + if json.Unmarshal(response.Error, &detail) == nil && detail.Message != "" { + message := detail.Message + if c.APIKey != "" { + message = strings.ReplaceAll(message, c.APIKey, "[redacted]") + } + return s, fmt.Errorf("server returned an API error: %s", message) + } + return s, errors.New("server returned an API error") + } + if len(response.Choices) == 0 { + return s, errors.New("response contains no choices") + } + s.LatencySeconds = time.Since(start).Seconds() + s.PromptTokens = response.Usage.PromptTokens + s.CompletionTokens = response.Usage.CompletionTokens + if (s.PromptTokens != nil && *s.PromptTokens < 0) || (s.CompletionTokens != nil && *s.CompletionTokens < 0) { + return s, errors.New("response contains negative token counts") + } + return s, nil +} + +func (r *modelResult) summarize() { + r.MinSeconds = r.Samples[0].LatencySeconds + var seconds, tokens float64 + available := true + for _, s := range r.Samples { + seconds += s.LatencySeconds + r.MinSeconds = min(r.MinSeconds, s.LatencySeconds) + r.MaxSeconds = max(r.MaxSeconds, s.LatencySeconds) + if s.CompletionTokens == nil { + available = false + } else { + tokens += float64(*s.CompletionTokens) + } + } + r.MeanSeconds = seconds / float64(len(r.Samples)) + if available && seconds > 0 { + rate := tokens / seconds + r.CompletionTokensPerSecond = &rate + } +} diff --git a/core/cli/benchmark/benchmark_test.go b/core/cli/benchmark/benchmark_test.go new file mode 100644 index 000000000..fcd007f49 --- /dev/null +++ b/core/cli/benchmark/benchmark_test.go @@ -0,0 +1,242 @@ +// SPDX-License-Identifier: MIT +package benchmark + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "testing" + "time" + + "github.com/alecthomas/kong" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +func TestBenchmark(t *testing.T) { RegisterFailHandler(Fail); RunSpecs(t, "Benchmark") } + +var _ = Describe("Benchmark command", func() { + var cmd Command + var output bytes.Buffer + BeforeEach(func() { + cmd = Command{Models: []string{"a"}, Endpoint: "http://127.0.0.1:8080", Prompt: "hello", MaxTokens: 128, Runs: 2, Warmup: 1, Timeout: time.Second, JSON: true} + output.Reset() + }) + It("parses required models and defaults", func() { + var c Command + parser, err := kong.New(&c) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse(nil) + Expect(err).To(HaveOccurred()) + _, err = parser.Parse([]string{"a", "b"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.Models).To(Equal([]string{"a", "b"})) + Expect(c.Endpoint).To(Equal("http://127.0.0.1:8080")) + Expect(c.Runs).To(Equal(3)) + Expect(c.Warmup).To(Equal(1)) + Expect(c.MaxTokens).To(Equal(128)) + Expect(c.Timeout).To(Equal(5 * time.Minute)) + Expect(c.Prompt).NotTo(BeEmpty()) + }) + It("reads API key environment variables in priority order", func() { + for _, key := range []string{"LOCALAI_API_KEY", "API_KEY"} { + value, present := os.LookupEnv(key) + DeferCleanup(func() { + if present { + Expect(os.Setenv(key, value)).To(Succeed()) + } else { + Expect(os.Unsetenv(key)).To(Succeed()) + } + }) + } + Expect(os.Unsetenv("LOCALAI_API_KEY")).To(Succeed()) + Expect(os.Setenv("API_KEY", "fallback")).To(Succeed()) + var c Command + parser, err := kong.New(&c) + Expect(err).NotTo(HaveOccurred()) + _, err = parser.Parse([]string{"a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.APIKey).To(Equal("fallback")) + Expect(os.Setenv("LOCALAI_API_KEY", "preferred")).To(Succeed()) + _, err = parser.Parse([]string{"a"}) + Expect(err).NotTo(HaveOccurred()) + Expect(c.APIKey).To(Equal("preferred")) + }) + DescribeTable("normalizes endpoints", func(input, expected string) { + actual, err := completionURL(input) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(expected)) + }, + Entry("root", "http://localhost:8080", "http://localhost:8080/v1/chat/completions"), Entry("slash", "http://localhost:8080/", "http://localhost:8080/v1/chat/completions"), Entry("v1", "https://example.org/v1/", "https://example.org/v1/chat/completions"), Entry("proxy", "https://example.org/proxy/", "https://example.org/proxy/v1/chat/completions"), Entry("proxy v1", "https://example.org/proxy/v1", "https://example.org/proxy/v1/chat/completions")) + It("posts authenticated requests sequentially and excludes each model's warmup", func() { + var models []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer GinkgoRecover() + Expect(r.Method).To(Equal("POST")) + Expect(r.URL.Path).To(Equal("/proxy/v1/chat/completions")) + Expect(r.Header.Get("Authorization")).To(Equal("Bearer secret")) + Expect(r.Header.Get("Content-Type")).To(Equal("application/json")) + var body map[string]any + Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed()) + Expect(body["temperature"]).To(Equal(float64(0))) + Expect(body["stream"]).To(BeFalse()) + Expect(body["max_tokens"]).To(Equal(float64(128))) + Expect(body["messages"]).To(Equal([]any{map[string]any{"role": "user", "content": "hello"}})) + models = append(models, body["model"].(string)) + _, _ = fmt.Fprintf(w, `{"choices":[{}],"usage":{"prompt_tokens":5,"completion_tokens":%d}}`, len(models)) + })) + defer server.Close() + cmd.Endpoint = server.URL + "/proxy" + cmd.APIKey = "secret" + cmd.Models = []string{"a", "b"} + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + Expect(models).To(Equal([]string{"a", "a", "a", "b", "b", "b"})) + Expect(output.String()).NotTo(ContainSubstring("secret")) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + Expect(result.Results).To(HaveLen(2)) + Expect(result.Settings.Runs).To(Equal(2)) + Expect(result.Settings.Warmup).To(Equal(1)) + Expect(result.Settings.Prompt).To(Equal("hello")) + Expect(result.Settings.Temperature).To(BeZero()) + Expect(result.Settings.Stream).To(BeFalse()) + first := result.Results[0] + Expect(first.Samples).To(HaveLen(2)) + Expect(*first.Samples[0].CompletionTokens).To(Equal(2)) + Expect(*first.Samples[1].CompletionTokens).To(Equal(3)) + Expect(*first.Samples[0].PromptTokens).To(Equal(5)) + Expect(first.MinSeconds).To(BeNumerically(">", 0)) + Expect(first.MeanSeconds).To(BeNumerically(">=", first.MinSeconds)) + Expect(first.MaxSeconds).To(BeNumerically(">=", first.MeanSeconds)) + Expect(*first.CompletionTokensPerSecond).To(BeNumerically("~", 5/(first.Samples[0].LatencySeconds+first.Samples[1].LatencySeconds), 0.001)) + }) + DescribeTable("preserves missing and zero usage", func(usage string, available bool) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = fmt.Fprint(w, `{"choices":[{}]`+usage+`}`) })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.Warmup = 0 + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + if available { + Expect(*result.Results[0].CompletionTokensPerSecond).To(BeZero()) + } else { + Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil()) + } + cmd.JSON = false + output.Reset() + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + if !available { + Expect(output.String()).To(ContainSubstring("N/A")) + } + }, Entry("absent", "", false), Entry("empty", `,"usage":{}`, false), Entry("partial", `,"usage":{"prompt_tokens":0}`, false), Entry("zero", `,"usage":{"prompt_tokens":0,"completion_tokens":0}`, true)) + It("retains API error details while redacting the key", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"error":{"message":"model unavailable: secret"}}`) + })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.APIKey = "secret" + cmd.Warmup = 0 + err := cmd.run(context.Background(), &output) + Expect(err).To(MatchError(ContainSubstring(`model "a" run 1: server returned an API error: model unavailable: [redacted]`))) + Expect(output.Len()).To(BeZero()) + }) + It("marks throughput unavailable when one measured request omits usage", func() { + requests := 0 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ + if requests == 1 { + _, _ = fmt.Fprint(w, `{"choices":[{}],"usage":{"completion_tokens":2}}`) + } else { + _, _ = fmt.Fprint(w, `{"choices":[{}]}`) + } + })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.Warmup = 0 + Expect(cmd.run(context.Background(), &output)).To(Succeed()) + var result report + Expect(json.Unmarshal(output.Bytes(), &result)).To(Succeed()) + Expect(result.Results[0].CompletionTokensPerSecond).To(BeNil()) + Expect(result.Results[0].Samples[1].CompletionTokens).To(BeNil()) + }) + DescribeTable("fails without result output", func(status int, body string) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(status); _, _ = fmt.Fprint(w, body) })) + defer server.Close() + cmd.Endpoint = server.URL + cmd.APIKey = "secret" + err := cmd.run(context.Background(), &output) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`model "a" warmup 1`)) + Expect(err.Error()).NotTo(ContainSubstring("secret")) + Expect(output.Len()).To(BeZero()) + }, Entry("HTTP", 500, `secret`), Entry("API", 200, `{"error":{"message":"secret"}}`), Entry("JSON", 200, `invalid`), Entry("empty choices", 200, `{"choices":[]}`), Entry("trailing JSON", 200, `{"choices":[{}]} {}`), Entry("negative tokens", 200, `{"choices":[{}],"usage":{"completion_tokens":-1}}`)) + It("refuses redirects", func() { + reached := false + target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true })) + defer target.Close() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, target.URL, http.StatusTemporaryRedirect) + })) + defer server.Close() + cmd.Endpoint = server.URL + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(reached).To(BeFalse()) + Expect(output.Len()).To(BeZero()) + }) + It("honors cancellation", func() { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + err := cmd.run(ctx, &output) + Expect(err).To(MatchError(ContainSubstring("context canceled"))) + Expect(output.Len()).To(BeZero()) + }) + It("times out requests", func() { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { <-release })) + defer server.Close() + defer close(release) + cmd.Endpoint = server.URL + cmd.Timeout = 20 * time.Millisecond + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(output.Len()).To(BeZero()) + }) + It("times out while reading a response body", func() { + release := make(chan struct{}) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = fmt.Fprint(w, `{"choices":[`) + w.(http.Flusher).Flush() + <-release + })) + defer server.Close() + defer close(release) + cmd.Endpoint = server.URL + cmd.Timeout = 20 * time.Millisecond + Expect(cmd.run(context.Background(), &output)).To(MatchError(ContainSubstring("request timed out"))) + Expect(output.Len()).To(BeZero()) + }) + It("cancels an active request", func() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { cancel() })) + defer server.Close() + cmd.Endpoint = server.URL + Expect(cmd.run(ctx, &output)).To(MatchError(ContainSubstring("context canceled"))) + Expect(output.Len()).To(BeZero()) + }) + DescribeTable("rejects invalid inputs before requests", func(change func(*Command)) { + reached := false + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { reached = true })) + defer server.Close() + cmd.Endpoint = server.URL + change(&cmd) + Expect(cmd.run(context.Background(), &output)).NotTo(Succeed()) + Expect(reached).To(BeFalse()) + Expect(output.Len()).To(BeZero()) + }, Entry("runs", func(c *Command) { c.Runs = 0 }), Entry("warmup", func(c *Command) { c.Warmup = -1 }), Entry("tokens", func(c *Command) { c.MaxTokens = 0 }), Entry("timeout", func(c *Command) { c.Timeout = 0 }), Entry("prompt", func(c *Command) { c.Prompt = " " }), Entry("models", func(c *Command) { c.Models = nil }), Entry("blank model", func(c *Command) { c.Models = []string{"a", " "} }), Entry("scheme", func(c *Command) { c.Endpoint = "file:///tmp" }), Entry("host", func(c *Command) { c.Endpoint = "http:///v1" }), Entry("userinfo", func(c *Command) { c.Endpoint = "http://secret@localhost" }), Entry("query", func(c *Command) { c.Endpoint += "?secret" }), Entry("fragment", func(c *Command) { c.Endpoint += "#secret" })) +}) diff --git a/core/cli/cli.go b/core/cli/cli.go index 77bf128cc..2b17ff96d 100644 --- a/core/cli/cli.go +++ b/core/cli/cli.go @@ -1,12 +1,14 @@ package cli import ( + "github.com/mudler/LocalAI/core/cli/benchmark" cliContext "github.com/mudler/LocalAI/core/cli/context" "github.com/mudler/LocalAI/core/cli/worker" ) var CLI struct { cliContext.Context `embed:""` + Benchmark benchmark.Command `cmd:"" help:"Benchmark configured text models against a running LocalAI server"` Run RunCMD `cmd:"" help:"Run LocalAI, this the default command if no other command is specified. Run 'local-ai run --help' for more information" default:"withargs"` Chat ChatCMD `cmd:"" help:"Run the built-in terminal agent against a LocalAI server"` diff --git a/docs/content/features/benchmark.md b/docs/content/features/benchmark.md new file mode 100644 index 000000000..f57a7783c --- /dev/null +++ b/docs/content/features/benchmark.md @@ -0,0 +1,84 @@ ++++ +title = "Benchmark text models" +weight = 35 ++++ + +Use `local-ai benchmark` to compare text inference through a running LocalAI +server. Install and configure the models first. The command sends sequential, +non-streaming requests to `/v1/chat/completions` and reports full request latency +and end-to-end completion tokens per second. + +Pass one or more configured model names. To compare backends, configure separate +model aliases with the desired backend, then pass those aliases. Backend names +alone are not model names. The command does not install, discover, or unload +models; the server's normal loading and eviction settings still apply. + +For example, with models named `text-llama-cpp` and `text-vllm` configured: + +```sh +local-ai benchmark text-llama-cpp text-vllm --runs 5 --warmup 1 \ + --prompt 'Explain how a rainbow forms.' --max-tokens 128 +``` + +To save settings, per-model summaries, and every measured sample: + +```sh +local-ai benchmark text-llama-cpp text-vllm --json > benchmark.json +``` + +For an authenticated server, set `LOCALAI_API_KEY` or `API_KEY` in the environment. +The API key is excluded from the JSON report. Use `--endpoint` for a remote server +or reverse proxy: + +```sh +local-ai benchmark text-llama-cpp --endpoint https://localai.example.org/proxy/v1 +``` + +The endpoint accepts a server root, an optional `/v1` suffix, and trailing +slashes. A reverse proxy path prefix is preserved. Redirects are refused. +Credentials in the URL, query strings, and fragments are rejected. + +## Arguments and flags + +| Argument or flag | Default | Description | +|---|---|---| +| `MODEL ...` | Required | One or more configured text model names. | +| `--endpoint` | `http://127.0.0.1:8080` | Server URL, optionally ending in `/v1`. | +| `--api-key` | Unset | API key; also reads `LOCALAI_API_KEY`, then `API_KEY`. | +| `--prompt` | `Explain why the sky is blue.` | Nonblank user message repeated for every request. | +| `--max-tokens` | `128` | Positive maximum number of completion tokens per request. | +| `--runs` | `3` | Positive number of measured requests per model. | +| `--warmup` | `1` | Unmeasured requests before each model; zero disables warmups. | +| `--timeout` | `5m` | Positive timeout per request, including reading its response. | +| `--json` | `false` | Write JSON instead of a table. | +| `-h`, `--help` | | Show command help. | + +Every request sets temperature to `0` and streaming to `false`. Interrupting the +command cancels the active request. A failed request stops the benchmark with a +model and run error; results are written only after every model succeeds. + +## Reading the results + +Each model has minimum, mean, and maximum latency across measured requests. +Latency runs from sending the request through parsing the complete response. +It includes transport, queueing, prompt processing, generation, and response +parsing. This command does not measure time to first token. + +End-to-end completion tokens per second is the sum of server-reported completion +tokens divided by the sum of full request durations. It is not decode-only speed +or a substitute for `llama-bench` kernel measurements. If any measured response +omits completion token usage, throughput is `null` in JSON and `N/A` in the table. +Reported zero tokens remain zero. Missing prompt or completion counts remain +`null` in each JSON sample; the command never estimates tokens from text length. + +Warmups run separately for each model and do not appear in measurements. They can +absorb model loading time, but repeated prompts can also benefit from prompt +caching. With `--warmup 0`, measured requests can include model loading. Other +clients and server queueing can affect results; compare under similar load. + +Keep hardware, quantization, context size, backend settings, and prompt consistent +when comparing engines. Different model tokenizers can report different token +counts for the same text, and models can stop before `--max-tokens`. Temperature +zero does not guarantee identical output across models or engines. The JSON +settings describe the benchmark requests, not the server's full model +configuration; record that configuration alongside the report. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index f91beb6a6..d8ad4a17d 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -7,6 +7,8 @@ url = '/reference/cli-reference' Complete reference for all LocalAI command-line interface (CLI) parameters and environment variables. +For client-side text inference measurements, see [Benchmark text models]({{% relref "features/benchmark" %}}). + These options configure the LocalAI server process. To configure an individual model, see [Model Configuration]({{% relref "advanced/model-configuration" %}}). From 3dfb34376a75cecd4a0758251d7b9c7502e58de1 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Sun, 13 Sep 2026 23:11:40 +0200 Subject: [PATCH 10/86] feat(downloads): authenticate registries, galleries and downloads from a credentials file (#12028) * feat(credentials): parse and match download credential rules Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): keep secrets out of parse errors and tighten URL matching Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * feat(credentials): resolve secrets lazily and authenticate HTTP per hop Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): redact secrets in nested and store formatting Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * feat(credentials): add registry keychain and oras credential adapters Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): match repository rules for Docker Hub in the oras adapter Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * feat(downloads): authenticate HTTP downloads and gallery reads from the credentials store Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * feat(oci): authenticate registry pulls, resumes, blobs and cosign from the credentials store Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * feat(cli): load download credentials from --credentials-file Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * docs(credentials): correct the local-network registry rules Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): keep secrets out of match and YAML parse errors A match that fails to parse is no longer quoted in the Parse error, since it may be a URL with a token in it. Userinfo is detected before the scheme check, so ftp://user:token@host is refused as userinfo, and a match with a query string or fragment is refused because it can never apply and a query string is where signed URLs carry their token. Every YAML decode error is now redacted, not only type errors: quoted scalars such as a secret under a mismatched !!int tag are replaced and unquoted map keys are cut off. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(downloads): make auth errors name the real cause and never retry unresolved secrets AuthError now appends its cause, so a registry's DENIED or UNAUTHORIZED detail reaches the operator. HTTP auth errors print only the status text in place of the cause, because the downloader builds that cause from the requested URL, which can carry a signed query string. Registry pulls say that docker config credentials were tried too, and a download that carried a caller-provided credential (WithBearerToken, or an explicit authorization on gallery reads) reports that credential as rejected instead of blaming the store. The Range probe for a leftover partial file now returns an unresolved secret as a permanent error, like the download request already did. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): keep oras pulls anonymous on a broken docker helper and close bodies When docker config names a credsStore helper that cannot run, the oras credential func now logs at debug and returns no credential, so public pulls keep working as they did before the adapter existed. The transport closes the request body when a rule's secret cannot be resolved, as the RoundTripper contract requires. The redirect spec now uses a custom header rule on the origin, which net/http would not strip on its own, to prove the transport does not carry credentials to the next hop. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * test(oci): cover FetchImageBlob authentication against a private registry FetchImageBlob now has a spec that pulls a layer blob by digest from a basic-auth registry through the oras credential adapter, and one that shows the same fetch fails when no rule matches. oras only speaks HTTPS here, so the registry serves TLS and the spec points http.DefaultTransport, which retry.DefaultClient falls back to, at the test server's client for its duration instead of adding a transport seam to production code. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * docs(credentials): document auth error wording, ollama manifests and registry tokens The errors section now lists the registry and provided-credential messages and says the server's reason is appended. ollama:// manifests are fetched without credentials, so only blob downloads use the file. GHCR, Docker Hub and Quay need basic auth with the token as password, and match rules must not carry a query string or fragment. The backend gallery docs and the container troubleshooting section now point to the private sources page. Assisted-by: Claude:claude-opus-5 [Claude Code] Signed-off-by: Ettore Di Giacinto * fix(credentials): document trusted file path The credentials path comes from operator configuration. Mark the file read with a scoped G304 explanation to resolve the gosec false positive. Assisted-by: Codex:gpt-6 gosec --------- Signed-off-by: Ettore Di Giacinto Co-authored-by: Ettore Di Giacinto Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- cmd/local-ai/main.go | 6 + core/cli/context/context.go | 9 +- core/cli/credentials.go | 44 +++ core/cli/credentials_test.go | 67 ++++ core/cli/run.go | 6 + docs/content/advanced/private-sources.md | 127 +++++++ docs/content/features/backends.md | 2 + docs/content/features/model-gallery.md | 2 + docs/content/getting-started/containers.md | 1 + docs/content/reference/cli-reference.md | 1 + pkg/credentials/credential.go | 169 +++++++++ pkg/credentials/credential_test.go | 112 ++++++ pkg/credentials/credentials_suite_test.go | 45 +++ pkg/credentials/default.go | 18 + pkg/credentials/errors.go | 103 ++++++ pkg/credentials/errors_test.go | 75 ++++ pkg/credentials/registry.go | 107 ++++++ pkg/credentials/registry_test.go | 164 +++++++++ pkg/credentials/store.go | 377 +++++++++++++++++++++ pkg/credentials/store_test.go | 147 ++++++++ pkg/credentials/transport.go | 37 ++ pkg/credentials/transport_test.go | 129 +++++++ pkg/downloader/credentials_test.go | 213 ++++++++++++ pkg/downloader/uri.go | 40 ++- pkg/oci/auth.go | 28 ++ pkg/oci/blob.go | 2 + pkg/oci/cosignverify/verify.go | 3 +- pkg/oci/image.go | 11 +- pkg/oci/image_auth_internal_test.go | 207 +++++++++++ 29 files changed, 2238 insertions(+), 14 deletions(-) create mode 100644 core/cli/credentials.go create mode 100644 core/cli/credentials_test.go create mode 100644 docs/content/advanced/private-sources.md create mode 100644 pkg/credentials/credential.go create mode 100644 pkg/credentials/credential_test.go create mode 100644 pkg/credentials/credentials_suite_test.go create mode 100644 pkg/credentials/default.go create mode 100644 pkg/credentials/errors.go create mode 100644 pkg/credentials/errors_test.go create mode 100644 pkg/credentials/registry.go create mode 100644 pkg/credentials/registry_test.go create mode 100644 pkg/credentials/store.go create mode 100644 pkg/credentials/store_test.go create mode 100644 pkg/credentials/transport.go create mode 100644 pkg/credentials/transport_test.go create mode 100644 pkg/downloader/credentials_test.go create mode 100644 pkg/oci/auth.go create mode 100644 pkg/oci/image_auth_internal_test.go diff --git a/cmd/local-ai/main.go b/cmd/local-ai/main.go index b1799dbb8..ff520b6d9 100644 --- a/cmd/local-ai/main.go +++ b/cmd/local-ai/main.go @@ -105,6 +105,12 @@ For documentation and support: xlog.SetLogger(xlog.NewLogger(xlog.LogLevel(*cli.CLI.LogLevel), *cli.CLI.LogFormat, logOpts...)) + // Loaded here rather than in RunCMD so `worker`, `backends install` and + // `models install` authenticate the same way the server does. + if err := cli.LoadCredentials(cli.CLI.CredentialsFile); err != nil { + xlog.Fatal("Error loading credentials", "error", err) + } + // Run the thing! err = ctx.Run(&cli.CLI.Context) if err != nil { diff --git a/core/cli/context/context.go b/core/cli/context/context.go index c6d3f1176..752d39a6b 100644 --- a/core/cli/context/context.go +++ b/core/cli/context/context.go @@ -1,8 +1,9 @@ package cliContext type Context struct { - Debug bool `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"` - LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug,trace" help:"Set the level of logs to output [${enum}]"` - LogFormat *string `env:"LOCALAI_LOG_FORMAT" default:"default" enum:"default,text,json" help:"Set the format of logs to output [${enum}]"` - LogDedupLogs *bool `env:"LOCALAI_LOG_DEDUP" negatable:"" help:"Deduplicate consecutive identical log lines (auto-detected for terminals, use --log-dedup-logs to force on or --no-log-dedup-logs to force off)"` + Debug bool `env:"LOCALAI_DEBUG,DEBUG" default:"false" hidden:"" help:"DEPRECATED, use --log-level=debug instead. Enable debug logging"` + LogLevel *string `env:"LOCALAI_LOG_LEVEL" enum:"error,warn,info,debug,trace" help:"Set the level of logs to output [${enum}]"` + LogFormat *string `env:"LOCALAI_LOG_FORMAT" default:"default" enum:"default,text,json" help:"Set the format of logs to output [${enum}]"` + LogDedupLogs *bool `env:"LOCALAI_LOG_DEDUP" negatable:"" help:"Deduplicate consecutive identical log lines (auto-detected for terminals, use --log-dedup-logs to force on or --no-log-dedup-logs to force off)"` + CredentialsFile string `env:"LOCALAI_CREDENTIALS_FILE" help:"YAML file with credentials for private registries, galleries and download hosts (see https://localai.io/advanced/private-sources/)"` } diff --git a/core/cli/credentials.go b/core/cli/credentials.go new file mode 100644 index 000000000..7afce1e42 --- /dev/null +++ b/core/cli/credentials.go @@ -0,0 +1,44 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + + "github.com/mudler/xlog" + + "github.com/mudler/LocalAI/pkg/credentials" +) + +// credentialsFileName is looked up in the data path when no flag is given, so +// an install that keeps all its state on one volume needs no extra setting. +const credentialsFileName = "credentials.yaml" + +func resolveCredentialsFile(flag, dataPath string) string { + if flag != "" { + return flag + } + if dataPath == "" { + return "" + } + p := filepath.Join(dataPath, credentialsFileName) + if _, err := os.Stat(p); err != nil { + return "" + } + return p +} + +// LoadCredentials installs the download credentials for this process. An empty +// path leaves downloads anonymous; registries still honor docker config. +func LoadCredentials(path string) error { + if path == "" { + return nil + } + store, err := credentials.Load(path, os.LookupEnv) + if err != nil { + return fmt.Errorf("loading credentials file %q: %w", path, err) + } + credentials.SetDefault(store) + xlog.Info("Loaded download credentials", "file", path, "rules", store.Len()) + return nil +} diff --git a/core/cli/credentials_test.go b/core/cli/credentials_test.go new file mode 100644 index 000000000..99964eab9 --- /dev/null +++ b/core/cli/credentials_test.go @@ -0,0 +1,67 @@ +package cli + +import ( + "net/http" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/credentials" +) + +var _ = Describe("credentials file", Serial, func() { + Describe("resolveCredentialsFile", func() { + It("prefers the flag", func() { + dataPath := GinkgoT().TempDir() + Expect(os.WriteFile(filepath.Join(dataPath, "credentials.yaml"), nil, 0o600)).To(Succeed()) + Expect(resolveCredentialsFile("/etc/localai/creds.yaml", dataPath)).To(Equal("/etc/localai/creds.yaml")) + }) + + It("falls back to credentials.yaml in the data path when present", func() { + dataPath := GinkgoT().TempDir() + p := filepath.Join(dataPath, "credentials.yaml") + Expect(os.WriteFile(p, nil, 0o600)).To(Succeed()) + Expect(resolveCredentialsFile("", dataPath)).To(Equal(p)) + }) + + It("returns nothing when neither is set", func() { + Expect(resolveCredentialsFile("", GinkgoT().TempDir())).To(BeEmpty()) + Expect(resolveCredentialsFile("", "")).To(BeEmpty()) + }) + }) + + Describe("LoadCredentials", func() { + BeforeEach(func() { + prev := credentials.SetDefault(nil) + DeferCleanup(func() { credentials.SetDefault(prev) }) + }) + + It("installs the store and resolves _env from the process environment", func() { + GinkgoT().Setenv("LOCALAI_TEST_REGISTRY_TOKEN", "from-env") + p := filepath.Join(GinkgoT().TempDir(), "credentials.yaml") + Expect(os.WriteFile(p, []byte("- match: ghcr.io/acme\n bearer_env: LOCALAI_TEST_REGISTRY_TOKEN\n"), 0o600)).To(Succeed()) + + Expect(LoadCredentials(p)).To(Succeed()) + Expect(credentials.Default().Len()).To(Equal(1)) + c, ok := credentials.Default().Match("https://ghcr.io/acme/img") + Expect(ok).To(BeTrue()) + h := http.Header{} + Expect(c.ApplyHeaders(h)).To(Succeed()) + Expect(h.Get("Authorization")).To(Equal("Bearer from-env")) + }) + + It("fails on an invalid file and names it", func() { + p := filepath.Join(GinkgoT().TempDir(), "credentials.yaml") + Expect(os.WriteFile(p, []byte("- match: ghcr.io\n"), 0o600)).To(Succeed()) + Expect(LoadCredentials(p)).To(MatchError(ContainSubstring(p))) + Expect(credentials.Default()).To(BeNil()) + }) + + It("does nothing for an empty path", func() { + Expect(LoadCredentials("")).To(Succeed()) + Expect(credentials.Default()).To(BeNil()) + }) + }) +}) diff --git a/core/cli/run.go b/core/cli/run.go index 6bf377682..9e663687c 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -250,6 +250,12 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { return nil } + if ctx.CredentialsFile == "" { + if err := LoadCredentials(resolveCredentialsFile("", r.DataPath)); err != nil { + return err + } + } + activatedListeners, err := systemdActivatedListeners() if err != nil { return fmt.Errorf("loading systemd socket activation listeners: %w", err) diff --git a/docs/content/advanced/private-sources.md b/docs/content/advanced/private-sources.md new file mode 100644 index 000000000..3f02fce07 --- /dev/null +++ b/docs/content/advanced/private-sources.md @@ -0,0 +1,127 @@ ++++ +disableToc = false +title = "Private Registries and Galleries" +weight = 24 +url = '/advanced/private-sources' ++++ + +LocalAI can pull backends, models and gallery indexes from locations that need authentication: private OCI registries (GHCR, Quay, Harbor, Artifactory, ECR), internal HTTP servers, and private GitHub repositories. + +You give LocalAI a credentials file. Each entry matches a URL prefix and says how to authenticate. Gallery files never contain secrets, so you can share a gallery and keep the credentials separate. + +## Credentials file + +Pass the file with `--credentials-file` or `LOCALAI_CREDENTIALS_FILE`. If neither is set, `local-ai run` reads `credentials.yaml` from the data path (`LOCALAI_DATA_PATH`) when that file exists. + +```yaml +# Basic auth for a private registry. The password comes from an env var. +- match: ghcr.io/acme + username: bot + password_env: GHCR_TOKEN + +# Bearer token for an internal model server, read from a mounted secret. +- match: https://models.acme.internal/ + bearer_file: /var/run/secrets/acme/token + +# Private GitHub repository used by a gallery (github: URIs). +- match: github.com/acme + bearer_env: GITHUB_TOKEN + +# Artifactory-style API key header. +- match: artifactory.acme.internal + header: + name: X-JFrog-Art-Api + value_env: ART_KEY + +# Registry on a private network address. Image pulls from it are matched +# as http://, so the entry needs allow_insecure and no https:// scheme. +- match: 192.168.1.5:5000 + username: ci + password_file: /run/secrets/registry-password + allow_insecure: true +``` + +Each entry has: + +| Key | Description | +|-----|-------------| +| `match` | URL prefix: a host, optionally followed by a path, optionally with `http://` or `https://`. Required. It must not contain credentials (`user:token@host`), a query string (`?`) or a fragment (`#`). | +| `username` + `password`, `password_env` or `password_file` | Basic authentication. | +| `bearer`, `bearer_env` or `bearer_file` | Bearer token. For registries it is sent as a registry token. GHCR, Docker Hub and Quay do not accept a bearer entry: use basic auth with any `username` and the token as the `password`. | +| `header.name` + `header.value`, `header.value_env` or `header.value_file` | A custom header. HTTP downloads only; registries ignore it. | +| `allow_insecure` | Allow sending this credential over plain `http://`. Default `false`. Registries on local or private addresses need it, see [Registries on the local network](#registries-on-the-local-network). | + +Use exactly one authentication type per entry, and exactly one of the plain, `_env` or `_file` forms per secret. + +LocalAI checks the file when it starts and stops with an error if the file is not valid. Unknown keys are errors, so a misspelled key such as `pasword_env` does not load silently. For a value of the wrong type, the error gives the line number but does not show the value. + +LocalAI reads `_env` and `_file` values each time it needs them. A rotated Kubernetes secret mount is used without a restart. Trailing newlines in secret files are removed. + +## How matching works + +- Only `https://` URLs get credentials. `http://` URLs get credentials only from an entry with `allow_insecure: true` (see [Registries on the local network](#registries-on-the-local-network)). Other schemes never get credentials. +- The host must be equal. `ghcr.io` does not match `ghcr.io.evil.net`. +- The path matches whole segments. `ghcr.io/acme` matches `ghcr.io/acme/backend` but not `ghcr.io/acme-tools/backend`. +- A URL whose path has `.` or `..` segments never gets credentials. +- If more than one entry matches, the entry with the longest path wins. If two entries are equal, the first one in the file wins. +- If `match` has a scheme, the request must use that scheme. Without a scheme, the entry matches HTTPS, and also plain HTTP when `allow_insecure: true` is set. +- `github.com/` also matches the `raw.githubusercontent.com//...` URLs that `github:` URIs download from. +- `docker.io` also matches `index.docker.io` and `registry-1.docker.io`. +- LocalAI checks every redirect separately. If a server redirects a download to a CDN that no entry matches, LocalAI sends no credentials to the CDN. + +### Registries on the local network + +The registry client treats some registry names as local and uses `http` as their scheme: + +- a name that starts with `localhost:` (a port is given, for example `localhost:5000`) +- a name that ends in `.localhost`, with or without a port (for example `registry.localhost:5000`) +- a name that contains `127.0.0.1` or `::1` +- an IPv4 address in `10.0.0.0/8`, `172.16.0.0/12` or `192.168.0.0/16`, with or without a port + +For these registries, LocalAI always matches image pulls as `http:///`, whatever scheme the connection uses in the end. An entry written as `https://192.168.1.5:5000` never applies to image pulls from that registry. + +The entry for such a registry needs `allow_insecure: true`, and its `match` must have no scheme or use `http://`. No scheme is recommended, for example `match: 192.168.1.5:5000` with `allow_insecure: true`, because the same entry then also covers downloads that use HTTPS. + +## What uses the credentials + +- Gallery indexes and mirrors (`galleries`, `backend_galleries`), including `github:` URLs. +- Model files and model configs downloaded over HTTP(S) or `github:`. +- Backend images and `oci://` / `ollama://` models, including resumed layer downloads and cosign signature checks. + +For registries, LocalAI checks the credentials file first. If no entry matches, it uses your docker login (`~/.docker/config.json` or `DOCKER_CONFIG`). Hosts that already use `docker login` do not need a credentials file. + +For `ollama://` models, only the blob downloads use credentials. LocalAI fetches the model manifest without credentials, so the manifest must be readable anonymously. + +A credential passed directly by LocalAI (for example `HF_TOKEN` for managed Hugging Face artifacts) takes precedence over the file. + +## Errors + +When a server refuses a download with status 401 or 403, the error shows which case applies: + +- `authentication required for (status ): no credentials rule matches it`: add an entry that matches this URL. For registries the message continues with `and docker config credentials, if any, were not accepted`, because LocalAI also tried your docker login. +- `credential "" was rejected by (status )`: an entry matched, but the server did not accept it. Check the secret and its permissions. +- `the provided credential was rejected by (status )`: the download carried a credential that LocalAI got from somewhere other than the file (for example `HF_TOKEN`), so the file was not used. Check that credential. + +Each message ends with the reason the server gave, when there is one, for example a registry's `DENIED` detail. For HTTP downloads only the status text is shown, because the request URL can contain a signed query string. + +If an `_env` variable is not set or a `_file` cannot be read, LocalAI logs a warning at startup and keeps the entry, because a secret mount can appear later. A download that matches the entry fails with an error that names the variable or file. LocalAI does not retry that download. + +## Kubernetes + +Mount the file and the secrets from a Secret: + +```yaml +env: + - name: LOCALAI_CREDENTIALS_FILE + value: /etc/localai/credentials.yaml +volumeMounts: + - name: localai-credentials + mountPath: /etc/localai + readOnly: true +``` + +## Distributed mode + +The controller downloads models and sends them to the workers, so model downloads only need credentials on the controller. + +Each worker pulls its own backend images. To install backends from a private registry, give every worker the same credentials file. LocalAI does not send credentials over NATS. If a worker has no matching entry and no docker login for that registry, the install fails, and the node's install error says that no credentials rule matches. diff --git a/docs/content/features/backends.md b/docs/content/features/backends.md index 9ef87ee4d..1287e68f7 100644 --- a/docs/content/features/backends.md +++ b/docs/content/features/backends.md @@ -64,6 +64,8 @@ The URL needs to point to a valid yaml file, for example: Where URI is the path to an OCI container image. +To use a backend gallery or backend images that need authentication, such as a private registry, add a matching entry to the credentials file. See [Private Registries and Galleries]({{% relref "advanced/private-sources" %}}). + ### Backend Gallery Structure A backend gallery is a collection of YAML files, each defining a backend. Here's an example structure: diff --git a/docs/content/features/model-gallery.md b/docs/content/features/model-gallery.md index 4cd6f719a..f0088f583 100644 --- a/docs/content/features/model-gallery.md +++ b/docs/content/features/model-gallery.md @@ -106,6 +106,8 @@ GALLERIES=[{"name":"", "url":"") + if before, _, ok := strings.Cut(msg, "invalid map key:"); ok { + msg = before + "invalid map key" + } + return errors.New(msg) + } + msgs := make([]string, 0, len(te.Errors)) + for _, line := range te.Errors { + if m := unknownFieldRe.FindStringSubmatch(line); m != nil { + msgs = append(msgs, fmt.Sprintf("line %s: unknown key %s", m[1], m[2])) + continue + } + if m := errorLineRe.FindStringSubmatch(line); m != nil { + msgs = append(msgs, fmt.Sprintf("line %s: value has the wrong type", m[1])) + continue + } + msgs = append(msgs, "value has the wrong type") + } + return fmt.Errorf("credentials file has entries of the wrong type or unknown keys: %s", strings.Join(msgs, "; ")) +} + +// String, GoString and LogValue name rules by match only. They are defined on +// Store because fmt prints the unexported creds field without consulting +// Credential.String. String and GoString take a value so that both Store and +// *Store are covered. +func (s Store) String() string { + return fmt.Sprintf("credentials.Store(%d rules: %s)", len(s.creds), strings.Join(s.matches(), ", ")) +} + +func (s Store) GoString() string { + return s.String() +} + +// LogValue takes a pointer so that logging Default() before a store is +// installed does not panic inside slog. A Store value logged without it falls +// back to String (text) or an empty object (JSON). +func (s *Store) LogValue() slog.Value { + if s == nil { + return slog.GroupValue(slog.Int("rules", 0)) + } + return slog.GroupValue(slog.Int("rules", len(s.creds)), slog.Any("matches", s.matches())) +} + +func (s Store) matches() []string { + out := make([]string, len(s.creds)) + for i, c := range s.creds { + out[i] = c.Match + } + return out +} + +// Len reports how many rules the store holds. +func (s *Store) Len() int { + if s == nil { + return 0 + } + return len(s.creds) +} + +func newCredential(e fileEntry, lookupEnv LookupEnvFunc) (Credential, error) { + scheme, host, path, err := parseMatch(e.Match) + if err != nil { + return Credential{}, err + } + c := Credential{ + Match: e.Match, + allowInsecure: e.AllowInsecure, + scheme: scheme, + host: host, + path: path, + lookupEnv: lookupEnv, + username: e.Username, + } + if c.password, err = oneForm("password", e.Password, e.PasswordEnv, e.PasswordFile); err != nil { + return Credential{}, err + } + if c.bearer, err = oneForm("bearer", e.Bearer, e.BearerEnv, e.BearerFile); err != nil { + return Credential{}, err + } + + kinds := 0 + if e.Username != "" || c.password.set() { + if e.Username == "" || !c.password.set() { + return Credential{}, errors.New("basic auth needs both username and password") + } + kinds++ + c.Kind = KindBasic + } + if c.bearer.set() { + kinds++ + c.Kind = KindBearer + } + if e.Header != nil { + if e.Header.Name == "" { + return Credential{}, errors.New("header.name is required") + } + if c.headerValue, err = oneForm("value", e.Header.Value, e.Header.ValueEnv, e.Header.ValueFile); err != nil { + return Credential{}, err + } + if !c.headerValue.set() { + return Credential{}, errors.New("header value is required") + } + c.headerName = e.Header.Name + kinds++ + c.Kind = KindHeader + } + if kinds != 1 { + return Credential{}, fmt.Errorf("exactly one of basic (username + password), bearer, or header is required, found %d", kinds) + } + return c, nil +} + +func oneForm(field, literal, env, file string) (secretRef, error) { + n := 0 + for _, v := range []string{literal, env, file} { + if v != "" { + n++ + } + } + if n > 1 { + return secretRef{}, fmt.Errorf("set only one of %s, %s_env, %s_file", field, field, field) + } + r := secretRef{env: env, file: file} + if literal != "" { + r.literal = &literal + } + return r, nil +} + +// invalidMatchError marks a match that did not parse. Its message never +// includes the match text, and Parse relies on the type to leave it unquoted. +type invalidMatchError struct { + msg string +} + +func (e *invalidMatchError) Error() string { + return e.msg +} + +func invalidMatch(msg string) error { + return &invalidMatchError{msg: msg} +} + +func parseMatch(m string) (scheme, host, path string, err error) { + m = strings.TrimSpace(m) + if m == "" { + return "", "", "", invalidMatch("match is required") + } + // Match ignores query strings and fragments, so such a rule could never + // apply, and a query string is where signed URLs carry their token. + if strings.Contains(m, "?") { + return "", "", "", invalidMatch("match must not contain a query string") + } + if strings.Contains(m, "#") { + return "", "", "", invalidMatch("match must not contain a fragment") + } + before, rest, hasScheme := strings.Cut(m, "://") + if !hasScheme { + rest = m + } + host, path, _ = strings.Cut(rest, "/") + // Checked before the scheme so that a user:token@host match is refused as + // userinfo whatever its scheme: such a rule never matches a request, and + // Match is printed everywhere a rule is named. + if strings.Contains(host, "@") || (hasScheme && strings.Contains(before, "@")) { + return "", "", "", invalidMatch("match must not contain credentials (userinfo)") + } + if hasScheme { + scheme = strings.ToLower(before) + if scheme != "http" && scheme != "https" { + return "", "", "", invalidMatch("unsupported scheme in match: only http and https are allowed") + } + } + if host == "" { + return "", "", "", invalidMatch("match has no host") + } + return scheme, normalizeHost(scheme, host), strings.Trim(path, "/"), nil +} + +// normalizeHost folds spellings of the same endpoint together so a rule +// written one way matches a request made another way. +func normalizeHost(scheme, host string) string { + host = strings.ToLower(host) + if scheme == "http" { + host = strings.TrimSuffix(host, ":80") + } else { + host = strings.TrimSuffix(host, ":443") + } + switch host { + case "docker.io", "registry-1.docker.io": + return "index.docker.io" + } + return host +} + +type matchTarget struct { + host string + path string +} + +// Match returns the rule for rawURL: the longest matching path wins, and ties +// go to the rule that appears first in the file. +func (s *Store) Match(rawURL string) (Credential, bool) { + if s.Len() == 0 { + return Credential{}, false + } + u, err := url.Parse(rawURL) + if err != nil || u.Host == "" { + return Credential{}, false + } + scheme := strings.ToLower(u.Scheme) + // Allowlist: an empty scheme ("//host/path") or ws/ftp must never pick up + // credentials meant for HTTPS. + if scheme != "https" && scheme != "http" { + return Credential{}, false + } + // Prefix scoping is meaningless once a segment can climb out of it, and a + // server may resolve "..", including percent-encoded slashes, after we match. + if hasDotSegment(u.Path) || hasDotSegment(decodedEscapedPath(u)) { + return Credential{}, false + } + host := normalizeHost(scheme, u.Host) + path := strings.Trim(u.Path, "/") + + targets := []matchTarget{{host: host, path: path}} + // github: URIs are fetched from raw.githubusercontent.com, but operators + // think of the repository as github.com//. + if host == "raw.githubusercontent.com" { + targets = append(targets, matchTarget{host: "github.com", path: path}) + } + + best, bestLen := -1, -1 + for i, c := range s.creds { + if c.scheme != "" && c.scheme != scheme { + continue + } + if scheme == "http" && !c.allowInsecure { + continue + } + for _, t := range targets { + if c.host != t.host || !pathMatches(c.path, t.path) { + continue + } + if len(c.path) > bestLen { + best, bestLen = i, len(c.path) + } + } + } + if best < 0 { + return Credential{}, false + } + return s.creds[best], true +} + +func decodedEscapedPath(u *url.URL) string { + var out []string + for _, seg := range strings.Split(u.EscapedPath(), "/") { + dec, err := url.PathUnescape(seg) + if err != nil { + dec = seg + } + out = append(out, dec) + } + return strings.Join(out, "/") +} + +func hasDotSegment(p string) bool { + for _, seg := range strings.Split(p, "/") { + if seg == "." || seg == ".." { + return true + } + } + return false +} + +func pathMatches(rule, target string) bool { + return rule == "" || target == rule || strings.HasPrefix(target, rule+"/") +} diff --git a/pkg/credentials/store_test.go b/pkg/credentials/store_test.go new file mode 100644 index 000000000..7ce0c517b --- /dev/null +++ b/pkg/credentials/store_test.go @@ -0,0 +1,147 @@ +package credentials_test + +import ( + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/credentials" +) + +var _ = Describe("Parse", func() { + It("accepts an empty file", func() { + Expect(mustParse("").Len()).To(Equal(0)) + }) + + It("loads every valid entry kind", func() { + s := mustParse(` +- match: ghcr.io/acme + username: bot + password_env: GHCR_TOKEN +- match: https://models.acme.internal/ + bearer_file: /var/run/secrets/acme/token +- match: artifactory.acme.internal + header: + name: X-JFrog-Art-Api + value: key +`) + Expect(s.Len()).To(Equal(3)) + }) + + DescribeTable("rejects invalid entries", + func(doc, msg string) { + _, err := credentials.Parse([]byte(doc), noEnv) + Expect(err).To(MatchError(ContainSubstring(msg))) + }, + Entry("missing match", "- bearer: x\n", "match is required"), + Entry("no auth kind", "- match: ghcr.io\n", "exactly one of"), + Entry("two auth kinds", "- match: ghcr.io\n bearer: x\n username: u\n password: p\n", "exactly one of"), + Entry("username without password", "- match: ghcr.io\n username: u\n", "username and password"), + Entry("two forms for one secret", "- match: ghcr.io\n bearer: x\n bearer_env: Y\n", "only one of bearer"), + Entry("header without name", "- match: ghcr.io\n header:\n value: x\n", "header.name is required"), + Entry("header without value", "- match: ghcr.io\n header:\n name: X-Key\n", "header value is required"), + Entry("misspelled key", "- match: ghcr.io\n bearer: x\n pasword: y\n", "pasword"), + Entry("unsupported scheme", "- match: ftp://host\n bearer: x\n", "unsupported scheme"), + ) + + It("does not echo secret literals in validation errors", func() { + _, err := credentials.Parse([]byte("- match: ghcr.io\n bearer: hunter2\n bearer_env: ALSO\n"), noEnv) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("hunter2")) + }) + + It("rejects userinfo in a match without echoing it", func() { + _, err := credentials.Parse([]byte("- match: https://user:tok3n@ghcr.io\n bearer: x\n"), noEnv) + Expect(err).To(MatchError(ContainSubstring("userinfo"))) + Expect(err.Error()).NotTo(ContainSubstring("tok3n")) + }) + + DescribeTable("rejects a bad match without echoing the secret it holds", + func(doc, msg, secret string) { + _, err := credentials.Parse([]byte(doc), noEnv) + Expect(err).To(MatchError(ContainSubstring(msg))) + Expect(err.Error()).NotTo(ContainSubstring(secret)) + }, + Entry("userinfo with an unsupported scheme", "- match: ftp://user:tok123@host\n bearer: x\n", "userinfo", "tok123"), + Entry("query string", "- match: https://files.example.com/model?sig=abc123\n bearer: x\n", "query", "abc123"), + Entry("fragment", "- match: https://files.example.com/model#abc123\n bearer: x\n", "fragment", "abc123"), + ) + + It("does not echo a secret from a YAML tag decode error", func() { + _, err := credentials.Parse([]byte("- match: ghcr.io\n password: !!int hunter2\n username: u\n"), noEnv) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("hunter2")) + }) + + DescribeTable("does not echo secret literals in YAML type errors", + func(doc, secretFragment string) { + _, err := credentials.Parse([]byte(doc), noEnv) + Expect(err).To(MatchError(ContainSubstring("wrong type"))) + Expect(err.Error()).To(ContainSubstring("line 2")) + Expect(err.Error()).NotTo(ContainSubstring(secretFragment)) + }, + Entry("short value", "- match: ghcr.io\n header: hunter2\n", "hunter2"), + Entry("long value", "- match: ghcr.io\n header: supersecretvalue123\n", "superse"), + ) +}) + +var _ = Describe("Store.Match", func() { + var store *credentials.Store + + BeforeEach(func() { + store = mustParse(` +- match: ghcr.io/acme + bearer: a +- match: ghcr.io/acme/private + bearer: b +- match: https://files.example.com/models + bearer: c +- match: github.com/acme + bearer: d +- match: docker.io/acme + bearer: e +- match: registry.lan:5000 + username: ci + password: pw + allow_insecure: true +- match: plain.example.com + bearer: f +`) + }) + + DescribeTable("selects the rule for a URL", + func(rawURL, want string) { + c, ok := store.Match(rawURL) + if want == "" { + Expect(ok).To(BeFalse(), "unexpectedly matched %q", c.Match) + return + } + Expect(ok).To(BeTrue()) + Expect(c.Match).To(Equal(want)) + }, + Entry("repository prefix", "https://ghcr.io/acme/img", "ghcr.io/acme"), + Entry("exact path", "https://ghcr.io/acme", "ghcr.io/acme"), + Entry("longest prefix wins", "https://ghcr.io/acme/private/img", "ghcr.io/acme/private"), + Entry("path matches on segment boundaries", "https://ghcr.io/acme-evil/img", ""), + Entry("host must be equal, not a suffix", "https://ghcr.io.evil.net/acme/img", ""), + Entry("host is case-insensitive and default port is ignored", "https://GHCR.IO:443/acme/img", "ghcr.io/acme"), + Entry("scheme in the rule is enforced", "http://files.example.com/models/a.gguf", ""), + Entry("scheme in the rule matches", "https://files.example.com/models/a.gguf", "https://files.example.com/models"), + Entry("raw.githubusercontent.com is matched as github.com", "https://raw.githubusercontent.com/acme/gallery/main/index.yaml", "github.com/acme"), + Entry("docker.io is matched as index.docker.io", "https://index.docker.io/acme/img", "docker.io/acme"), + Entry("plain http when the rule opts in", "http://registry.lan:5000/team/img", "registry.lan:5000"), + Entry("no credentials over plain http by default", "http://plain.example.com/x", ""), + Entry("unparseable target", "not a url", ""), + Entry("scheme-relative target", "//ghcr.io/acme/img", ""), + Entry("ftp target", "ftp://ghcr.io/acme/img", ""), + Entry("websocket target", "ws://ghcr.io/acme/img", ""), + Entry("dot-dot segment", "https://ghcr.io/acme/../other/img", ""), + Entry("percent-encoded dot-dot segment", "https://ghcr.io/acme%2F..%2Fother/img", ""), + ) + + It("is safe on a nil store", func() { + var nilStore *credentials.Store + _, ok := nilStore.Match("https://ghcr.io/acme/img") + Expect(ok).To(BeFalse()) + Expect(nilStore.Len()).To(Equal(0)) + }) +}) diff --git a/pkg/credentials/transport.go b/pkg/credentials/transport.go new file mode 100644 index 000000000..7b8a07519 --- /dev/null +++ b/pkg/credentials/transport.go @@ -0,0 +1,37 @@ +package credentials + +import "net/http" + +type transport struct { + base http.RoundTripper +} + +// Transport authenticates every request it carries, including each redirect +// hop, with the default store's matching rule. +func Transport(base http.RoundTripper) http.RoundTripper { + return transport{base: base} +} + +func (t transport) RoundTrip(req *http.Request) (*http.Response, error) { + // A caller that set Authorization itself (WithBearerToken, the HF + // materializer) has chosen its credential; the store must not replace it. + if req.Header.Get("Authorization") != "" { + return t.base.RoundTrip(req) + } + c, ok := Default().Match(req.URL.String()) + if !ok { + return t.base.RoundTrip(req) + } + // The credential goes on a clone. net/http builds every redirect hop from + // the original request's headers, so nothing added here follows the + // request to another host: each hop is matched on its own. + authed := req.Clone(req.Context()) + if err := c.ApplyHeaders(authed.Header); err != nil { + // RoundTripper must close the body even when it fails before sending. + if req.Body != nil { + _ = req.Body.Close() + } + return nil, err + } + return t.base.RoundTrip(authed) +} diff --git a/pkg/credentials/transport_test.go b/pkg/credentials/transport_test.go new file mode 100644 index 000000000..5c5d4b010 --- /dev/null +++ b/pkg/credentials/transport_test.go @@ -0,0 +1,129 @@ +package credentials_test + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/credentials" + "github.com/mudler/LocalAI/pkg/httpclient" +) + +var _ = Describe("Transport", Serial, func() { + var client *http.Client + + BeforeEach(func() { + client = httpclient.New( + httpclient.WithFollowRedirects(), + httpclient.WithTransport(credentials.Transport(httpclient.HardenedTransport())), + ) + }) + + get := func(rawURL string, header http.Header) { + req, err := http.NewRequest(http.MethodGet, rawURL, nil) + Expect(err).NotTo(HaveOccurred()) + for k, v := range header { + req.Header[k] = v + } + resp, err := client.Do(req) + Expect(err).NotTo(HaveOccurred()) + Expect(resp.Body.Close()).To(Succeed()) + } + + It("authenticates each redirect hop with its own rule and sends nothing where no rule matches", func() { + var originAuth, originKey, mirrorAuth, mirrorKey, cdnAuth, cdnKey string + cdn := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cdnAuth = r.Header.Get("Authorization") + cdnKey = r.Header.Get("X-Key") + })) + DeferCleanup(cdn.Close) + mirror := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mirrorAuth = r.Header.Get("Authorization") + mirrorKey = r.Header.Get("X-Key") + http.Redirect(w, r, cdn.URL+"/blob", http.StatusFound) + })) + DeferCleanup(mirror.Close) + origin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + originAuth = r.Header.Get("Authorization") + originKey = r.Header.Get("X-Key") + http.Redirect(w, r, mirror.URL+"/file", http.StatusFound) + })) + DeferCleanup(origin.Close) + + // The origin uses a header rule because net/http strips Authorization + // on a cross-host redirect by itself but knows nothing of X-Key, so + // only a custom header proves the transport does not copy credentials + // to the next hop. + useStore(fmt.Sprintf("- match: %s\n header:\n name: X-Key\n value: origin-key\n allow_insecure: true\n- match: %s\n bearer: mirror-token\n allow_insecure: true\n", origin.URL, mirror.URL)) + get(origin.URL+"/start", nil) + + Expect(originKey).To(Equal("origin-key")) + Expect(originAuth).To(BeEmpty()) + Expect(mirrorAuth).To(Equal("Bearer mirror-token")) + Expect(mirrorKey).To(BeEmpty()) + Expect(cdnAuth).To(BeEmpty()) + Expect(cdnKey).To(BeEmpty()) + }) + + It("leaves an explicit Authorization header to the caller", func() { + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Header.Get("Authorization") + })) + DeferCleanup(srv.Close) + useStore(fmt.Sprintf("- match: %s\n bearer: store-token\n allow_insecure: true\n", srv.URL)) + + get(srv.URL, http.Header{"Authorization": {"Bearer explicit"}}) + Expect(seen).To(Equal("Bearer explicit")) + }) + + It("fails the request when the matching rule cannot be resolved", func() { + srv := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + DeferCleanup(srv.Close) + useStore(fmt.Sprintf("- match: %s\n bearer_env: MISSING_TOKEN\n allow_insecure: true\n", srv.URL)) + + req, err := http.NewRequest(http.MethodGet, srv.URL, nil) + Expect(err).NotTo(HaveOccurred()) + _, err = client.Do(req) + Expect(err).To(MatchError(ContainSubstring("MISSING_TOKEN"))) + }) + + It("closes the request body when the matching rule cannot be resolved", func() { + useStore("- match: https://files.example.com\n bearer_env: MISSING_TOKEN\n") + body := &closeRecorder{Reader: strings.NewReader("payload")} + req, err := http.NewRequest(http.MethodPost, "https://files.example.com/upload", body) + Expect(err).NotTo(HaveOccurred()) + + _, err = credentials.Transport(httpclient.HardenedTransport()).RoundTrip(req) + Expect(err).To(HaveOccurred()) + Expect(body.closed).To(BeTrue()) + }) + + It("sends nothing when no store is installed", func() { + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Header.Get("Authorization") + })) + DeferCleanup(srv.Close) + prev := credentials.SetDefault(nil) + DeferCleanup(func() { credentials.SetDefault(prev) }) + + get(srv.URL, nil) + Expect(seen).To(BeEmpty()) + }) +}) + +type closeRecorder struct { + io.Reader + closed bool +} + +func (c *closeRecorder) Close() error { + c.closed = true + return nil +} diff --git a/pkg/downloader/credentials_test.go b/pkg/downloader/credentials_test.go new file mode 100644 index 000000000..6d74de2ca --- /dev/null +++ b/pkg/downloader/credentials_test.go @@ -0,0 +1,213 @@ +package downloader_test + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sync/atomic" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/credentials" + "github.com/mudler/LocalAI/pkg/downloader" +) + +var _ = Describe("downloads with a credentials store", Serial, func() { + useStore := func(doc string) { + s, err := credentials.Parse([]byte(doc), func(string) (string, bool) { return "", false }) + Expect(err).NotTo(HaveOccurred()) + prev := credentials.SetDefault(s) + DeferCleanup(func() { credentials.SetDefault(prev) }) + } + + requireBearer := func(token string, requests *atomic.Int32) *httptest.Server { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Header.Get("Authorization") != "Bearer "+token { + w.WriteHeader(http.StatusUnauthorized) + return + } + _, _ = w.Write([]byte("- name: private-model\n")) + })) + DeferCleanup(srv.Close) + return srv + } + + readGallery := func(rawURL string) (string, error) { + var body string + err := downloader.URI(rawURL).ReadWithAuthorizationAndCallback(context.Background(), GinkgoT().TempDir(), "", + func(_ string, b []byte) error { + body = string(b) + return nil + }) + return body, err + } + + It("authenticates a gallery index read", func() { + var requests atomic.Int32 + srv := requireBearer("gallery-token", &requests) + useStore(fmt.Sprintf("- match: %s\n bearer: gallery-token\n allow_insecure: true\n", srv.URL)) + + body, err := readGallery(srv.URL + "/index.yaml") + Expect(err).NotTo(HaveOccurred()) + Expect(body).To(ContainSubstring("private-model")) + }) + + It("authenticates a file download", func() { + var requests atomic.Int32 + srv := requireBearer("file-token", &requests) + useStore(fmt.Sprintf("- match: %s/models\n bearer: file-token\n allow_insecure: true\n", srv.URL)) + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + Expect(downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil)).To(Succeed()) + data, err := os.ReadFile(target) + Expect(err).NotTo(HaveOccurred()) + Expect(string(data)).To(ContainSubstring("private-model")) + }) + + It("reports a missing rule on 401 without retrying", func() { + var requests atomic.Int32 + srv := requireBearer("file-token", &requests) + useStore("") + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + err := downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil) + var authErr *credentials.AuthError + Expect(errors.As(err, &authErr)).To(BeTrue(), "got %v", err) + Expect(authErr.Match).To(BeEmpty()) + Expect(authErr.Status).To(Equal(http.StatusUnauthorized)) + Expect(requests.Load()).To(BeEquivalentTo(1)) + }) + + It("names the rule a gallery server rejected", func() { + var requests atomic.Int32 + srv := requireBearer("right-token", &requests) + useStore(fmt.Sprintf("- match: %s\n bearer: wrong-token\n allow_insecure: true\n", srv.URL)) + + _, err := readGallery(srv.URL + "/index.yaml") + var authErr *credentials.AuthError + Expect(errors.As(err, &authErr)).To(BeTrue(), "got %v", err) + Expect(authErr.Match).To(Equal(srv.URL)) + Expect(err.Error()).NotTo(ContainSubstring("wrong-token")) + }) + + It("does not treat an unreadable secret as a retryable network error", func() { + var requests atomic.Int32 + srv := requireBearer("file-token", &requests) + useStore(fmt.Sprintf("- match: %s\n bearer_env: MISSING_TOKEN\n allow_insecure: true\n", srv.URL)) + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + err := downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil) + Expect(err).To(MatchError(ContainSubstring("MISSING_TOKEN"))) + Expect(errors.Is(err, credentials.ErrUnresolvedSecret)).To(BeTrue()) + Expect(downloader.IsRetryable(context.Background(), err)).To(BeFalse()) + Expect(requests.Load()).To(BeEquivalentTo(0)) + }) + + It("does not retry an unreadable secret met while probing a resume", func() { + var requests atomic.Int32 + srv := requireBearer("file-token", &requests) + useStore(fmt.Sprintf("- match: %s\n bearer_env: MISSING_TOKEN\n allow_insecure: true\n", srv.URL)) + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + Expect(os.WriteFile(target+".partial", []byte("- name: priv"), 0o600)).To(Succeed()) + err := downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil) + Expect(errors.Is(err, credentials.ErrUnresolvedSecret)).To(BeTrue(), "got %v", err) + Expect(downloader.IsRetryable(context.Background(), err)).To(BeFalse()) + Expect(requests.Load()).To(BeEquivalentTo(0)) + }) + + It("lets an explicit bearer token win over a matching rule", func() { + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Header.Get("Authorization") + _, _ = w.Write([]byte("- name: private-model\n")) + })) + DeferCleanup(srv.Close) + useStore(fmt.Sprintf("- match: %s\n bearer: store-token\n allow_insecure: true\n", srv.URL)) + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + Expect(downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil, + downloader.WithBearerToken("explicit-token"))).To(Succeed()) + Expect(seen).To(Equal("Bearer explicit-token")) + }) + + It("blames the provided bearer token, not the store, when it is rejected", func() { + var requests atomic.Int32 + srv := requireBearer("right-token", &requests) + rule := srv.URL + "/models" + useStore(fmt.Sprintf("- match: %s\n bearer: store-token\n allow_insecure: true\n", rule)) + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + err := downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil, + downloader.WithBearerToken("wrong-token")) + Expect(err).To(MatchError(ContainSubstring("provided credential was rejected"))) + Expect(err).To(MatchError(ContainSubstring("status 401"))) + Expect(err.Error()).NotTo(ContainSubstring(fmt.Sprintf("credential %q", rule))) + Expect(err.Error()).NotTo(ContainSubstring("credentials rule")) + Expect(err.Error()).NotTo(ContainSubstring("wrong-token")) + var authErr *credentials.AuthError + Expect(errors.As(err, &authErr)).To(BeTrue(), "got %v", err) + Expect(authErr.Match).To(BeEmpty()) + }) + + It("blames the provided authorization, not the store, when a gallery rejects it", func() { + var requests atomic.Int32 + srv := requireBearer("right-token", &requests) + useStore(fmt.Sprintf("- match: %s\n bearer: store-token\n allow_insecure: true\n", srv.URL)) + + err := downloader.URI(srv.URL+"/index.yaml").ReadWithAuthorizationAndCallback(context.Background(), GinkgoT().TempDir(), "Bearer wrong-token", + func(string, []byte) error { return nil }) + Expect(err).To(MatchError(ContainSubstring("provided credential was rejected"))) + Expect(err.Error()).NotTo(ContainSubstring("no credentials rule")) + Expect(err.Error()).NotTo(ContainSubstring("wrong-token")) + }) + + It("keeps signed query strings out of auth errors", func() { + var requests atomic.Int32 + srv := requireBearer("right-token", &requests) + useStore("") + + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + err := downloader.URI(srv.URL+"/models/model.yaml?X-Amz-Signature=topsecret").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).NotTo(ContainSubstring("topsecret")) + }) + + It("behaves as before with no store installed", func() { + var seen []string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = append(seen, r.Header.Get("Authorization")) + _, _ = w.Write([]byte("- name: public\n")) + })) + DeferCleanup(srv.Close) + prev := credentials.SetDefault(nil) + DeferCleanup(func() { credentials.SetDefault(prev) }) + + _, err := readGallery(srv.URL + "/index.yaml") + Expect(err).NotTo(HaveOccurred()) + target := filepath.Join(GinkgoT().TempDir(), "model.yaml") + Expect(downloader.URI(srv.URL+"/models/model.yaml").DownloadFileWithContext(context.Background(), target, "", 0, 1, nil, + downloader.WithBearerToken("explicit-token"))).To(Succeed()) + Expect(seen).To(Equal([]string{"", "Bearer explicit-token"})) + }) + + It("keeps anonymous downloads anonymous when no rule matches", func() { + var seen string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + seen = r.Header.Get("Authorization") + _, _ = w.Write([]byte("- name: public\n")) + })) + DeferCleanup(srv.Close) + useStore("- match: https://elsewhere.example.com\n bearer: unrelated\n") + + _, err := readGallery(srv.URL + "/index.yaml") + Expect(err).NotTo(HaveOccurred()) + Expect(seen).To(BeEmpty()) + }) +}) diff --git a/pkg/downloader/uri.go b/pkg/downloader/uri.go index afa5e0d81..8da23931d 100644 --- a/pkg/downloader/uri.go +++ b/pkg/downloader/uri.go @@ -22,6 +22,7 @@ import ( "github.com/mudler/xlog" "github.com/mudler/LocalAI/internal" + "github.com/mudler/LocalAI/pkg/credentials" "github.com/mudler/LocalAI/pkg/httpclient" "github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/utils" @@ -217,7 +218,16 @@ func (uri URI) ReadWithAuthorizationAndCallback(ctx context.Context, basePath st // source was down. DownloadFile has always checked the status; this path // never did. if response.StatusCode >= 400 { - return fmt.Errorf("failed to read url %q, invalid status code %d", url, response.StatusCode) + err := fmt.Errorf("failed to read url %q, invalid status code %d", url, response.StatusCode) + if response.StatusCode == http.StatusUnauthorized || response.StatusCode == http.StatusForbidden { + // The credentials transport leaves a caller-set Authorization + // alone, so a rejection here is about that header, not the store. + if authorization != "" { + return credentials.HTTPProvidedCredentialError(response.Request.URL, response.StatusCode, err) + } + return credentials.HTTPAuthError(response.Request.URL, response.StatusCode, err) + } + return err } // Read the response body @@ -437,11 +447,16 @@ func downloadHTTPClient() *http.Client { defer downloadClientMu.Unlock() if downloadClientCached == nil || downloadClientTimeout != DownloadResponseHeaderTimeout { downloadClientTimeout = DownloadResponseHeaderTimeout - opts := []httpclient.Option{httpclient.WithFollowRedirects()} + base := httpclient.HardenedTransport() + // httpclient only applies the header timeout to a bare *http.Transport, + // and the credential wrapper hides it, so set it on the base directly. if downloadClientTimeout > 0 { - opts = append(opts, httpclient.WithResponseHeaderTimeout(downloadClientTimeout)) + base.ResponseHeaderTimeout = downloadClientTimeout } - downloadClientCached = httpclient.New(opts...) + downloadClientCached = httpclient.New( + httpclient.WithFollowRedirects(), + httpclient.WithTransport(credentials.Transport(base)), + ) } return downloadClientCached } @@ -685,6 +700,9 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string resumable := false if uri.LooksLikeHTTPURL() { support, err := uri.checkServerSupportsRangeHeader(ctx, dopts.bearerToken) + if errors.Is(err, credentials.ErrUnresolvedSecret) { + return fmt.Errorf("failed to check if uri server supports range header: %w", err) + } if err != nil { // The probe only ever fails on transport trouble (the status is // not consulted), so it says nothing permanent about the URL. It @@ -761,6 +779,11 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string } return ctx.Err() } + // An unreadable secret is a configuration problem that a retry + // cannot fix. + if errors.Is(err, credentials.ErrUnresolvedSecret) { + return fmt.Errorf("failed to download file %q: %w", filePath, err) + } // The transport failed before the response was established (reset // connection, refused dial, TLS hiccup). Nothing about it is // specific to this URL, so another attempt may well succeed. @@ -781,6 +804,15 @@ func (uri URI) DownloadFileWithContext(ctx context.Context, filePath, sha string } if resp.StatusCode >= 400 { err := fmt.Errorf("failed to download url %q, invalid status code %d", url, resp.StatusCode) + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden { + _ = resp.Body.Close() + // The credentials transport leaves a WithBearerToken header + // alone, so a rejection here is about that token, not the store. + if dopts.bearerToken != "" { + return credentials.HTTPProvidedCredentialError(resp.Request.URL, resp.StatusCode, err) + } + return credentials.HTTPAuthError(resp.Request.URL, resp.StatusCode, err) + } // 5xx and 429 describe the server's current state, not the request; // every other 4xx (missing file, bad auth) is settled and retrying // it only delays the real error. diff --git a/pkg/oci/auth.go b/pkg/oci/auth.go new file mode 100644 index 000000000..17803056f --- /dev/null +++ b/pkg/oci/auth.go @@ -0,0 +1,28 @@ +package oci + +import ( + "errors" + "net/http" + + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" + + "github.com/mudler/LocalAI/pkg/credentials" +) + +// wrapAuthError turns a registry 401/403 into a credentials.AuthError so the +// operator is told whether a rule was sent, instead of a bare UNAUTHORIZED. +func wrapAuthError(imageRef string, err error) error { + var terr *transport.Error + if !errors.As(err, &terr) { + return err + } + if terr.StatusCode != http.StatusUnauthorized && terr.StatusCode != http.StatusForbidden { + return err + } + matchURL := "" + if ref, perr := name.ParseReference(imageRef); perr == nil { + matchURL = credentials.RegistryURL(ref.Context()) + } + return credentials.NewRegistryAuthError(matchURL, imageRef, terr.StatusCode, err) +} diff --git a/pkg/oci/blob.go b/pkg/oci/blob.go index e034c4162..2eeb08094 100644 --- a/pkg/oci/blob.go +++ b/pkg/oci/blob.go @@ -6,6 +6,7 @@ import ( "io" "os" + "github.com/mudler/LocalAI/pkg/credentials" "github.com/mudler/LocalAI/pkg/xio" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -38,6 +39,7 @@ func FetchImageBlob(ctx context.Context, r, reference, dst string, statusReader Cache: auth.NewCache(), } client.SetUserAgent(UserAgent()) + client.Credential = credentials.OrasCredential(r) repo.Client = client // https://github.com/oras-project/oras/blob/main/cmd/oras/internal/option/remote.go#L364 diff --git a/pkg/oci/cosignverify/verify.go b/pkg/oci/cosignverify/verify.go index 579b0d8c6..a34dca2a6 100644 --- a/pkg/oci/cosignverify/verify.go +++ b/pkg/oci/cosignverify/verify.go @@ -31,6 +31,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/mudler/LocalAI/internal" + "github.com/mudler/LocalAI/pkg/credentials" "github.com/sigstore/sigstore-go/pkg/root" "github.com/sigstore/sigstore-go/pkg/tuf" "github.com/sigstore/sigstore-go/pkg/verify" @@ -303,7 +304,7 @@ func (v *Verifier) remoteOptions(ctx context.Context) []remote.Option { if v.auth != nil { opts = append(opts, remote.WithAuth(staticAuth{auth: v.auth})) } else { - opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + opts = append(opts, remote.WithAuthFromKeychain(credentials.Keychain())) } return opts } diff --git a/pkg/oci/image.go b/pkg/oci/image.go index c44f7e5eb..56f9c9b41 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -26,6 +26,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/google/go-containerregistry/pkg/v1/remote/transport" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/mudler/LocalAI/pkg/credentials" "github.com/mudler/LocalAI/pkg/xio" ) @@ -111,7 +112,7 @@ func newBlobRangeOpener(imageRef string, layer v1.Layer, auth *registrytypes.Aut var authenticator authn.Authenticator if auth != nil { authenticator = staticAuth{auth} - } else if authenticator, err = authn.DefaultKeychain.Resolve(repo.Registry); err != nil { + } else if authenticator, err = credentials.Keychain().Resolve(repo); err != nil { authenticator = authn.Anonymous } blobURL := fmt.Sprintf("%s://%s/v2/%s/blobs/%s", repo.Registry.Scheme(), repo.RegistryStr(), repo.RepositoryStr(), digest.String()) @@ -406,12 +407,12 @@ func GetImage(targetImage, targetPlatform string, auth *registrytypes.AuthConfig if auth != nil { opts = append(opts, remote.WithAuth(staticAuth{auth})) } else { - opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + opts = append(opts, remote.WithAuthFromKeychain(credentials.Keychain())) } image, err = remote.Image(ref, opts...) - return image, err + return image, wrapAuthError(targetImage, err) } // GetImageDigest returns the OCI image digest for the given image reference without downloading it. @@ -454,12 +455,12 @@ func GetImageDigest(targetImage, targetPlatform string, auth *registrytypes.Auth if auth != nil { opts = append(opts, remote.WithAuth(staticAuth{auth})) } else { - opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + opts = append(opts, remote.WithAuthFromKeychain(credentials.Keychain())) } desc, err := remote.Head(ref, opts...) if err != nil { - return "", err + return "", wrapAuthError(targetImage, err) } return desc.Digest.String(), nil diff --git a/pkg/oci/image_auth_internal_test.go b/pkg/oci/image_auth_internal_test.go new file mode 100644 index 000000000..0e50f2d0f --- /dev/null +++ b/pkg/oci/image_auth_internal_test.go @@ -0,0 +1,207 @@ +package oci + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "time" + + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/random" + "github.com/google/go-containerregistry/pkg/v1/remote" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/mudler/LocalAI/pkg/credentials" +) + +const ( + registryUser = "bot" + registryPass = "s3cret" +) + +// requireBasicAuth fronts a registry the way a private registry behaves: every +// request without the right credentials gets a basic-auth challenge. +func requireBasicAuth(inner http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if u, p, ok := r.BasicAuth(); !ok || u != registryUser || p != registryPass { + w.Header().Set("WWW-Authenticate", `Basic realm="localai-test"`) + w.WriteHeader(http.StatusUnauthorized) + return + } + inner.ServeHTTP(w, r) + }) +} + +func useCredentials(doc string) { + s, err := credentials.Parse([]byte(doc), func(string) (string, bool) { return "", false }) + Expect(err).NotTo(HaveOccurred()) + prev := credentials.SetDefault(s) + DeferCleanup(func() { credentials.SetDefault(prev) }) +} + +// isolateDockerConfig stops the test host's docker or podman login from +// satisfying a spec, and returns the directory that holds config.json. +func isolateDockerConfig() string { + home := GinkgoT().TempDir() + dockerDir := filepath.Join(home, ".docker") + Expect(os.MkdirAll(dockerDir, 0o700)).To(Succeed()) + GinkgoT().Setenv("HOME", home) + GinkgoT().Setenv("DOCKER_CONFIG", dockerDir) + GinkgoT().Setenv("REGISTRY_AUTH_FILE", filepath.Join(home, "no-such-auth.json")) + GinkgoT().Setenv("XDG_RUNTIME_DIR", home) + return dockerDir +} + +func pushPrivateImage(serverURL, repoPath string) string { + imageRef := strings.TrimPrefix(serverURL, "http://") + "/" + repoPath + ":latest" + ref, err := name.ParseReference(imageRef) + Expect(err).NotTo(HaveOccurred()) + img, err := random.Image(4096, 1) + Expect(err).NotTo(HaveOccurred()) + Expect(remote.Write(ref, img, remote.WithAuth(&authn.Basic{Username: registryUser, Password: registryPass}))).To(Succeed()) + return imageRef +} + +var _ = Describe("registry credentials", Serial, func() { + var ( + server *httptest.Server + dockerDir string + imageRef string + ) + + BeforeEach(func() { + dockerDir = isolateDockerConfig() + server = httptest.NewServer(requireBasicAuth(registry.New())) + DeferCleanup(server.Close) + imageRef = pushPrivateImage(server.URL, "acme/backend") + }) + + It("pulls a private image with a credentials rule", func() { + useCredentials(fmt.Sprintf("- match: %s/acme\n username: %s\n password: %s\n allow_insecure: true\n", server.URL, registryUser, registryPass)) + + _, err := GetImage(imageRef, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + digest, err := GetImageDigest(imageRef, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + Expect(digest).To(HavePrefix("sha256:")) + }) + + It("reports that no rule matches when the registry demands auth", func() { + useCredentials("") + + _, err := GetImage(imageRef, "", nil, nil) + var authErr *credentials.AuthError + Expect(errors.As(err, &authErr)).To(BeTrue(), "got %v", err) + Expect(authErr.Match).To(BeEmpty()) + Expect(authErr.Status).To(Equal(http.StatusUnauthorized)) + }) + + It("names the rule the registry rejected", func() { + rule := server.URL + "/acme" + useCredentials(fmt.Sprintf("- match: %s\n username: %s\n password: not-it\n allow_insecure: true\n", rule, registryUser)) + + _, err := GetImageDigest(imageRef, "", nil, nil) + var authErr *credentials.AuthError + Expect(errors.As(err, &authErr)).To(BeTrue(), "got %v", err) + Expect(authErr.Match).To(Equal(rule)) + Expect(err.Error()).NotTo(ContainSubstring("not-it")) + }) + + It("still authenticates from docker config when no rule matches", func() { + host := strings.TrimPrefix(server.URL, "http://") + encoded := base64.StdEncoding.EncodeToString([]byte(registryUser + ":" + registryPass)) + Expect(os.WriteFile(filepath.Join(dockerDir, "config.json"), + []byte(fmt.Sprintf(`{"auths":{%q:{"auth":%q}}}`, host, encoded)), 0o600)).To(Succeed()) + useCredentials("") + + _, err := GetImage(imageRef, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + }) + + It("authenticates the Range requests that resume an interrupted layer", func() { + reg := &droppingBlobRegistry{inner: requireBasicAuth(registry.New())} + dropping := httptest.NewServer(reg) + DeferCleanup(dropping.Close) + prev := layerRetryBackoff + layerRetryBackoff = func(int) time.Duration { return 0 } + DeferCleanup(func() { layerRetryBackoff = prev }) + + ref := pushPrivateImage(dropping.URL, "acme/backend") + useCredentials(fmt.Sprintf("- match: %s/acme\n username: %s\n password: %s\n allow_insecure: true\n", dropping.URL, registryUser, registryPass)) + + pulled, err := GetImage(ref, "", nil, nil) + Expect(err).NotTo(HaveOccurred()) + tarPath := filepath.Join(GinkgoT().TempDir(), "image.tar") + Expect(DownloadOCIImageTar(context.Background(), pulled, ref, tarPath, nil)).To(Succeed()) + + reg.mu.Lock() + defer reg.mu.Unlock() + Expect(reg.rangeRequests).NotTo(BeEmpty()) + }) +}) + +var _ = Describe("FetchImageBlob registry credentials", Serial, func() { + var ( + server *httptest.Server + repo string + digest string + ) + + BeforeEach(func() { + isolateDockerConfig() + // oras only speaks HTTPS unless a repository opts into plain HTTP, + // which FetchImageBlob does not, so the registry has to serve TLS. + server = httptest.NewTLSServer(requireBasicAuth(registry.New())) + DeferCleanup(server.Close) + + // FetchImageBlob builds its oras client on retry.DefaultClient, whose + // transport falls back to http.DefaultTransport on every request. + // Swapping it for the test server's client is the only way to trust + // the test certificate without a production seam; the spec is Serial + // so nothing else sees the swap. + prev := http.DefaultTransport + http.DefaultTransport = server.Client().Transport + DeferCleanup(func() { http.DefaultTransport = prev }) + + repo = strings.TrimPrefix(server.URL, "https://") + "/acme/model" + ref, err := name.ParseReference(repo + ":latest") + Expect(err).NotTo(HaveOccurred()) + img, err := random.Image(4096, 1) + Expect(err).NotTo(HaveOccurred()) + Expect(remote.Write(ref, img, + remote.WithAuth(&authn.Basic{Username: registryUser, Password: registryPass}), + remote.WithTransport(server.Client().Transport))).To(Succeed()) + layers, err := img.Layers() + Expect(err).NotTo(HaveOccurred()) + d, err := layers[0].Digest() + Expect(err).NotTo(HaveOccurred()) + digest = d.String() + }) + + It("fetches a private blob with a credentials rule", func() { + useCredentials(fmt.Sprintf("- match: %s/acme\n username: %s\n password: %s\n", server.URL, registryUser, registryPass)) + + dst := filepath.Join(GinkgoT().TempDir(), "blob") + Expect(FetchImageBlob(context.Background(), repo, digest, dst, nil)).To(Succeed()) + info, err := os.Stat(dst) + Expect(err).NotTo(HaveOccurred()) + Expect(info.Size()).To(BeNumerically(">", 0)) + }) + + It("fails to fetch a private blob when no rule matches", func() { + useCredentials("") + + dst := filepath.Join(GinkgoT().TempDir(), "blob") + err := FetchImageBlob(context.Background(), repo, digest, dst, nil) + Expect(err).To(MatchError(ContainSubstring("basic credential not found"))) + }) +}) From 66fa90a5030163b219227ee031a049e7ac211174 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 14:36:02 +0200 Subject: [PATCH 11/86] chore: :arrow_up: Update CrispStrobe/CrispASR to `a4db74cfd13c8674a7815608baaa505e673ce930` (#12035) :arrow_up: Update CrispStrobe/CrispASR Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/go/crispasr/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/go/crispasr/Makefile b/backend/go/crispasr/Makefile index 521578eb8..9714eaa50 100644 --- a/backend/go/crispasr/Makefile +++ b/backend/go/crispasr/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # CrispASR version (release tag) CRISPASR_REPO?=https://github.com/CrispStrobe/CrispASR -CRISPASR_VERSION?=ab87fba65a4ade1ef30ccaa881458a0914aae557 +CRISPASR_VERSION?=a4db74cfd13c8674a7815608baaa505e673ce930 SO_TARGET?=libgocrispasr.so CMAKE_ARGS+=-DBUILD_SHARED_LIBS=OFF From cf892965591d998b4a943123fb70729a76393d6d Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 14:36:15 +0200 Subject: [PATCH 12/86] chore: :arrow_up: Update 0xShug0/audio.cpp to `ff1bcc4555ff99c4383329b0b21b52a18cc8b3cd` (#12033) :arrow_up: Update 0xShug0/audio.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> --- backend/cpp/audio-cpp/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/cpp/audio-cpp/Makefile b/backend/cpp/audio-cpp/Makefile index 87e9cb732..eab9f9463 100644 --- a/backend/cpp/audio-cpp/Makefile +++ b/backend/cpp/audio-cpp/Makefile @@ -9,7 +9,7 @@ # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean # rebuild and so the bump bot can see the pin. -AUDIO_CPP_VERSION?=5ba81ac54fb071b835680973f8868546b4db372b +AUDIO_CPP_VERSION?=ff1bcc4555ff99c4383329b0b21b52a18cc8b3cd AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) From c50b46996cacb61d5290af9dba6e3c0efff8fd78 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:24:31 +0200 Subject: [PATCH 13/86] chore: :arrow_up: Update antirez/ds4 to `a04f46fa423e45712c8c7e430eff422479f314a3` (#12013) * :arrow_up: Update antirez/ds4 Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(ds4): link engram support in GPU builds The bumped ds4 engine moved DeepSeek v4.1 engram helpers into a new object. Build and link that object for CUDA and Metal while retaining the upstream CPU object set. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/ds4/CMakeLists.txt | 8 ++++++-- backend/cpp/ds4/Makefile | 16 ++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/backend/cpp/ds4/CMakeLists.txt b/backend/cpp/ds4/CMakeLists.txt index 5783db942..b22b22044 100644 --- a/backend/cpp/ds4/CMakeLists.txt +++ b/backend/cpp/ds4/CMakeLists.txt @@ -70,6 +70,7 @@ target_include_directories(hw_grpc_proto PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) set(DS4_OBJS "${DS4_DIR}/ds4.o") if(DS4_GPU STREQUAL "cuda") list(APPEND DS4_OBJS + "${DS4_DIR}/ds4_engram.o" "${DS4_DIR}/ds4_cuda.o" "${DS4_DIR}/cuda/mmq/ds4_ggml_stubs.o" "${DS4_DIR}/cuda/mmq/ds4_mmq.o" @@ -79,14 +80,17 @@ if(DS4_GPU STREQUAL "cuda") "${DS4_DIR}/cuda/mmq/mmvq.o" "${DS4_DIR}/cuda/mmq/ds4_repack.o") elseif(DS4_GPU STREQUAL "metal") - list(APPEND DS4_OBJS "${DS4_DIR}/ds4_metal.o") + list(APPEND DS4_OBJS + "${DS4_DIR}/ds4_engram.o" + "${DS4_DIR}/ds4_metal.o") elseif(DS4_GPU STREQUAL "cpu") set(DS4_OBJS "${DS4_DIR}/ds4_cpu.o") endif() # Upstream splits image preprocessing, distributed inference, tensor-parallel # transport, the SSD expert cache, and layer placement into GPU-agnostic -# translation units. Link them regardless of DS4_GPU. +# translation units. Link the common subset regardless of DS4_GPU; engram +# lookup is part of the CUDA and Metal engines only. list(APPEND DS4_OBJS "${DS4_DIR}/ds4_image.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_distributed.o") list(APPEND DS4_OBJS "${DS4_DIR}/ds4_tp.o") diff --git a/backend/cpp/ds4/Makefile b/backend/cpp/ds4/Makefile index 39ab3c760..1b66811ad 100644 --- a/backend/cpp/ds4/Makefile +++ b/backend/cpp/ds4/Makefile @@ -1,10 +1,10 @@ # ds4 backend Makefile. # -# Upstream pin lives below as DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804 +# Upstream pin lives below as DS4_VERSION?=a04f46fa423e45712c8c7e430eff422479f314a3 # (.github/bump_deps.sh) can find and update it - matches the # llama-cpp / ik-llama-cpp / turboquant convention. -DS4_VERSION?=6289c516273979173abbc062209a81dd3706b804 +DS4_VERSION?=a04f46fa423e45712c8c7e430eff422479f314a3 DS4_REPO?=https://github.com/antirez/ds4 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST)))) @@ -80,17 +80,17 @@ endif endif # Upstream splits image preprocessing, distributed inference, tensor-parallel -# transport, the SSD expert cache, and layer placement into GPU-agnostic -# translation units. They are shared by every GPU mode, so append them -# unconditionally below. +# transport, the SSD expert cache, layer placement, and engram lookup into +# GPU-agnostic translation units. They are shared by every GPU mode, so append +# them unconditionally below. ifeq ($(BUILD_TYPE),cublas) CMAKE_ARGS += -DDS4_GPU=cuda - DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o \ + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_cuda.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_engram.o \ cuda/mmq/ds4_ggml_stubs.o cuda/mmq/ds4_mmq.o cuda/mmq/ds4_mmq_d2r.o \ cuda/mmq/quantize.o cuda/mmq/mmid.o cuda/mmq/mmvq.o cuda/mmq/ds4_repack.o else ifeq ($(UNAME_S),Darwin) CMAKE_ARGS += -DDS4_GPU=metal - DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + DS4_OBJ_TARGET := ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_engram.o else # CPU reference path (Linux only - macOS CPU path is broken by VM bug per ds4 README). CMAKE_ARGS += -DDS4_GPU=cpu @@ -121,7 +121,7 @@ ds4/ds4.o: ds4 ifeq ($(BUILD_TYPE),cublas) +$(MAKE) -C ds4 $(DS4_ARCH_MAKEVARS) $(DS4_OBJ_TARGET) else ifeq ($(UNAME_S),Darwin) - +$(MAKE) -C ds4 ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o + +$(MAKE) -C ds4 ds4.o ds4_image.o ds4_metal.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o ds4_engram.o else +$(MAKE) -C ds4 ds4_cpu.o ds4_image.o ds4_distributed.o ds4_tp.o ds4_ssd.o ds4_layer_pack.o endif From 13b780d38759d225992fa36d42593a844276a5aa Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:24:35 +0200 Subject: [PATCH 14/86] chore: :arrow_up: Update mudler/vllm.cpp to `60990ee784101f74f6d1775575e9e89dfb26f73a` (#12014) * :arrow_up: Update mudler/vllm.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(vllm-cpp): mirror ABI v26 model params Mirror the new KV-cache dtype and sliding-window fields so the Go POD matches the bumped vllm.cpp header on LP64. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/go/vllm-cpp/Makefile | 2 +- backend/go/vllm-cpp/govllmcpp.go | 47 +++++++++++++++-------------- backend/go/vllm-cpp/vllmcpp_test.go | 22 ++++++++------ 3 files changed, 37 insertions(+), 34 deletions(-) diff --git a/backend/go/vllm-cpp/Makefile b/backend/go/vllm-cpp/Makefile index 598d814c3..0e379f090 100644 --- a/backend/go/vllm-cpp/Makefile +++ b/backend/go/vllm-cpp/Makefile @@ -11,7 +11,7 @@ JOBS?=$(shell nproc --ignore=1 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || e # vllm.cpp version VLLM_CPP_REPO?=https://github.com/mudler/vllm.cpp -VLLM_CPP_VERSION?=6bf3abb580982f4fd2e4525ef37802ee0ce28981 +VLLM_CPP_VERSION?=60990ee784101f74f6d1775575e9e89dfb26f73a # MLX GEMM provider (darwin/metal only; see the metal branch below for why). # Consumed as the prebuilt pip wheel: building MLX from source needs `xcrun diff --git a/backend/go/vllm-cpp/govllmcpp.go b/backend/go/vllm-cpp/govllmcpp.go index 20e587a6b..db7555cd2 100644 --- a/backend/go/vllm-cpp/govllmcpp.go +++ b/backend/go/vllm-cpp/govllmcpp.go @@ -1,6 +1,6 @@ package main -// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v23). +// purego bindings for the vllm.cpp stable C ABI (include/vllm.h, ABI v26). // // The structs below are hand-mirrored PODs of the C declarations, with // explicit padding so the Go layout matches the C layout on linux/darwin @@ -21,7 +21,7 @@ import ( // the header of the VLLM_CPP_VERSION pinned in the Makefile: the build checks // the two against each other, because a mismatch is only caught at runtime by // registerLib, where it takes the backend down on every load (issue #11379). -const abiVersion = 23 +const abiVersion = 26 // The ABI's tri-state toggles (enable_prefix_caching ABI v7, // enable_jump_forward ABI v10) share one encoding: 0 is NOT "off", it is @@ -51,28 +51,28 @@ const ( vllmOK = 0 ) -// cModelParams mirrors vllm_model_params. The int32 fields sit in pairs so the -// interior needs no padding on LP64, but the struct is 8-aligned (it holds -// pointers) and ends on a lone int32, so the trailing pad is explicit. Offsets -// and total size are asserted in vllmcpp_test.go. +// cModelParams mirrors vllm_model_params. Go's natural alignment and the +// explicit pad after LanguageModelOnly match the C layout on LP64. Offsets and +// total size are asserted in vllmcpp_test.go. type cModelParams struct { - ModelPath uintptr // const char* - TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) - BlockSize int32 - NumBlocks int32 - MaxModelLen int32 - MaxNumSeqs int32 - ToolParser uintptr // const char*; NULL = auto-detect (ABI v4) - ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5) - SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6) - EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7) - MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) - SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) - KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) - OffloadConfig uintptr // const char* JSON; NULL = no weight offload - EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) - // v14/v16 tail. LocalAI sets none of these (0 is "auto" for the device and - // "unset" for both sizing knobs, i.e. the pre-v14 engine byte for byte), but + ModelPath uintptr // const char* + TokenizerConfigPath uintptr // const char*; NULL = /... (ABI v9) + BlockSize int32 + NumBlocks int32 + MaxModelLen int32 + MaxNumSeqs int32 + ToolParser uintptr // const char*; NULL = auto-detect (ABI v4) + ReasoningParser uintptr // const char*; NULL = auto-detect (ABI v5) + SpeculativeConfig uintptr // const char* JSON; NULL = no speculation (ABI v6) + EnablePrefixCaching int32 // tri-state 0/1/2 (ABI v7) + MaxNumBatchedTokens int32 // <= 0 = per-arch default (ABI v9) + SchedulingPolicy uintptr // const char*; NULL = "fcfs" (ABI v9) + KVTransferConfig uintptr // const char* JSON; NULL = no connector (ABI v9) + OffloadConfig uintptr // const char* JSON; NULL = no weight offload + EnableJumpForward int32 // tri-state 0/1/2 (ABI v10) + DisableSlidingWindow int32 // tri-state 0/1/2 (ABI v26) + // LocalAI sets none of the device and sizing fields below (0 is "auto" for + // the device and "unset" for both sizing knobs), but // the fields MUST be mirrored: the C side reads sizeof(vllm_model_params) // bytes off the pointer we hand it, so a Go struct that stopped at // EnableJumpForward would have vllm_engine_load read 24 bytes past our @@ -84,6 +84,7 @@ type cModelParams struct { _ [4]byte LimitMMPerPrompt uintptr // const char* JSON; NULL = default limits (ABI v19) MMProjPath uintptr // const char*; NULL = no GGUF projector (ABI v22) + KVCacheDType uintptr // const char*; NULL = auto (ABI v24) } // cSamplingParams mirrors vllm_sampling_params (structured fields included). diff --git a/backend/go/vllm-cpp/vllmcpp_test.go b/backend/go/vllm-cpp/vllmcpp_test.go index 55f6e211a..201fd1692 100644 --- a/backend/go/vllm-cpp/vllmcpp_test.go +++ b/backend/go/vllm-cpp/vllmcpp_test.go @@ -16,7 +16,7 @@ func TestVllmCpp(t *testing.T) { RunSpecs(t, "vllm-cpp suite") } -// The Go POD mirrors must match the C struct layout of vllm.h (ABI v23) +// The Go POD mirrors must match the C struct layout of vllm.h (ABI v26) // byte-for-byte: these offsets are the C offsets on LP64 (linux/darwin // amd64+arm64). A failure here means govllmcpp.go drifted from vllm.h. var _ = Describe("C ABI struct mirrors", func() { @@ -24,7 +24,7 @@ var _ = Describe("C ABI struct mirrors", func() { // VLLM_ABI_VERSION in the vllm.h of VLLM_CPP_VERSION (Makefile). // Moving the pin past this without growing the mirrors below ships a // backend that refuses every load at startup (issue #11379). - Expect(abiVersion).To(Equal(23)) + Expect(abiVersion).To(Equal(26)) }) It("cModelParams matches vllm_model_params", func() { @@ -44,15 +44,17 @@ var _ = Describe("C ABI struct mirrors", func() { Expect(unsafe.Offsetof(p.KVTransferConfig)).To(Equal(uintptr(72))) Expect(unsafe.Offsetof(p.OffloadConfig)).To(Equal(uintptr(80))) Expect(unsafe.Offsetof(p.EnableJumpForward)).To(Equal(uintptr(88))) - Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(92))) - // 96: gpu_memory_utilization is a double, so it takes the next + Expect(unsafe.Offsetof(p.DisableSlidingWindow)).To(Equal(uintptr(92))) + Expect(unsafe.Offsetof(p.Device)).To(Equal(uintptr(96))) + // 104: gpu_memory_utilization is a double, so it takes the next // 8-aligned slot after the int32 pair. Go pads identically. - Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(96))) - Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(104))) - Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(112))) - Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(120))) - Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(128))) - Expect(unsafe.Sizeof(p)).To(Equal(uintptr(136))) + Expect(unsafe.Offsetof(p.GPUMemoryUtil)).To(Equal(uintptr(104))) + Expect(unsafe.Offsetof(p.KVCacheMemoryBytes)).To(Equal(uintptr(112))) + Expect(unsafe.Offsetof(p.LanguageModelOnly)).To(Equal(uintptr(120))) + Expect(unsafe.Offsetof(p.LimitMMPerPrompt)).To(Equal(uintptr(128))) + Expect(unsafe.Offsetof(p.MMProjPath)).To(Equal(uintptr(136))) + Expect(unsafe.Offsetof(p.KVCacheDType)).To(Equal(uintptr(144))) + Expect(unsafe.Sizeof(p)).To(Equal(uintptr(152))) }) It("cSamplingParams matches vllm_sampling_params (ABI v8)", func() { From 6ca6a073e9a4a16650d75f652200b8778112205c Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:27:38 +0200 Subject: [PATCH 15/86] chore: :arrow_up: Update ggml-org/llama.cpp to `5f436dddb440a288ee5611d7d1eca564a6aca9f4` (#12008) * :arrow_up: Update ggml-org/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(llama-cpp): disable PCH for SYCL builds IntelLLVM cannot bundle the precompiled headers introduced by the latest llama.cpp update. Disable PCH for both SYCL variants while retaining it for the other backends. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/llama-cpp/Makefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/cpp/llama-cpp/Makefile b/backend/cpp/llama-cpp/Makefile index 739757999..6320042b2 100644 --- a/backend/cpp/llama-cpp/Makefile +++ b/backend/cpp/llama-cpp/Makefile @@ -1,5 +1,5 @@ -LLAMA_VERSION?=df03399b885831b2a1603b3abb0d8c156808e363 +LLAMA_VERSION?=5f436dddb440a288ee5611d7d1eca564a6aca9f4 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp CMAKE_ARGS?= @@ -67,6 +67,7 @@ ifeq ($(BUILD_TYPE),sycl_f16) CMAKE_ARGS+=-DGGML_SYCL=ON \ -DCMAKE_C_COMPILER=icx \ -DCMAKE_CXX_COMPILER=icpx \ + -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ -DCMAKE_CXX_FLAGS="-fsycl" \ -DGGML_SYCL_F16=ON endif @@ -75,6 +76,7 @@ ifeq ($(BUILD_TYPE),sycl_f32) CMAKE_ARGS+=-DGGML_SYCL=ON \ -DCMAKE_C_COMPILER=icx \ -DCMAKE_CXX_COMPILER=icpx \ + -DCMAKE_DISABLE_PRECOMPILE_HEADERS=ON \ -DCMAKE_CXX_FLAGS="-fsycl" endif From 21d33a1ce2c84d418911c6c7e74f3606c43d4e63 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:29:34 +0200 Subject: [PATCH 16/86] chore: :arrow_up: Update leejet/stable-diffusion.cpp to `42d6c0ab92fe6595776b28e3f7c8925db79b31f5` (#12034) * :arrow_up: Update leejet/stable-diffusion.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(stablediffusion): use effective video frame rate Capture the frame rate selected by stable-diffusion.cpp and use it when muxing the generated frames. This keeps video timing aligned with upstream model-specific adjustments. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/go/stablediffusion-ggml/Makefile | 2 +- backend/go/stablediffusion-ggml/cpp/gosd.cpp | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/backend/go/stablediffusion-ggml/Makefile b/backend/go/stablediffusion-ggml/Makefile index cca0daca6..7dca7e39e 100644 --- a/backend/go/stablediffusion-ggml/Makefile +++ b/backend/go/stablediffusion-ggml/Makefile @@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1) # stablediffusion.cpp (ggml) STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp -STABLEDIFFUSION_GGML_VERSION?=7f410a3793c5bba8eb198e962ce7a3d6095f9d89 +STABLEDIFFUSION_GGML_VERSION?=42d6c0ab92fe6595776b28e3f7c8925db79b31f5 CMAKE_ARGS+=-DGGML_MAX_NAME=128 diff --git a/backend/go/stablediffusion-ggml/cpp/gosd.cpp b/backend/go/stablediffusion-ggml/cpp/gosd.cpp index b876df256..4a1911015 100644 --- a/backend/go/stablediffusion-ggml/cpp/gosd.cpp +++ b/backend/go/stablediffusion-ggml/cpp/gosd.cpp @@ -1410,9 +1410,10 @@ int gen_video(sd_vid_gen_params_t *p, int steps, char *dst, float cfg_scale, int // Generate int num_frames_out = 0; + int effective_fps = fps; sd_image_t* frames = nullptr; sd_audio_t* audio = nullptr; - bool ok = generate_video(sd_c, p, &frames, &num_frames_out, &audio); + bool ok = generate_video(sd_c, p, &frames, &num_frames_out, &audio, &effective_fps); std::free(p); if (!ok || !frames || num_frames_out == 0) { @@ -1425,7 +1426,7 @@ int gen_video(sd_vid_gen_params_t *p, int steps, char *dst, float cfg_scale, int fprintf(stderr, "Generated %d frames, muxing to %s via ffmpeg\n", num_frames_out, dst); - int rc = ffmpeg_mux_raw_to_mp4(frames, num_frames_out, fps, audio, dst); + int rc = ffmpeg_mux_raw_to_mp4(frames, num_frames_out, effective_fps, audio, dst); for (int i = 0; i < num_frames_out; i++) { if (frames[i].data) free(frames[i].data); From 9c157b88bb34aab67b14533acd96298faff99f12 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 15:40:14 +0200 Subject: [PATCH 17/86] chore: :arrow_up: Update PrismML-Eng/llama.cpp to `d8f26eec76da6d09bb708bcba51ef64b8cd868a3` (#12005) * :arrow_up: Update PrismML-Eng/llama.cpp Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(bonsai): follow updated fork APIs The updated PrismML fork implements common_json_error directly, so preserve the shared gRPC source instead of rewriting it to the removed nlohmann exception type. Build and copy the renamed ggml-rpc-server target as well. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/bonsai/Makefile | 8 +++---- backend/cpp/bonsai/patch-grpc-server.sh | 24 --------------------- scripts/build/bonsai-json-compat_test.sh | 27 +++++++++--------------- 3 files changed, 13 insertions(+), 46 deletions(-) delete mode 100644 backend/cpp/bonsai/patch-grpc-server.sh diff --git a/backend/cpp/bonsai/Makefile b/backend/cpp/bonsai/Makefile index 522791d42..4d7e462c5 100644 --- a/backend/cpp/bonsai/Makefile +++ b/backend/cpp/bonsai/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of the `prism` branch on https://github.com/PrismML-Eng/llama.cpp. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -BONSAI_VERSION?=312bb2a93ea2bf798333fa859614fbf913ecb9e2 +BONSAI_VERSION?=d8f26eec76da6d09bb708bcba51ef64b8cd868a3 LLAMA_REPO?=https://github.com/PrismML-Eng/llama.cpp CMAKE_ARGS?= @@ -41,7 +41,6 @@ define bonsai-build # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build purge - bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-$(1)-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:$(1)$(RESET)) @@ -80,7 +79,6 @@ bonsai-cpu-all: # and are applied by apply-patches.sh below. rm -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/patches $(MAKE) -C $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build purge - bash $(CURRENT_MAKEFILE_DIR)/patch-grpc-server.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-score-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp bash $(LLAMA_CPP_DIR)/disable-tts-task.sh $(CURRENT_MAKEFILE_DIR)/../bonsai-cpu-all-build/grpc-server.cpp $(info $(GREEN)I bonsai build info:cpu-all-variants$(RESET)) @@ -96,10 +94,10 @@ bonsai-cpu-all: @echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/ bonsai-grpc: - $(call bonsai-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target rpc-server) + $(call bonsai-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target ggml-rpc-server) bonsai-rpc-server: bonsai-grpc - cp -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-grpc-build/llama.cpp/build/bin/rpc-server bonsai-rpc-server + cp -rf $(CURRENT_MAKEFILE_DIR)/../bonsai-grpc-build/llama.cpp/build/bin/ggml-rpc-server bonsai-rpc-server package: bash package.sh diff --git a/backend/cpp/bonsai/patch-grpc-server.sh b/backend/cpp/bonsai/patch-grpc-server.sh deleted file mode 100644 index aa9b23a50..000000000 --- a/backend/cpp/bonsai/patch-grpc-server.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash -# Adapt the shared llama.cpp gRPC source to the older JSON API in Bonsai. - -set -euo pipefail - -if [[ $# -ne 1 ]]; then - echo "usage: $0 " >&2 - exit 2 -fi - -SRC=$1 -if [[ ! -f "$SRC" ]]; then - echo "grpc-server.cpp not found at $SRC" >&2 - exit 2 -fi - -if grep -q 'common_json_error' "$SRC"; then - echo "==> patching $SRC to use the Bonsai JSON exception type" - awk '{ gsub(/common_json_error/, "json::parse_error"); print }' "$SRC" > "$SRC.tmp" - mv "$SRC.tmp" "$SRC" - echo "==> Bonsai JSON exception patch OK" -else - echo "==> $SRC already uses a Bonsai-compatible JSON exception type, skipping" -fi diff --git a/scripts/build/bonsai-json-compat_test.sh b/scripts/build/bonsai-json-compat_test.sh index c0cfc4f88..442cc8873 100644 --- a/scripts/build/bonsai-json-compat_test.sh +++ b/scripts/build/bonsai-json-compat_test.sh @@ -2,24 +2,17 @@ set -euo pipefail ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -PATCHER="$ROOT/backend/cpp/bonsai/patch-grpc-server.sh" -WORK=$(mktemp -d) -trap 'rm -rf "$WORK"' EXIT +MAKEFILE="$ROOT/backend/cpp/bonsai/Makefile" +GRPC_SERVER="$ROOT/backend/cpp/llama-cpp/grpc-server.cpp" -cat > "$WORK/grpc-server.cpp" <<'EOF' -try { - json::parse("{"); -} catch (const common_json_error& e) { -} -EOF +grep -q 'catch (const common_json_error& e)' "$GRPC_SERVER" -bash "$PATCHER" "$WORK/grpc-server.cpp" -grep -q 'catch (const json::parse_error& e)' "$WORK/grpc-server.cpp" -! grep -q 'common_json_error' "$WORK/grpc-server.cpp" +if grep -q 'patch-grpc-server.sh' "$MAKEFILE"; then + echo "Bonsai must preserve the common_json_error API provided by its pinned fork" >&2 + exit 1 +fi -# A repeated preparation pass must not change the generated source. -cp "$WORK/grpc-server.cpp" "$WORK/once.cpp" -bash "$PATCHER" "$WORK/grpc-server.cpp" -cmp "$WORK/once.cpp" "$WORK/grpc-server.cpp" +grep -q -- '--target grpc-server --target ggml-rpc-server' "$MAKEFILE" +grep -q '/llama.cpp/build/bin/ggml-rpc-server bonsai-rpc-server' "$MAKEFILE" -echo "PASS: Bonsai uses its fork-compatible JSON exception" +echo "PASS: Bonsai uses its pinned fork's JSON and RPC APIs" From f9dab888fe18288a7f2f4e71d1cab42d9058f84b Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Mon, 14 Sep 2026 16:07:29 +0200 Subject: [PATCH 18/86] chore: :arrow_up: Update TheTom/llama-cpp-turboquant to `407f3237bfb3eeaff61546797de3d8c1a96be748` (#12003) * :arrow_up: Update TheTom/llama-cpp-turboquant Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * fix(turboquant): follow renamed RPC server target The pinned TurboQuant fork now exposes the RPC server as ggml-rpc-server. Build that target and copy its renamed artifact while preserving LocalAI's packaged turboquant-rpc-server name. Assisted-by: Codex:gpt-5 Signed-off-by: Ettore Di Giacinto --------- Signed-off-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Signed-off-by: Ettore Di Giacinto Co-authored-by: mudler <2420543+mudler@users.noreply.github.com> Co-authored-by: Ettore Di Giacinto --- backend/cpp/turboquant/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/cpp/turboquant/Makefile b/backend/cpp/turboquant/Makefile index 9a70bb1b5..27a113796 100644 --- a/backend/cpp/turboquant/Makefile +++ b/backend/cpp/turboquant/Makefile @@ -1,7 +1,7 @@ # Pinned to the HEAD of feature/turboquant-kv-cache on https://github.com/TheTom/llama-cpp-turboquant. # Auto-bumped nightly by .github/workflows/bump_deps.yaml. -TURBOQUANT_VERSION?=8a891f4b566efdbd3cea92fafee3227a0a267683 +TURBOQUANT_VERSION?=407f3237bfb3eeaff61546797de3d8c1a96be748 LLAMA_REPO?=https://github.com/TheTom/llama-cpp-turboquant CMAKE_ARGS?= @@ -101,10 +101,10 @@ turboquant-cpu-all: @echo "Collected ggml shared backends:" && ls -la ggml-shared-libs/ turboquant-grpc: - $(call turboquant-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target rpc-server) + $(call turboquant-build,grpc,-DGGML_RPC=ON -DGGML_AVX=off -DGGML_AVX2=off -DGGML_AVX512=off -DGGML_FMA=off -DGGML_F16C=off -DGGML_BMI2=off,--target grpc-server --target ggml-rpc-server) turboquant-rpc-server: turboquant-grpc - cp -rf $(CURRENT_MAKEFILE_DIR)/../turboquant-grpc-build/llama.cpp/build/bin/rpc-server turboquant-rpc-server + cp -rf $(CURRENT_MAKEFILE_DIR)/../turboquant-grpc-build/llama.cpp/build/bin/ggml-rpc-server turboquant-rpc-server package: bash package.sh From 997d403de43f2ac573b4450a41bb648750b4d8c1 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 15 Sep 2026 00:57:14 +0200 Subject: [PATCH 19/86] feat(nodes): add fleet operations dashboard (#12046) * feat(nodes): report CPU telemetry Assisted-by: Codex:gpt-6 * feat(nodes): add fleet view utilities Assisted-by: Codex:gpt-6 * feat(nodes): add fleet operations dashboard Replace the panel roster with aggregate capacity gauges, fleet filtering and selection, bounded bulk actions, and an on-demand node inspector. Extend node details and distributed-mode documentation with CPU and models-disk telemetry. Assisted-by: Codex:gpt-6 * fix(nodes): harden fleet lifecycle actions Assisted-by: Codex:gpt-6 * fix(nodes): restore compact fleet composition Keep fleet health, capacity, and attention in one compact overview at ordinary desktop widths. The inspector now overlays the roster until the workbench can preserve a useful table beside it. Assisted-by: Codex:gpt-6 * feat(nodes): add accessible running models workbench Assisted-by: Codex:gpt-6 * fix(nodes): correct model view ARIA links Keep each tab panel available for its controlling tab while native hidden state removes inactive content from accessibility navigation. Model controls now expose only supported state and valid inspector relationships. Assisted-by: Codex:gpt-6 * fix(nodes): align lifecycle and capacity states Pending nodes now expose approval wherever node actions appear, while other lifecycle controls follow the server transition rules. Capacity totals exclude incomplete readings so missing availability remains unknown. Assisted-by: Codex:gpt-6 * fix(nodes): restore approved dashboard composition Assisted-by: Codex:gpt-6 * fix(nodes): integrate operate navigation Assisted-by: Codex:gpt-6 * fix(nodes): restore low density fleet view Assisted-by: Codex:gpt-6 * fix(nodes): preserve complete operate menu Assisted-by: Codex:gpt-6 * fix(nodes): preserve inspector workspace height Assisted-by: Codex:gpt-6 * fix(nodes): restore standard operate navigation Assisted-by: Codex:gpt-6 * feat(ui): add collapsible console rail Assisted-by: Codex:gpt-6 * feat(nodes): stop models from fleet view Assisted-by: Codex:gpt-6 * fix(nodes): make inspector a full height drawer Assisted-by: Codex:gpt-6 * fix(model): stop mixed local and remote placements Assisted-by: Codex:gpt-6 * fix(ui): announce action menu navigation Assisted-by: Codex:gpt-6 * fix(nodes): keep inspector within viewport Assisted-by: Codex:gpt-6 * fix(ui): preserve focus across model actions Assisted-by: Codex:gpt-6 --------- Co-authored-by: Ettore Di Giacinto --- core/http/endpoints/localai/nodes.go | 41 +- core/http/endpoints/localai/nodes_test.go | 65 ++ core/http/react-ui/e2e/console-narrow.spec.js | 12 + .../e2e/installed-model-logs-link.spec.js | 20 + core/http/react-ui/e2e/navigation.spec.js | 22 + core/http/react-ui/e2e/nodes-detail.spec.js | 58 +- .../e2e/nodes-fleet-dashboard.spec.js | 747 ++++++++++++++++++ core/http/react-ui/e2e/nodes-roster.spec.js | 73 +- core/http/react-ui/inline-style-baseline.txt | 2 +- core/http/react-ui/src/App.css | 272 ++++++- .../react-ui/src/components/ActionMenu.jsx | 17 +- .../react-ui/src/components/ConfirmDialog.jsx | 13 +- core/http/react-ui/src/components/Popover.jsx | 24 +- .../src/components/console/ConsoleLayout.jsx | 30 +- .../src/components/nodes/ClusterOverview.jsx | 93 +++ .../src/components/nodes/ModelFleetTable.jsx | 63 ++ .../src/components/nodes/ModelInspector.jsx | 84 ++ .../src/components/nodes/NodeFleetTable.jsx | 92 +++ .../src/components/nodes/NodeInspector.jsx | 107 +++ .../src/components/nodes/nodeStatus.js | 19 + .../components/nodes/useInspectorDrawer.js | 90 +++ core/http/react-ui/src/pages/NodeDetail.jsx | 48 +- core/http/react-ui/src/pages/Nodes.jsx | 624 ++++++++------- core/http/react-ui/src/utils/nodeFleet.js | 362 +++++++++ .../http/react-ui/src/utils/nodeFleet.test.js | 355 +++++++++ .../nodes/heartbeat_checkpoint_test.go | 25 + core/services/nodes/registry.go | 110 ++- core/services/nodes/registry_test.go | 85 ++ core/services/nodes/unloader.go | 5 + core/services/nodes/unloader_test.go | 17 + core/services/worker/registration.go | 27 + core/services/worker/registration_test.go | 41 + docs/content/features/distributed-mode.md | 44 +- docs/content/operations/backend-monitor.md | 2 +- pkg/model/loader.go | 4 +- pkg/model/process.go | 25 +- pkg/model/remote_shutdown_test.go | 45 ++ pkg/xsysinfo/cpu.go | 40 + pkg/xsysinfo/cpu_test.go | 20 + 39 files changed, 3384 insertions(+), 439 deletions(-) create mode 100644 core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js create mode 100644 core/http/react-ui/src/components/nodes/ClusterOverview.jsx create mode 100644 core/http/react-ui/src/components/nodes/ModelFleetTable.jsx create mode 100644 core/http/react-ui/src/components/nodes/ModelInspector.jsx create mode 100644 core/http/react-ui/src/components/nodes/NodeFleetTable.jsx create mode 100644 core/http/react-ui/src/components/nodes/NodeInspector.jsx create mode 100644 core/http/react-ui/src/components/nodes/useInspectorDrawer.js create mode 100644 core/http/react-ui/src/utils/nodeFleet.js create mode 100644 core/http/react-ui/src/utils/nodeFleet.test.js create mode 100644 pkg/xsysinfo/cpu_test.go diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index 220682b92..331c7203b 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -75,15 +75,18 @@ func GetNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { // RegisterNodeRequest is the request body for registering a new worker node. type RegisterNodeRequest struct { - Name string `json:"name"` - NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent" - Address string `json:"address"` - HTTPAddress string `json:"http_address,omitempty"` - Token string `json:"token,omitempty"` - TotalVRAM uint64 `json:"total_vram,omitempty"` - AvailableVRAM uint64 `json:"available_vram,omitempty"` - TotalRAM uint64 `json:"total_ram,omitempty"` - AvailableRAM uint64 `json:"available_ram,omitempty"` + Name string `json:"name"` + NodeType string `json:"node_type,omitempty"` // "backend" (default) or "agent" + Address string `json:"address"` + HTTPAddress string `json:"http_address,omitempty"` + Token string `json:"token,omitempty"` + TotalVRAM uint64 `json:"total_vram,omitempty"` + AvailableVRAM uint64 `json:"available_vram,omitempty"` + TotalRAM uint64 `json:"total_ram,omitempty"` + AvailableRAM uint64 `json:"available_ram,omitempty"` + CPULogicalCores uint64 `json:"cpu_logical_cores,omitempty"` + CPUUsagePercent float64 `json:"cpu_usage_percent,omitempty"` + CPULoad1 float64 `json:"cpu_load_1,omitempty"` // TotalDisk / AvailableDisk describe the filesystem backing the worker's // MODELS directory (where staged weights land), not the root filesystem. // Omitted by workers that predate the fields; the scheduler treats @@ -182,6 +185,9 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au AvailableVRAM: req.AvailableVRAM, TotalRAM: req.TotalRAM, AvailableRAM: req.AvailableRAM, + CPULogicalCores: req.CPULogicalCores, + CPUUsagePercent: req.CPUUsagePercent, + CPULoad1: req.CPULoad1, TotalDisk: req.TotalDisk, AvailableDisk: req.AvailableDisk, GPUVendor: req.GPUVendor, @@ -381,7 +387,8 @@ func HeartbeatEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { var updatePtr *nodes.HeartbeatUpdate if update.AvailableVRAM != nil || update.TotalVRAM != nil || update.AvailableRAM != nil || - update.AvailableDisk != nil || update.TotalDisk != nil || update.GPUVendor != "" { + update.AvailableDisk != nil || update.TotalDisk != nil || update.GPUVendor != "" || + update.CPUUsagePercent != nil || update.CPULoad1 != nil { updatePtr = &update } @@ -431,6 +438,12 @@ func DrainNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { ctx := c.Request().Context() id := c.Param("id") if err := registry.MarkDraining(ctx, id); err != nil { + if errors.Is(err, nodes.ErrNodeNotFound) { + return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) + } + if errors.Is(err, nodes.ErrNodeStatusConflict) { + return c.JSON(http.StatusConflict, nodeError(http.StatusConflict, "node must be healthy to drain")) + } xlog.Error("Failed to drain node", "id", id, "error", err) return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to drain node")) } @@ -443,7 +456,13 @@ func ResumeNodeEndpoint(registry *nodes.NodeRegistry) echo.HandlerFunc { return func(c echo.Context) error { ctx := c.Request().Context() id := c.Param("id") - if err := registry.MarkHealthy(ctx, id); err != nil { + if err := registry.ResumeNode(ctx, id); err != nil { + if errors.Is(err, nodes.ErrNodeNotFound) { + return c.JSON(http.StatusNotFound, nodeError(http.StatusNotFound, "node not found")) + } + if errors.Is(err, nodes.ErrNodeStatusConflict) { + return c.JSON(http.StatusConflict, nodeError(http.StatusConflict, "node must be draining to resume")) + } xlog.Error("Failed to resume node", "id", id, "error", err) return c.JSON(http.StatusInternalServerError, nodeError(http.StatusInternalServerError, "failed to resume node")) } diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 19e6a6b07..8390f1a48 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -58,6 +58,35 @@ var _ = Describe("Node HTTP handlers", func() { }) Describe("RegisterNodeEndpoint", func() { + It("binds, persists, and lists CPU telemetry", func() { + e := echo.New() + body := `{"name":"cpu-worker","address":"10.0.0.9:50051","cpu_logical_cores":12,"cpu_usage_percent":145,"cpu_load_1":2.5}` + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) + req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) + rec := httptest.NewRecorder() + + Expect(RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{})(e.NewContext(req, rec))).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusCreated)) + + node, err := registry.GetByName(context.Background(), "cpu-worker") + Expect(err).ToNot(HaveOccurred()) + Expect(node.CPULogicalCores).To(Equal(uint64(12))) + Expect(node.CPUUsagePercent).To(Equal(float64(100))) + Expect(node.CPULoad1).To(Equal(2.5)) + + listRecorder := httptest.NewRecorder() + listContext := e.NewContext(httptest.NewRequest(http.MethodGet, "/api/nodes", nil), listRecorder) + Expect(ListNodesEndpoint(registry)(listContext)).To(Succeed()) + var listed []map[string]any + Expect(json.Unmarshal(listRecorder.Body.Bytes(), &listed)).To(Succeed()) + Expect(listed).To(HaveLen(1)) + Expect(listed[0]).To(SatisfyAll( + HaveKeyWithValue("cpu_logical_cores", float64(12)), + HaveKeyWithValue("cpu_usage_percent", float64(100)), + HaveKeyWithValue("cpu_load_1", 2.5), + )) + }) + It("registers a backend node and returns 201", func() { e := echo.New() body := `{"name":"worker-1","address":"10.0.0.1:50051"}` @@ -449,6 +478,42 @@ var _ = Describe("Node HTTP handlers", func() { }) }) + Describe("Node lifecycle endpoints", func() { + request := func(handler echo.HandlerFunc, id string) *httptest.ResponseRecorder { + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/api/nodes/"+id, nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetParamNames("id") + c.SetParamValues(id) + Expect(handler(c)).To(Succeed()) + return rec + } + + It("accepts healthy drain followed by draining resume", func() { + Expect(registry.Register(context.Background(), &nodes.BackendNode{ + ID: "lifecycle", Name: "lifecycle", Address: "10.0.0.10:50051", + }, true)).To(Succeed()) + + Expect(request(DrainNodeEndpoint(registry), "lifecycle").Code).To(Equal(http.StatusOK)) + Expect(request(ResumeNodeEndpoint(registry), "lifecycle").Code).To(Equal(http.StatusOK)) + }) + + It("returns conflict when a pending node is drained or resumed", func() { + Expect(registry.Register(context.Background(), &nodes.BackendNode{ + ID: "pending-lifecycle", Name: "pending-lifecycle", Address: "10.0.0.11:50051", + }, false)).To(Succeed()) + + Expect(request(DrainNodeEndpoint(registry), "pending-lifecycle").Code).To(Equal(http.StatusConflict)) + Expect(request(ResumeNodeEndpoint(registry), "pending-lifecycle").Code).To(Equal(http.StatusConflict)) + }) + + It("returns not found for missing nodes", func() { + Expect(request(DrainNodeEndpoint(registry), "missing").Code).To(Equal(http.StatusNotFound)) + Expect(request(ResumeNodeEndpoint(registry), "missing").Code).To(Equal(http.StatusNotFound)) + }) + }) + Describe("GetNodeModelsEndpoint", func() { It("returns revision and cleanup state without serialized model options", func() { ctx := context.Background() diff --git a/core/http/react-ui/e2e/console-narrow.spec.js b/core/http/react-ui/e2e/console-narrow.spec.js index a7f7bfc73..0e9780806 100644 --- a/core/http/react-ui/e2e/console-narrow.spec.js +++ b/core/http/react-ui/e2e/console-narrow.spec.js @@ -1,6 +1,18 @@ import { test, expect } from './coverage-fixtures.js' test.describe('Operate console on a narrow screen', () => { + test('ignores the desktop collapsed preference', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 800 }) + await page.addInitScript(() => localStorage.setItem('localai_console_rail_collapsed', 'true')) + await page.goto('/app/operate') + + const rail = page.locator('.console-rail') + await expect(rail).toHaveCSS('width', '374px') + await expect(rail.getByText('Operate', { exact: true })).toBeVisible() + await expect(rail.getByRole('button', { name: 'Expand Operate navigation' })).toBeVisible() + await expect(rail.locator('.console-rail-collapse')).toBeHidden() + }) + test('expanding the rail leaves the overview on screen', async ({ page }) => { await page.setViewportSize({ width: 390, height: 800 }) await page.goto('/app/operate') diff --git a/core/http/react-ui/e2e/installed-model-logs-link.spec.js b/core/http/react-ui/e2e/installed-model-logs-link.spec.js index 62fcadf27..e43e16081 100644 --- a/core/http/react-ui/e2e/installed-model-logs-link.spec.js +++ b/core/http/react-ui/e2e/installed-model-logs-link.spec.js @@ -21,4 +21,24 @@ test.describe('Installed model backend logs link', () => { await expect(page).toHaveURL(/\/app\/backend-logs\//) }) + + test('arrow navigation announces the active action through the focused menu', async ({ page }) => { + await page.goto('/app/models?view=installed') + await page.locator('[data-testid="installed-models-rail-item"]').first().click() + const trigger = page.locator('button.action-menu__trigger').first() + await trigger.focus() + await trigger.press('Enter') + + const menu = page.getByRole('menu') + await expect(menu).toBeFocused() + const firstItem = menu.getByRole('menuitem').first() + await expect(menu).toHaveAttribute('aria-activedescendant', await firstItem.getAttribute('id')) + + await menu.press('ArrowDown') + const secondItem = menu.getByRole('menuitem').nth(1) + await expect(menu).toHaveAttribute('aria-activedescendant', await secondItem.getAttribute('id')) + await expect(menu).toBeFocused() + await expect(firstItem).toHaveAttribute('tabindex', '-1') + await expect(secondItem).toHaveAttribute('tabindex', '-1') + }) }) diff --git a/core/http/react-ui/e2e/navigation.spec.js b/core/http/react-ui/e2e/navigation.spec.js index 1a29c07e4..0f35c51aa 100644 --- a/core/http/react-ui/e2e/navigation.spec.js +++ b/core/http/react-ui/e2e/navigation.spec.js @@ -55,4 +55,26 @@ test.describe('Navigation', () => { await expect(rail.locator('a.nav-item[href="/app/fine-tune"]')).toBeVisible() await expect(rail.locator('a.nav-item[href="/app/face"]')).toBeVisible() }) + + test('desktop console rail collapses to accessible icons and persists globally', async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }) + await page.goto('/app/backends') + + const rail = page.locator('.console-rail') + const collapse = rail.getByRole('button', { name: 'Collapse Operate navigation' }) + await expect(collapse).toBeVisible() + await collapse.click() + await expect(rail).toHaveClass(/console-rail--collapsed/) + await expect(rail).toHaveCSS('width', '60px') + await expect(rail.getByRole('link', { name: 'Backends', exact: true })).toHaveClass(/active/) + await expect(rail.getByRole('link', { name: 'Overview', exact: true })).toHaveAttribute('title', 'Overview') + await expect.poll(() => page.evaluate(() => localStorage.getItem('localai_console_rail_collapsed'))).toBe('true') + + await page.goto('/app/agents') + const buildRail = page.locator('.console-rail') + await expect(buildRail).toHaveClass(/console-rail--collapsed/) + await expect(buildRail.getByRole('button', { name: 'Expand Build navigation' })).toBeVisible() + await page.reload() + await expect(page.locator('.console-rail')).toHaveClass(/console-rail--collapsed/) + }) }) diff --git a/core/http/react-ui/e2e/nodes-detail.spec.js b/core/http/react-ui/e2e/nodes-detail.spec.js index 65690ba49..866894a71 100644 --- a/core/http/react-ui/e2e/nodes-detail.spec.js +++ b/core/http/react-ui/e2e/nodes-detail.spec.js @@ -1,9 +1,9 @@ import { test, expect } from './coverage-fixtures.js' const ID = 'n1' -async function mockNode(page) { +async function mockNode(page, overrides = {}) { await page.route(`**/api/nodes/${ID}`, r => r.fulfill({ status: 200, contentType: 'application/json', - body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', total_vram: 24e9, available_vram: 12e9, max_replicas_per_model: 1, labels: { env: 'prod' } }) })) + body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', total_vram: 24e9, available_vram: 12e9, total_disk: 100e9, available_disk: 40e9, cpu_logical_cores: 16, cpu_usage_percent: 25, cpu_load_1: 2.5, max_replicas_per_model: 1, labels: { env: 'prod' }, ...overrides }) })) await page.route(`**/api/nodes/${ID}/models`, r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([{ node_id: ID, model_name: 'llama-3.3', state: 'loaded', in_flight: 0, replica_index: 0 }]) })) await page.route(`**/api/nodes/${ID}/backends`, r => r.fulfill({ status: 200, contentType: 'application/json', @@ -19,16 +19,64 @@ test.describe('Node detail page', () => { await expect(page.getByText('llama-3.3')).toBeVisible() await expect(page.getByText('llama-cpp')).toBeVisible() await expect(page.getByText('env=prod')).toBeVisible() + await expect(page.getByText('25.0% of 16 cores')).toBeVisible() + await expect(page.getByText('2.50 load (1m)')).toBeVisible() + await expect(page.getByText('37.3 GB / 93.1 GB')).toBeVisible() }) test('is reachable by clicking a roster panel', async ({ page }) => { await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([{ id: ID, name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' }]) })) - await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) - await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) await mockNode(page) await page.goto('/app/nodes') - await page.locator('.node-panel').filter({ hasText: 'alpha' }).getByText('alpha').click() + await page.getByRole('button', { name: 'Inspect alpha' }).click() + await page.getByRole('link', { name: 'Open full node details' }).click() await expect(page).toHaveURL(new RegExp(`/app/nodes/${ID}$`)) }) + + for (const [status, action] of [['healthy', 'Drain'], ['draining', 'Resume'], ['unhealthy', null], ['offline', null], ['unknown', null]]) { + test(`shows only the accepted lifecycle action for ${status} nodes`, async ({ page }) => { + await mockNode(page, { status }) + await page.goto(`/app/nodes/${ID}`) + await expect(page.locator('.page-title').first()).toBeVisible({ timeout: 15_000 }) + await expect(page.getByRole('button', { name: /Approve/ })).toHaveCount(0) + await expect(page.getByRole('button', { name: /Drain/ })).toHaveCount(action === 'Drain' ? 1 : 0) + await expect(page.getByRole('button', { name: /Resume/ })).toHaveCount(action === 'Resume' ? 1 : 0) + await expect(page.locator('.page-header__meta .btn-danger')).toContainText('Remove') + }) + } + + test('approves a pending node and refreshes its lifecycle controls', async ({ page }) => { + let status = 'pending' + let approvalRequests = 0 + await page.route(`**/api/nodes/${ID}`, r => r.fulfill({ status: 200, contentType: 'application/json', + body: JSON.stringify({ id: ID, name: 'alpha', node_type: 'backend', status, labels: {} }) })) + await page.route(`**/api/nodes/${ID}/models`, r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.route(`**/api/nodes/${ID}/backends`, r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.route(`**/api/nodes/${ID}/approve`, async r => { + approvalRequests += 1 + status = 'healthy' + await r.fulfill({ status: 200, contentType: 'application/json', body: '{}' }) + }) + await page.goto(`/app/nodes/${ID}`) + + await page.getByRole('button', { name: 'Approve' }).click() + await expect.poll(() => approvalRequests).toBe(1) + await expect(page.getByText('Node approved')).toBeVisible() + await expect(page.getByRole('button', { name: 'Drain' })).toBeVisible() + await expect(page.locator('.page-header__meta .btn-danger')).toContainText('Remove') + }) + + test('renders valid totals with missing available capacity as No data', async ({ page }) => { + await mockNode(page, { + total_vram: 24e9, available_vram: undefined, + total_ram: 32e9, available_ram: undefined, + total_disk: 100e9, available_disk: undefined, + }) + await page.goto(`/app/nodes/${ID}`) + await expect(page.locator('.node-detail__metrics')).toContainText('VRAM') + await expect(page.locator('.node-detail__metrics')).toContainText('RAM') + await expect(page.locator('.node-detail__metrics')).toContainText('Models disk free') + await expect(page.locator('.node-detail__metrics').getByText('No data')).toHaveCount(3) + }) }) diff --git a/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js b/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js new file mode 100644 index 000000000..adcb5a822 --- /dev/null +++ b/core/http/react-ui/e2e/nodes-fleet-dashboard.spec.js @@ -0,0 +1,747 @@ +import { test, expect } from './coverage-fixtures.js' + +const baseNodes = [ + { id: 'n1', name: 'atlas', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', labels: { zone: 'east' }, total_vram: 100, available_vram: 40, total_ram: 200, available_ram: 100, total_disk: 1000, available_disk: 600, cpu_logical_cores: 8, cpu_usage_percent: 25, cpu_load_1: 1.5, model_count: 3, in_flight_count: 2, last_heartbeat: '2026-09-14T00:00:00Z' }, + { id: 'n2', name: 'borealis', node_type: 'backend', address: '10.0.0.2:50051', status: 'pending', labels: { zone: 'west' }, total_vram: 100, available_vram: 10, total_ram: 200, available_ram: 10, total_disk: 1000, available_disk: 100, cpu_logical_cores: 16, cpu_usage_percent: 50, cpu_load_1: 4, model_count: 1, in_flight_count: 0 }, + { id: 'n3', name: 'legacy', node_type: 'agent', address: '10.0.0.3:50051', status: 'offline', labels: {} }, +] + +async function mockNodes(page, nodes = baseNodes) { + await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) })) +} + +async function mockFullOperateNavigation(page) { + await page.route('**/api/features', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ distributed: true }), + })) + await page.route('**/api/auth/status', route => route.fulfill({ + status: 200, + contentType: 'application/json', + body: JSON.stringify({ + authEnabled: true, + staticApiKeyRequired: false, + providers: ['local'], + user: { id: 'admin', name: 'Admin', role: 'admin', provider: 'local' }, + }), + })) +} + +const baseModels = [ + { id: 'r1', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.1:50101', state: 'loaded', in_flight: 2, backend_type: 'llama-cpp', last_used: '2026-09-14T10:00:00Z' }, + { id: 'r2', node_id: 'n1', model_name: 'Llama 3.2', replica_index: 1, address: '10.0.0.1:50102', state: 'loaded', in_flight: 0, backend_type: 'llama-cpp', last_used: '2026-09-14T10:30:00Z' }, + { id: 'r3', node_id: 'n2', model_name: 'Llama 3.2', replica_index: 0, address: '10.0.0.2:50101', state: 'loaded', in_flight: 1, backend_type: 'vllm', last_used: '2026-09-14T11:00:00Z' }, + { id: 'r4', node_id: 'missing', model_name: 'Whisper large v3', replica_index: 0, address: '10.0.0.9:50101', state: 'loaded', in_flight: 0, backend_type: 'whisper', last_used: '2026-09-14T09:00:00Z' }, +] + +test.describe('Nodes fleet dashboard', () => { + test('uses the standard Operate navigation at a desktop viewport', async ({ page }) => { + await mockFullOperateNavigation(page) + await mockNodes(page, [baseNodes[0]]) + await page.goto('/app/nodes') + + const primaryOperate = page.locator('.sidebar-nav a.nav-item', { hasText: 'Operate' }) + await expect(primaryOperate).toBeVisible({ timeout: 15_000 }) + await expect(primaryOperate).toHaveClass(/active/) + + const rail = page.locator('.console-layout > .console-rail') + await expect(rail).toBeVisible() + await expect(rail.locator('a.nav-item')).toHaveCount(13) + await expect(rail.locator('a[href="/app/nodes"]')).toHaveClass(/active/) + await expect(rail.locator('a[href$="/swagger/index.html"]')).toHaveAttribute('target', '_blank') + }) + + test('uses the standard collapsible Operate rail on mobile', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }) + await mockFullOperateNavigation(page) + await mockNodes(page, [baseNodes[0]]) + await page.goto('/app/nodes') + + await page.getByRole('button', { name: 'Open menu' }).click() + await expect(page.locator('.sidebar-nav a.nav-item', { hasText: 'Operate' })).toBeVisible() + await page.getByRole('button', { name: 'Close menu' }).click() + + const rail = page.locator('.console-layout > .console-rail') + await expect(rail).toBeVisible() + await expect(rail.locator('.console-rail-groups')).toBeHidden() + await rail.getByRole('button', { name: 'Expand Operate navigation' }).click() + await expect(rail.locator('.console-rail-groups')).toBeVisible() + await expect(rail.locator('a.nav-item')).toHaveCount(13) + }) + + test('shows aggregate health, capacity, attention filtering, search, sorting, and grouping', async ({ page }) => { + await mockNodes(page) + await page.goto('/app/nodes') + await expect(page.getByLabel('Fleet health summary')).toContainText('3 nodes', { timeout: 15_000 }) + await expect(page.getByLabel('VRAM capacity')).toContainText('150 B / 200 B') + await expect(page.getByLabel('CPU capacity')).toContainText('10 busy / 24 cores') + await expect(page.getByLabel('Models disk capacity')).toContainText('1.3 KB / 2 KB') + await expect(page.getByRole('button', { name: /Needs attention.*2/ })).toBeVisible() + await page.getByRole('button', { name: /Low VRAM/ }).click() + await expect(page.getByRole('row', { name: /borealis/ })).toBeVisible() + await expect(page.getByRole('row', { name: /atlas/ })).toHaveCount(0) + await page.getByRole('button', { name: /Low VRAM/ }).click() + await page.getByRole('searchbox', { name: 'Search nodes' }).fill('legacy') + await expect(page.getByRole('row', { name: /legacy/ })).toBeVisible() + await page.getByRole('searchbox', { name: 'Search nodes' }).fill('') + await page.getByRole('button', { name: /Sort by node/ }).click() + await expect(page.locator('tbody tr').first()).toContainText('legacy') + await page.getByLabel('Group nodes').selectOption('label:zone') + await expect(page.getByText('Unlabelled', { exact: true })).toBeVisible() + await expect(page.getByText('east', { exact: true })).toBeVisible() + }) + + test('mounts only 50 rows and clamps pagination for a 1,000-node fleet', async ({ page }) => { + const nodes = Array.from({ length: 1000 }, (_, index) => ({ id: `node-${index}`, name: `worker-${String(index).padStart(4, '0')}`, node_type: 'backend', address: `10.0.${Math.floor(index / 255)}.${index % 255}:50051`, status: 'healthy' })) + await mockNodes(page, nodes) + await page.goto('/app/nodes') + await expect(page.locator('tbody tr')).toHaveCount(50, { timeout: 15_000 }) + await expect(page.getByText('Page 1 of 20')).toBeVisible() + await page.getByRole('button', { name: 'Next page' }).click() + await expect(page.getByText('Page 2 of 20')).toBeVisible() + await page.getByRole('searchbox', { name: 'Search nodes' }).fill('worker-0000') + await expect(page.getByText('Page 1 of 1')).toBeVisible() + }) + + test('filters bulk actions by lifecycle state, reports skipped nodes, and prevents overlapping batches', async ({ page }) => { + const nodes = Array.from({ length: 12 }, (_, index) => ({ + id: `n${index}`, + name: `worker-${index}`, + node_type: 'backend', + status: index === 9 ? 'pending' : index === 10 ? 'draining' : index === 11 ? 'offline' : 'healthy', + })) + await mockNodes(page, nodes) + let active = 0 + let peak = 0 + const drainRequests = [] + const resumeRequests = [] + await page.route('**/api/nodes/*/drain', async route => { + active += 1 + peak = Math.max(peak, active) + await new Promise(resolve => setTimeout(resolve, 30)) + active -= 1 + const id = route.request().url().split('/').at(-2) + drainRequests.push(id) + await route.fulfill({ status: id === 'n8' ? 500 : 200, contentType: 'application/json', body: id === 'n8' ? '{"error":"failed"}' : '{}' }) + }) + await page.route('**/api/nodes/*/resume', async route => { + const id = route.request().url().split('/').at(-2) + resumeRequests.push(id) + await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' }) + }) + await page.goto('/app/nodes') + await page.getByRole('checkbox', { name: 'Select visible nodes' }).check() + await page.getByRole('searchbox', { name: 'Search nodes' }).fill('worker-1') + await expect(page.getByText('12 selected')).toBeVisible() + await page.getByRole('button', { name: 'Drain selected' }).evaluate(button => { + button.click() + button.click() + }) + await expect.poll(() => drainRequests.length).toBe(9) + expect(peak).toBeLessThanOrEqual(8) + expect(drainRequests.sort()).toEqual(Array.from({ length: 9 }, (_, index) => `n${index}`).sort()) + await expect(page.getByText(/8 succeeded, 1 failed, 3 skipped/)).toBeVisible() + + await page.getByRole('button', { name: 'Resume selected' }).click() + await expect.poll(() => resumeRequests).toEqual(['n10']) + await expect(page.getByText(/1 succeeded, 0 failed, 11 skipped/)).toBeVisible() + expect(drainRequests).not.toContain('n9') + expect(resumeRequests).not.toContain('n9') + }) + + test('disables bulk controls and remove confirmation while removal is running', async ({ page }) => { + await mockNodes(page, [baseNodes[0]]) + let finishRemove + await page.route('**/api/nodes/n1', async route => { + if (route.request().method() !== 'DELETE') return route.fallback() + await new Promise(resolve => { finishRemove = resolve }) + await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' }) + }) + await page.goto('/app/nodes') + await page.getByRole('checkbox', { name: 'Select atlas' }).check() + await page.getByRole('button', { name: 'Remove selected' }).click() + await page.getByRole('button', { name: 'Remove nodes' }).click() + + await expect(page.getByRole('button', { name: 'Removing…' })).toBeDisabled() + await expect(page.getByRole('button', { name: 'Cancel' })).toBeDisabled() + finishRemove() + await expect(page.getByText(/1 succeeded, 0 failed, 0 skipped/)).toBeVisible() + }) + + test('fetches backends only when an inspector opens and shows unknown legacy metrics', async ({ page }) => { + await mockNodes(page) + let backendRequests = 0 + await page.route('**/api/nodes/n3/backends', route => { + backendRequests += 1 + return route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"tool-runner"}]' }) + }) + await page.goto('/app/nodes') + expect(backendRequests).toBe(0) + await page.getByRole('button', { name: 'Inspect legacy' }).click() + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector).toContainText('legacy') + await expect(inspector).toContainText('No data') + await expect(inspector).toContainText('1 backend') + expect(backendRequests).toBe(1) + await expect(page.getByRole('link', { name: 'Open full node details' })).toHaveAttribute('href', '/app/nodes/n3') + }) + + test('keeps pending approval visible', async ({ page }) => { + await mockNodes(page, [baseNodes[1]]) + await page.route('**/api/nodes/n2/approve', route => route.fulfill({ status: 200, contentType: 'application/json', body: '{}' })) + await page.goto('/app/nodes') + await expect(page.getByRole('button', { name: 'Approve borealis' })).toBeVisible({ timeout: 15_000 }) + }) + + test('shows inspector lifecycle controls only for server-accepted states', async ({ page }) => { + const statuses = ['healthy', 'draining', 'pending', 'unhealthy', 'offline', 'unknown'] + await mockNodes(page, statuses.map((status, index) => ({ + id: `state-${index}`, + name: `node-${status}`, + node_type: 'backend', + status, + }))) + await page.route('**/api/nodes/*/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + + for (const status of statuses) { + await page.getByRole('button', { name: `Inspect node-${status}` }).click() + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector.getByRole('button', { name: 'Approve', exact: true })).toHaveCount(status === 'pending' ? 1 : 0) + await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toHaveCount(status === 'healthy' ? 1 : 0) + await expect(inspector.getByRole('button', { name: 'Resume', exact: true })).toHaveCount(status === 'draining' ? 1 : 0) + await inspector.getByRole('button', { name: 'Close node inspector' }).click() + } + }) + + test('approves a pending node from the model-to-node drilldown', async ({ page }) => { + let status = 'pending' + let approvalRequests = 0 + await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([ + { id: 'pending-node', name: 'pending-worker', node_type: 'backend', status }, + ]) })) + await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([ + { id: 'replica', node_id: 'pending-node', model_name: 'Pending model', replica_index: 0, state: 'loaded' }, + ]) })) + await page.route('**/api/nodes/pending-node/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.route('**/api/nodes/pending-node/approve', async route => { + approvalRequests += 1 + status = 'healthy' + await route.fulfill({ status: 200, contentType: 'application/json', body: '{}' }) + }) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: 'Running models' }).click() + await page.getByRole('button', { name: 'Inspect Pending model' }).click() + await page.getByRole('button', { name: 'Open node pending-worker' }).click() + + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await inspector.getByRole('button', { name: 'Approve', exact: true }).click() + await expect.poll(() => approvalRequests).toBe(1) + await expect(page.getByText('Node approved')).toBeVisible() + await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeVisible() + }) + + test('keeps incomplete capacity unknown throughout the fleet view', async ({ page }) => { + await mockNodes(page, [{ + id: 'incomplete', name: 'incomplete-capacity', node_type: 'backend', status: 'healthy', + total_vram: 100, total_ram: 200, total_disk: 300, + }]) + await page.route('**/api/nodes/incomplete/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + + await expect(page.getByLabel('VRAM capacity')).toContainText('No data') + await expect(page.getByLabel('VRAM capacity')).toContainText('1 node unavailable') + await expect(page.getByRole('button', { name: /Low VRAM.*0/ })).toBeVisible() + const row = page.getByRole('row', { name: /incomplete-capacity/ }) + await expect(row.getByText('No data')).toHaveCount(3) + await page.getByRole('button', { name: 'Inspect incomplete-capacity' }).click() + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector.getByText('No data')).toHaveCount(4) + }) + + test('announces complete and partial capacity coverage without adding visible clutter', async ({ page }) => { + const partialNode = { + ...baseNodes[0], + id: 'n-partial', + name: 'partial-capacity', + total_vram: 100, + available_vram: 50, + total_ram: undefined, + available_ram: undefined, + } + await mockNodes(page, [baseNodes[0], partialNode]) + await page.goto('/app/nodes') + + const vram = page.getByLabel('VRAM capacity', { exact: true }) + const ram = page.getByLabel('RAM capacity', { exact: true }) + await expect(vram).toContainText('Capacity coverage: 2 of 2 nodes reporting; 0 unknown.', { timeout: 15_000 }) + await expect(ram).toContainText('Capacity coverage: 1 of 2 nodes reporting; 1 unknown.') + await expect(vram.locator('.fleet-gauge__coverage')).toHaveCount(0) + await expect(ram.locator('.fleet-gauge__coverage')).toHaveText('1 node unavailable') + }) + + test('keeps checkbox keyboard activation from opening the inspector', async ({ page }) => { + await mockNodes(page, [baseNodes[0]]) + await page.goto('/app/nodes') + + const checkbox = page.getByRole('checkbox', { name: 'Select atlas' }) + await checkbox.focus() + await checkbox.press('Space') + + await expect(checkbox).toBeChecked() + await expect(page.getByRole('complementary', { name: 'Node inspector' })).toHaveCount(0) + }) + + test('keeps the low-density composition while inspecting at a desktop viewport', async ({ page }) => { + await page.setViewportSize({ width: 1600, height: 1050 }) + await mockNodes(page) + await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"},{"name":"whisper"}]' })) + await page.goto('/app/nodes') + + const overview = page.getByRole('region', { name: 'Fleet overview' }) + const workbench = page.getByRole('region', { name: 'Fleet workbench' }) + await expect(overview).toBeVisible({ timeout: 15_000 }) + await expect(workbench).toBeVisible() + await expect(page.locator('.console-layout > .console-rail')).toBeVisible() + await expect(page.locator('.fleet-select-wrap')).toHaveCount(3) + await expect(page.getByLabel('Filter status')).toHaveCSS('appearance', 'none') + await expect(page.locator('.fleet-bulkbar')).toHaveCount(0) + + const overviewBefore = await overview.boundingBox() + const fleetBefore = await page.locator('#fleet-nodes-panel').boundingBox() + const cells = overview.locator('.fleet-overview__cell') + await expect(cells).toHaveCount(5) + const cellTops = await cells.evaluateAll(items => items.map(cell => Math.round(cell.getBoundingClientRect().top))) + expect(new Set(cellTops).size).toBe(1) + const attention = page.getByRole('complementary', { name: 'Attention queue' }) + await expect(attention).toBeVisible() + const overviewBox = await overview.boundingBox() + const attentionBox = await attention.boundingBox() + expect(attentionBox.y).toBeGreaterThanOrEqual(overviewBox.y + overviewBox.height) + + const checkbox = page.getByRole('checkbox', { name: 'Select atlas' }) + await checkbox.check() + await expect(page.getByRole('row', { name: /atlas/ })).toHaveClass(/is-selected/) + await expect(page.locator('.fleet-bulkbar')).toBeVisible() + await expect(page.getByRole('button', { name: 'Clear selection' })).toBeVisible() + + const inspectNode = page.getByRole('button', { name: 'Inspect atlas' }) + await inspectNode.focus() + await inspectNode.press('Enter') + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector).toBeVisible() + await expect(inspector.getByRole('heading', { name: 'Node' })).toBeVisible() + await expect(inspector.getByRole('heading', { name: 'Resources' })).toBeVisible() + await expect(inspector.getByRole('heading', { name: 'Workload' })).toBeVisible() + await expect(inspector.locator('.node-inspector__resource')).toHaveCount(2) + + const overviewAfter = await overview.boundingBox() + const fleetAfter = await page.locator('#fleet-nodes-panel').boundingBox() + expect(Math.abs(overviewAfter.width - overviewBefore.width)).toBeLessThanOrEqual(1) + expect(Math.abs(fleetBefore.width - fleetAfter.width)).toBeLessThanOrEqual(1) + await expect(inspector).toHaveCSS('position', 'fixed') + await expect.poll(async () => { + const inspectorBox = await inspector.boundingBox() + return Math.max( + Math.abs(inspectorBox.x + inspectorBox.width - 1584), + Math.abs(inspectorBox.y - 16), + Math.abs(inspectorBox.height - 1018), + ) + }).toBeLessThanOrEqual(1) + }) + + test('reflows the overview and presents a contained drawer at a narrow viewport', async ({ page }) => { + await page.setViewportSize({ width: 640, height: 900 }) + await mockNodes(page, [baseNodes[0]]) + await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + + const overview = page.getByRole('region', { name: 'Fleet overview' }) + await expect(overview).toBeVisible({ timeout: 15_000 }) + await expect(page.locator('.console-layout > .console-rail')).toBeVisible() + const overviewBox = await overview.boundingBox() + expect(overviewBox.width).toBeGreaterThan(500) + const cells = overview.locator('.fleet-overview__cell') + const tops = await cells.evaluateAll(items => items.map(item => Math.round(item.getBoundingClientRect().top))) + expect(new Set(tops).size).toBeGreaterThan(1) + await expect(overview.getByLabel('Fleet health summary')).toHaveCSS('grid-column-start', '1') + await expect(overview.getByLabel('Fleet health summary')).toHaveCSS('grid-column-end', '-1') + + const narrowInspectNode = page.getByRole('button', { name: 'Inspect atlas' }) + await narrowInspectNode.focus() + await narrowInspectNode.press('Enter') + const inspector = page.getByRole('dialog', { name: 'Node inspector' }) + await expect(inspector).toBeVisible() + await expect(inspector).toHaveAttribute('aria-modal', 'true') + await expect(inspector).toHaveCSS('position', 'fixed') + await expect(page.locator('.node-inspector__scrim')).toBeVisible() + await expect(page.locator('body')).toHaveCSS('overflow', 'hidden') + await expect(page.getByRole('region', { name: 'Fleet workbench', includeHidden: true })).toHaveAttribute('inert', '') + await expect(page.getByRole('region', { name: 'Fleet workbench', includeHidden: true })).toHaveAttribute('aria-hidden', 'true') + const workbenchBox = await page.locator('.fleet-workbench').boundingBox() + expect(workbenchBox.width).toBeLessThanOrEqual(600) + await expect.poll(async () => (await inspector.boundingBox()).y).toBeLessThanOrEqual(1) + const inspectorBox = await inspector.boundingBox() + expect(inspectorBox.height).toBe(900) + const close = inspector.getByRole('button', { name: 'Close node inspector' }) + await expect(close).toBeFocused() + await close.press('Shift+Tab') + await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeFocused() + await page.keyboard.press('Tab') + await expect(close).toBeFocused() + await page.keyboard.press('Escape') + await expect(inspector).toHaveCount(0) + await expect(page.locator('body')).not.toHaveCSS('overflow', 'hidden') + await expect(page.getByRole('region', { name: 'Fleet workbench' })).not.toHaveAttribute('inert', '') + await expect(page.getByRole('region', { name: 'Fleet workbench' })).not.toHaveAttribute('aria-hidden', 'true') + await expect(narrowInspectNode).toBeFocused() + }) + + test('loads running models once on activation and drills model to node and back', async ({ page }) => { + await page.setViewportSize({ width: 1600, height: 1050 }) + await mockNodes(page, baseNodes.map(node => node.id === 'n2' ? { ...node, status: 'healthy' } : node)) + let modelRequests = 0 + let backendRequests = 0 + await page.route('**/api/nodes/models', route => { + modelRequests += 1 + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) }) + }) + await page.route('**/api/nodes/n1/backends', route => { + backendRequests += 1 + return route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"}]' }) + }) + await page.goto('/app/nodes') + await expect(page.getByRole('table', { name: 'Fleet nodes' })).toBeVisible({ timeout: 15_000 }) + expect(modelRequests).toBe(0) + expect(backendRequests).toBe(0) + + const nodesTab = page.getByRole('tab', { name: 'Nodes' }) + const modelsTab = page.getByRole('tab', { name: 'Running models' }) + const nodesPanel = page.locator('#fleet-nodes-panel') + const modelsPanel = page.locator('#fleet-models-panel') + await expect(nodesTab).toHaveAttribute('aria-controls', 'fleet-nodes-panel') + await expect(nodesTab).toHaveAttribute('tabindex', '0') + await expect(modelsTab).toHaveAttribute('aria-controls', 'fleet-models-panel') + await expect(modelsTab).toHaveAttribute('tabindex', '-1') + await expect(nodesPanel).toHaveAttribute('role', 'tabpanel') + await expect(nodesPanel).toHaveAttribute('aria-labelledby', 'fleet-nodes-tab') + await expect(nodesPanel).not.toHaveAttribute('hidden', '') + await expect(nodesPanel).toBeVisible() + await expect(modelsPanel).toHaveAttribute('role', 'tabpanel') + await expect(modelsPanel).toHaveAttribute('aria-labelledby', 'fleet-models-tab') + await expect(modelsPanel).toHaveAttribute('hidden', '') + await expect(modelsPanel).toBeHidden() + await expect(modelsPanel.getByRole('table', { name: 'Running models' })).toHaveCount(0) + await expect(nodesPanel.locator('tbody tr')).toHaveCount(3) + + await nodesTab.focus() + await nodesTab.press('ArrowRight') + await expect(modelsTab).toBeFocused() + await expect(modelsTab).toHaveAttribute('aria-selected', 'true') + await expect(modelsTab).toHaveAttribute('tabindex', '0') + await expect(nodesTab).toHaveAttribute('tabindex', '-1') + await expect(nodesPanel).toHaveAttribute('hidden', '') + await expect(nodesPanel).toBeHidden() + await expect(modelsPanel).not.toHaveAttribute('hidden', '') + await expect(modelsPanel).toBeVisible() + await expect(page.getByText('Current loaded replicas on healthy nodes')).toBeVisible() + await expect(page.getByRole('table', { name: 'Running models' })).toBeVisible() + await expect(page.getByRole('row', { name: /Llama 3.2/ })).toContainText('3') + expect(modelRequests).toBe(1) + expect(backendRequests).toBe(0) + + const modelControl = page.getByRole('button', { name: 'Inspect Llama 3.2' }) + await expect(modelControl).not.toHaveAttribute('aria-selected') + await expect(modelControl).toHaveAttribute('aria-pressed', 'false') + await expect(modelControl).toHaveAttribute('aria-expanded', 'false') + await expect(modelControl).not.toHaveAttribute('aria-controls') + await modelControl.focus() + await modelControl.press('Enter') + const modelInspector = page.getByRole('complementary', { name: 'Model inspector' }) + const closeModel = page.getByRole('button', { name: 'Close model inspector' }) + await expect(closeModel).toBeFocused() + await expect(modelControl).not.toHaveAttribute('aria-selected') + await expect(modelControl).toHaveAttribute('aria-pressed', 'true') + await expect(modelControl).toHaveAttribute('aria-expanded', 'true') + await expect(modelControl).toHaveAttribute('aria-current', 'true') + await expect(modelControl).toHaveAttribute('aria-controls', 'model-inspector') + await expect(modelInspector).toContainText('2 replicas') + await expect(modelInspector).toContainText('borealis') + const atlasControl = modelInspector.getByRole('button', { name: /Open node atlas/ }) + await atlasControl.focus() + await atlasControl.press('Enter') + await expect(page.getByRole('complementary', { name: 'Node inspector' })).toBeVisible() + await expect(modelControl).toHaveAttribute('aria-pressed', 'true') + await expect(modelControl).toHaveAttribute('aria-expanded', 'false') + await expect(modelControl).not.toHaveAttribute('aria-controls') + const backToModel = page.getByRole('button', { name: 'Back to Llama 3.2' }) + await expect(backToModel).toBeFocused() + await expect.poll(() => backendRequests).toBe(1) + expect(modelRequests).toBe(1) + await backToModel.press('Enter') + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toBeVisible() + await expect(page.getByRole('button', { name: /Open node atlas/ })).toBeFocused() + await closeModel.click() + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0) + await expect(modelControl).toBeFocused() + await expect(modelControl).not.toHaveAttribute('aria-selected') + await expect(modelControl).toHaveAttribute('aria-pressed', 'false') + await expect(modelControl).toHaveAttribute('aria-expanded', 'false') + await expect(modelControl).not.toHaveAttribute('aria-controls') + + await page.getByRole('button', { name: 'Inspect Whisper large v3' }).click() + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('missing') + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toContainText('Unknown') + await page.getByRole('button', { name: 'Close model inspector' }).click() + + await modelsTab.focus() + await modelsTab.press('ArrowLeft') + await expect(nodesTab).toBeFocused() + await expect(nodesTab).toHaveAttribute('aria-selected', 'true') + await expect(nodesPanel).not.toHaveAttribute('hidden', '') + await expect(modelsPanel).toHaveAttribute('hidden', '') + await expect(modelsPanel.locator('tbody tr')).toHaveCount(2) + await nodesTab.press('ArrowRight') + await expect(modelsPanel.locator('tbody tr')).toHaveCount(2) + expect(modelRequests).toBe(1) + }) + + test('treats the model inspector as a modal drawer on mobile', async ({ page }) => { + await page.setViewportSize({ width: 390, height: 844 }) + await mockNodes(page) + await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) })) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: 'Running models' }).click() + await page.getByRole('button', { name: 'Inspect Llama 3.2' }).click() + + const inspector = page.getByRole('dialog', { name: 'Model inspector' }) + await expect(inspector).toHaveAttribute('aria-modal', 'true') + await expect(inspector.getByRole('button', { name: 'Close model inspector' })).toBeFocused() + await expect(page.locator('.fleet-workbench')).toHaveAttribute('inert', '') + await inspector.getByRole('button', { name: 'Close model inspector' }).press('Shift+Tab') + await expect(inspector.getByRole('button', { name: 'Close', exact: true })).toBeFocused() + await page.keyboard.press('Tab') + await expect(inspector.getByRole('button', { name: 'Close model inspector' })).toBeFocused() + }) + + test('keeps the node inspector open when Escape dismisses its confirmation dialog', async ({ page }) => { + await mockNodes(page) + await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + await page.getByRole('checkbox', { name: 'Select atlas' }).check() + await page.getByRole('button', { name: 'Inspect atlas' }).click() + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector).toBeVisible() + + await page.getByRole('button', { name: 'Remove selected' }).click() + await expect(page.getByRole('alertdialog')).toBeVisible() + await page.keyboard.press('Escape') + + await expect(page.getByRole('alertdialog')).toHaveCount(0) + await expect(inspector).toBeVisible() + }) + + test('stops a running model once from an accessible row menu and refreshes inventory', async ({ page }) => { + await mockNodes(page) + let modelRequests = 0 + let stopRequests = 0 + let stopBody + let finishStop + await page.route('**/api/nodes/models', route => { + modelRequests += 1 + const rows = modelRequests === 1 ? baseModels : baseModels.filter(row => row.model_name !== 'Llama 3.2') + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(rows) }) + }) + await page.route('**/backend/shutdown', async route => { + stopRequests += 1 + stopBody = route.request().postDataJSON() + await new Promise(resolve => { finishStop = resolve }) + await route.fulfill({ status: 200, contentType: 'application/json', body: '{"message":"ok"}' }) + }) + await page.goto('/app/nodes') + const modelsTab = page.getByRole('tab', { name: 'Running models' }) + await modelsTab.click() + + const trigger = page.getByRole('button', { name: 'Actions for Llama 3.2' }) + await expect(trigger).toBeVisible() + await page.evaluate(() => new Promise(resolve => requestAnimationFrame(() => requestAnimationFrame(resolve)))) + await expect(modelsTab).toBeFocused() + + await trigger.focus() + await trigger.press('Enter') + const menu = page.getByRole('menu', { name: 'Llama 3.2 actions' }) + await expect(menu).toBeVisible() + await expect(menu).toBeFocused() + await menu.press('Escape') + await expect(menu).toHaveCount(0) + await expect(trigger).toBeFocused() + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0) + + await trigger.click() + await page.locator('.model-workbench__scope').click() + await expect(menu).toHaveCount(0) + await expect(page.getByRole('complementary', { name: 'Model inspector' })).toHaveCount(0) + + await trigger.click() + await menu.getByRole('menuitem', { name: 'Stop model…' }).click() + const dialog = page.getByRole('alertdialog') + await expect(dialog).toContainText('Stop Llama 3.2?') + await expect(dialog).toContainText('Llama 3.2 has 3 loaded replicas across 2 unique nodes. This will stop all loaded placements on those nodes.') + await expect(dialog.getByRole('button', { name: 'Stop model' })).toBeFocused() + await page.keyboard.press('Tab') + await expect(dialog.getByRole('button', { name: 'Cancel' })).toBeFocused() + await page.keyboard.press('Shift+Tab') + await expect(dialog.getByRole('button', { name: 'Stop model' })).toBeFocused() + await dialog.getByRole('button', { name: 'Cancel' }).click() + await expect(dialog).toHaveCount(0) + await expect(trigger).toBeFocused() + + await trigger.click() + await menu.getByRole('menuitem', { name: 'Stop model…' }).click() + await dialog.getByRole('button', { name: 'Stop model' }).evaluate(button => { + button.click() + button.click() + }) + + await expect(dialog.getByRole('button', { name: 'Stopping…' })).toBeDisabled() + await expect(dialog.getByRole('button', { name: 'Cancel' })).toBeDisabled() + await expect.poll(() => stopRequests).toBe(1) + expect(stopBody).toEqual({ model: 'Llama 3.2' }) + finishStop() + await expect.poll(() => modelRequests).toBe(2) + await expect(page.getByText('Stopped Llama 3.2: 3 replicas across 2 nodes.')).toBeVisible() + await expect(page.getByRole('button', { name: 'Inspect Llama 3.2' })).toHaveCount(0) + }) + + test('refreshes model inventory and warns about partial shutdown after a stop failure', async ({ page }) => { + await mockNodes(page) + let modelRequests = 0 + let stopRequests = 0 + await page.route('**/api/nodes/models', route => { + modelRequests += 1 + return route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(baseModels) }) + }) + await page.route('**/backend/shutdown', route => { + stopRequests += 1 + return route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"controller timed out"}' }) + }) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: 'Running models' }).click() + await page.getByRole('button', { name: 'Actions for Whisper large v3' }).click() + await page.getByRole('menuitem', { name: 'Stop model…' }).click() + await page.getByRole('alertdialog').getByRole('button', { name: 'Stop model' }).click() + + await expect.poll(() => stopRequests).toBe(1) + await expect.poll(() => modelRequests).toBe(2) + await expect(page.getByText(/Could not stop Whisper large v3:.*Some replicas may already have stopped\./)).toBeVisible() + await expect(page.getByRole('button', { name: 'Inspect Whisper large v3' })).toBeVisible() + }) + + test('moves focus into and restores it from the node inspector', async ({ page }) => { + await mockNodes(page, [baseNodes[0]]) + await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + + const nodeControl = page.getByRole('button', { name: 'Inspect atlas' }) + await nodeControl.click() + const closeNode = page.getByRole('button', { name: 'Close node inspector' }) + await expect(closeNode).toBeFocused() + await closeNode.click() + await expect(nodeControl).toBeFocused() + }) + + test('keeps drawer actions visible for a one-row filtered fleet', async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 900 }) + await mockNodes(page, baseNodes) + await page.route('**/api/nodes/n1/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[{"name":"llama-cpp"}]' })) + await page.goto('/app/nodes') + await page.getByRole('searchbox', { name: 'Search nodes' }).fill('atlas') + await expect(page.getByRole('row', { name: /atlas/ })).toBeVisible() + await page.getByRole('button', { name: 'Inspect atlas' }).click() + + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector.getByRole('heading', { name: 'Resources' })).toBeVisible() + await expect(inspector.getByRole('heading', { name: 'Workload' })).toBeVisible() + await expect(inspector.getByRole('link', { name: 'Open full node details' })).toBeVisible() + await expect(inspector.getByRole('button', { name: 'Drain', exact: true })).toBeVisible() + + const inspectorBox = await inspector.boundingBox() + expect(Math.abs(inspectorBox.y - 16)).toBeLessThanOrEqual(1) + expect(Math.abs(inspectorBox.height - 868)).toBeLessThanOrEqual(1) + await expect(inspector.locator('.node-inspector__actions')).toHaveCSS('display', 'grid') + }) + + test('keeps a desktop drawer in the visible viewport after opening from a long roster', async ({ page }) => { + await page.setViewportSize({ width: 1280, height: 800 }) + const nodes = Array.from({ length: 50 }, (_, index) => ({ + id: `long-${index}`, + name: `long-worker-${String(index).padStart(2, '0')}`, + node_type: 'backend', + status: 'healthy', + })) + await mockNodes(page, nodes) + await page.route('**/api/nodes/long-49/backends', route => route.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.goto('/app/nodes') + + await page.getByRole('button', { name: 'Inspect long-worker-49' }).click() + expect(await page.evaluate(() => window.scrollY)).toBeGreaterThan(1000) + const inspector = page.getByRole('complementary', { name: 'Node inspector' }) + await expect(inspector).toHaveCSS('position', 'fixed') + await expect(inspector.getByRole('heading', { name: 'long-worker-49' })).toBeVisible() + await expect(inspector.getByRole('link', { name: 'Open full node details' })).toBeVisible() + const box = await inspector.boundingBox() + expect(Math.abs(box.y - 16)).toBeLessThanOrEqual(1) + expect(Math.abs(box.height - 768)).toBeLessThanOrEqual(1) + }) + + test('shows model loading, error, retry, and empty states', async ({ page }) => { + await mockNodes(page) + let finishFirst + let attempts = 0 + await page.route('**/api/nodes/models', async route => { + attempts += 1 + if (attempts === 1) { + await new Promise(resolve => { finishFirst = resolve }) + return route.fulfill({ status: 500, contentType: 'application/json', body: '{"error":"database unavailable"}' }) + } + return route.fulfill({ status: 200, contentType: 'application/json', body: '[]' }) + }) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: 'Running models' }).click() + await expect(page.getByText('Loading running models…')).toBeVisible() + finishFirst() + await expect(page.getByText('Unable to load running models')).toBeVisible() + await page.getByRole('button', { name: 'Retry loading running models' }).click() + await expect(page.getByText('No running models')).toBeVisible() + expect(attempts).toBe(2) + }) + + test('mounts 50 of 1,000 running models and supports search and sorting', async ({ page }) => { + await mockNodes(page) + const models = Array.from({ length: 1000 }, (_, index) => ({ + id: `replica-${index}`, + node_id: 'n1', + model_name: `model-${String(index).padStart(4, '0')}`, + replica_index: 0, + address: `10.0.0.1:${51000 + index}`, + state: 'loaded', + in_flight: index % 7, + backend_type: 'llama-cpp', + last_used: new Date(Date.UTC(2026, 8, 1, 0, index)).toISOString(), + })) + await page.route('**/api/nodes/models', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(models) })) + await page.goto('/app/nodes') + await page.getByRole('tab', { name: 'Running models' }).click() + await expect(page.getByRole('table', { name: 'Running models' }).locator('tbody tr')).toHaveCount(50) + await expect(page.getByText('Page 1 of 20')).toBeVisible() + await page.getByRole('button', { name: 'Next model page' }).click() + await expect(page.getByText('Page 2 of 20')).toBeVisible() + await page.getByRole('searchbox', { name: 'Search running models' }).fill('model-0000') + await expect(page.locator('#fleet-models-panel').getByText('Page 1 of 1')).toBeVisible() + await page.getByRole('searchbox', { name: 'Search running models' }).fill('') + await page.getByRole('button', { name: /Sort by model/ }).click() + await expect(page.getByRole('table', { name: 'Running models' }).locator('tbody tr').first()).toContainText('model-0999') + }) + +}) diff --git a/core/http/react-ui/e2e/nodes-roster.spec.js b/core/http/react-ui/e2e/nodes-roster.spec.js index c6396d5b3..721f99766 100644 --- a/core/http/react-ui/e2e/nodes-roster.spec.js +++ b/core/http/react-ui/e2e/nodes-roster.spec.js @@ -1,64 +1,35 @@ import { test, expect } from './coverage-fixtures.js' async function mockCluster(page, nodes) { - await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) })) - await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) - await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) + await page.route('**/api/nodes', route => route.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify(nodes) })) } -test.describe('Nodes roster header', () => { - test('shows a cluster pulse line and no stat-card grid', async ({ page }) => { +test.describe('Nodes fleet roster', () => { + test('uses the fleet response without prefetching models or backends', async ({ page }) => { + const requests = [] + page.on('request', request => requests.push(request.url())) await mockCluster(page, [ - { id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' }, - { id: 'n2', name: 'beta', node_type: 'backend', address: '10.0.0.2:50051', status: 'draining' }, + { id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy', model_count: 3 }, + { id: 'a1', name: 'agent-1', node_type: 'agent', address: '10.0.0.9:50051', status: 'draining', model_count: 0 }, ]) await page.goto('/app/nodes') - await expect(page.locator('.cluster-pulse')).toBeVisible({ timeout: 15_000 }) - await expect(page.locator('.cluster-pulse')).toContainText('2 nodes') - await expect(page.locator('.stat-grid')).toHaveCount(0) + await expect(page.getByRole('table', { name: 'Fleet nodes' })).toBeVisible({ timeout: 15_000 }) + await expect(page.getByRole('tab', { name: 'Nodes' })).toHaveAttribute('aria-selected', 'true') + await page.getByRole('tab', { name: 'Nodes' }).click() + await expect(page.getByRole('row', { name: /alpha/ })).toContainText('3') + expect(requests.some(url => url.includes('/api/nodes/models'))).toBe(false) + expect(requests.some(url => /\/api\/nodes\/[^/]+\/backends/.test(url))).toBe(false) }) - test('shows an approval callout for pending nodes', async ({ page }) => { - await mockCluster(page, [{ id: 'n3', name: 'gamma', node_type: 'backend', address: '10.0.0.3:50051', status: 'pending' }]) + test('preserves the empty worker setup experience', async ({ page }) => { + await mockCluster(page, []) await page.goto('/app/nodes') - await expect(page.locator('.attention-callout')).toContainText('approval', { timeout: 15_000 }) - }) -}) - -test.describe('Nodes roster panels', () => { - test('shows used and total system RAM reported by a worker', async ({ page }) => { - await mockCluster(page, [ - { - id: 'n1', - name: 'alpha', - node_type: 'backend', - address: '10.0.0.1:50051', - status: 'healthy', - total_ram: 8_000_000_000, - available_ram: 3_000_000_000, - }, - ]) - - await page.goto('/app/nodes') - await expect(page.locator('.node-panel').filter({ hasText: 'alpha' })).toContainText('RAM 4.7 GB / 7.5 GB', { timeout: 15_000 }) - }) - - test('shows model chips without clicking and filters by type', async ({ page }) => { - await page.route('**/api/nodes', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([ - { id: 'n1', name: 'alpha', node_type: 'backend', address: '10.0.0.1:50051', status: 'healthy' }, - { id: 'a1', name: 'agent-1', node_type: 'agent', address: '10.0.0.9:50051', status: 'healthy' }, - ]) })) - await page.route('**/api/nodes/models', r => r.fulfill({ status: 200, contentType: 'application/json', body: JSON.stringify([ - { node_id: 'n1', model_name: 'llama-3.3', state: 'loaded', in_flight: 2, replica_index: 0 }, - ]) })) - await page.route('**/api/nodes/scheduling', r => r.fulfill({ status: 200, contentType: 'application/json', body: '[]' })) - - await page.goto('/app/nodes') - // model chip visible without any expand click - await expect(page.locator('.node-panel').filter({ hasText: 'alpha' }).getByText('llama-3.3')).toBeVisible({ timeout: 15_000 }) - // segmented filter: Agent shows the agent node, hides the backend node - await page.getByRole('radio', { name: /Agent/ }).click() - await expect(page.getByText('agent-1')).toBeVisible() - await expect(page.getByText('alpha')).toHaveCount(0) + await expect(page.getByText('No workers registered yet')).toBeVisible({ timeout: 15_000 }) + }) + + test('preserves the distributed-disabled setup experience', async ({ page }) => { + await page.route('**/api/nodes', route => route.fulfill({ status: 503, body: 'Service Unavailable' })) + await page.goto('/app/nodes') + await expect(page.getByText('Distributed Mode Not Enabled')).toBeVisible({ timeout: 15_000 }) }) }) diff --git a/core/http/react-ui/inline-style-baseline.txt b/core/http/react-ui/inline-style-baseline.txt index a08796291..4d0e90cbc 100644 --- a/core/http/react-ui/inline-style-baseline.txt +++ b/core/http/react-ui/inline-style-baseline.txt @@ -1 +1 @@ -514 +512 diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index 679e16522..853e5d4c9 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -382,7 +382,6 @@ padding-left: var(--spacing-md); padding-right: var(--spacing-md); } - .nav-external { font-size: 0.55rem; margin-left: auto; @@ -9062,8 +9061,8 @@ button.collapsible-header:focus-visible { } .console-rail-header__title { display: inline-flex; align-items: center; gap: var(--spacing-sm); } .console-rail-header__title i { color: var(--color-primary); font-size: 0.9rem; } -.console-rail-toggle { - display: none; +.console-rail-toggle, +.console-rail-collapse { width: 34px; height: 34px; place-items: center; @@ -9073,7 +9072,10 @@ button.collapsible-header:focus-visible { color: var(--color-text-secondary); cursor: pointer; } -.console-rail-toggle:hover { border-color: var(--color-border-strong); color: var(--color-text-primary); } +.console-rail-toggle { display: none; } +.console-rail-collapse { display: grid; } +.console-rail-toggle:hover, +.console-rail-collapse:hover { border-color: var(--color-border-strong); color: var(--color-text-primary); } .console-rail-groups { display: flex; flex-direction: column; gap: var(--spacing-xs); } .console-group { display: flex; flex-direction: column; gap: 1px; } .console-group + .console-group { @@ -9101,14 +9103,78 @@ button.collapsible-header:focus-visible { .console-rail .nav-item:hover:not(.active) { transform: translateX(2px); } .console-rail .nav-item.active { box-shadow: none; } .console-rail .nav-item.active .nav-icon { color: var(--color-primary); } +.console-rail--collapsed { + flex-basis: 60px; + width: 60px; + padding-inline: 6px; +} +.console-rail--collapsed .console-rail-header { + justify-content: center; + padding-inline: 0; +} +.console-rail--collapsed .console-rail-header__title, +.console-rail--collapsed .console-group-title, +.console-rail--collapsed .nav-label, +.console-rail--collapsed .nav-external { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; +} +.console-rail--collapsed .console-group + .console-group { + margin-top: var(--spacing-xs); + padding-top: var(--spacing-xs); +} +.console-rail--collapsed .nav-item { + justify-content: center; + min-height: 38px; + padding: 7px; +} +.console-rail--collapsed .nav-icon { width: auto; } +.console-rail--collapsed .nav-signal { + position: absolute; + top: 1px; + right: 1px; + min-width: 14px; + padding: 1px 3px; + font-size: 0.5rem; + line-height: 1.2; +} .console-body { flex: 1 1 auto; min-width: 0; + width: 100%; } @media (max-width: 768px) { .console-layout { flex-direction: column; padding: var(--spacing-sm); } - .console-rail { position: static; flex-basis: auto; width: 100%; } + .console-rail, + .console-rail--collapsed { position: static; flex-basis: auto; width: 100%; padding: var(--spacing-sm); } + .console-rail--collapsed .console-rail-header { justify-content: space-between; padding: var(--spacing-sm) var(--spacing-sm) var(--spacing-xs); } + .console-rail--collapsed .console-rail-header__title, + .console-rail--collapsed .console-group-title, + .console-rail--collapsed .nav-label, + .console-rail--collapsed .nav-external { + position: static; + width: auto; + height: auto; + padding: revert; + margin: 0; + overflow: visible; + clip: auto; + white-space: nowrap; + border: 0; + } + .console-rail--collapsed .console-group-title { padding: var(--spacing-xs) var(--spacing-sm); } + .console-rail--collapsed .nav-item { justify-content: flex-start; min-height: 44px; padding: 7px var(--spacing-sm); } + .console-rail--collapsed .nav-icon { width: 18px; } + .console-rail--collapsed .nav-signal { position: static; min-width: 0; padding: 1px 6px; font-size: var(--text-xs); line-height: inherit; } .console-rail-toggle { display: grid; } + .console-rail-collapse { display: none; } .console-rail-groups { display: none; } /* Thirteen destinations stacked in one column is taller than a phone, so opening the menu used to push the page's own heading past the fold: the @@ -9677,6 +9743,202 @@ button.collapsible-header:focus-visible { .model-chip__state { opacity: 0.85; font-style: normal; } .node-filter { margin-bottom: var(--spacing-lg); } .node-detail__metrics { display: flex; gap: var(--spacing-xl); margin: var(--spacing-md) 0 var(--spacing-lg); flex-wrap: wrap; } +.node-detail__metric-note { display: block; color: var(--color-text-muted); font-size: var(--text-xs); margin-top: 2px; } + +/* Nodes fleet operations dashboard */ +.nodes-fleet-page { container-name: fleet-page; container-type: inline-size; isolation: isolate; position: relative; } +.nodes-fleet-page--inspecting { container-type: normal; } +.page-transition:has(.nodes-fleet-page--inspecting) { animation: none !important; transform: none !important; } +.nodes-fleet-page__header { align-items: flex-start; margin-bottom: 18px; } +.nodes-fleet-page__header .page-title { font-size: 1.75rem; letter-spacing: -.03em; } +.nodes-fleet-page__header .page-header__supporting { font-size: var(--text-xs); margin-top: 3px; } +.fleet-kicker { display: block; color: var(--color-text-muted); font-size: .625rem; font-weight: 650; letter-spacing: .08em; margin-bottom: 4px; text-transform: uppercase; } +.fleet-overview { + background: var(--color-bg-secondary); + border: 1px solid var(--color-border-subtle); + border-radius: var(--radius-lg); + display: grid; + grid-template-columns: minmax(265px, 1.45fr) repeat(4, minmax(145px, .8fr)); + margin-bottom: 12px; + overflow: hidden; +} +.fleet-overview__cell { height: 204px; min-width: 0; padding: 22px 20px; } +.fleet-overview__cell + .fleet-overview__cell { border-left: 1px solid var(--color-border-subtle); } +.fleet-health__headline { align-items: baseline; display: flex; gap: 8px; margin: 10px 0 22px; white-space: nowrap; } +.fleet-health__headline strong { font-size: 1.625rem; letter-spacing: -.025em; } +.fleet-health__headline span { color: var(--color-text-muted); font-size: .75rem; } +.fleet-health__bar { display: block; height: 11px; margin-bottom: 17px; width: 100%; } +.fleet-health__segment--healthy { fill: var(--color-success); } +.fleet-health__segment--draining, .fleet-health__segment--pending { fill: var(--color-warning); } +.fleet-health__segment--unhealthy { fill: var(--color-error); } +.fleet-health__legend { display: grid; gap: 8px; grid-template-columns: repeat(3, minmax(0, 1fr)); } +.fleet-health__legend > div { min-width: 0; } +.fleet-health__legend span { align-items: center; color: var(--color-text-muted); display: flex; font-size: .625rem; gap: 6px; white-space: nowrap; } +.fleet-health__legend strong { display: block; font-size: .75rem; margin-top: 4px; white-space: nowrap; } +.fleet-health__dot { background: var(--color-error); border-radius: 50%; display: inline-block; height: 7px; width: 7px; } +.fleet-health__dot--healthy { background: var(--color-success); } +.fleet-health__dot--draining { background: var(--color-warning); } +.fleet-gauge { align-content: start; display: grid; justify-items: center; } +.fleet-gauge .fleet-kicker { justify-self: start; white-space: nowrap; } +.fleet-gauge__graphic { height: 73px; margin-top: 18px; position: relative; width: 136px; } +.fleet-gauge__graphic svg { height: 73px; overflow: visible; width: 136px; } +.fleet-gauge__graphic path { fill: none; stroke-linecap: butt; stroke-width: 7; } +.fleet-gauge__track { stroke: var(--color-bg-tertiary); } +.fleet-gauge__value { stroke: var(--fleet-gauge-color); } +.fleet-gauge--vram { --fleet-gauge-color: var(--color-success); } +.fleet-gauge--ram { --fleet-gauge-color: var(--color-primary); } +.fleet-gauge--cpu { --fleet-gauge-color: var(--color-info); } +.fleet-gauge--disk { --fleet-gauge-color: var(--color-info); } +.fleet-gauge__graphic strong { bottom: 0; font-size: 1rem; left: 0; position: absolute; text-align: center; width: 136px; } +.fleet-gauge__value-text { font-family: var(--font-mono); font-size: .6875rem; font-weight: 600; margin-top: 4px; text-align: center; white-space: nowrap; } +.fleet-gauge__detail, .fleet-gauge__coverage { color: var(--color-text-muted); font-size: .5625rem; line-height: 1.35; margin-top: 3px; text-align: center; } +.fleet-gauge__coverage { margin-top: 1px; } +.fleet-attention { align-items: center; background: var(--color-warning-light); border: 1px solid var(--color-warning-border); border-radius: var(--radius-md); display: flex; gap: 18px; margin-bottom: 24px; min-height: 52px; padding: 8px 15px; } +.fleet-attention__title { align-items: center; display: flex; flex: 0 0 auto; font-size: var(--text-xs); gap: 9px; } +.fleet-attention__title i { color: var(--color-warning); } +.fleet-attention__filters { align-items: center; display: flex; flex: 1; flex-wrap: wrap; gap: 6px; justify-content: flex-end; } +.fleet-attention__filter { align-items: center; background: transparent; border: 1px solid transparent; border-radius: var(--radius-full); color: var(--color-text-secondary); cursor: pointer; display: flex; font: inherit; font-size: .625rem; gap: 6px; min-height: 28px; padding: 3px 9px; text-align: left; } +.fleet-attention__filter:hover, .fleet-attention__filter.is-active { color: var(--color-primary); } +.fleet-attention__filter:hover { background: var(--color-bg-hover); } +.fleet-attention__filter.is-active { background: var(--color-bg-secondary); border-color: var(--color-border-strong); box-shadow: 0 1px 2px rgba(0, 0, 0, .12); } +.fleet-attention__filter strong { color: var(--color-text-primary); } +.fleet-workbench { background: var(--color-bg-secondary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-lg); container-name: fleet-workbench; container-type: inline-size; overflow: hidden; } +.fleet-workbench__tabs { align-items: stretch; background: var(--color-bg-secondary); border-bottom: 1px solid var(--color-border-subtle); display: flex; min-height: 43px; padding: 0 11px; } +.fleet-workbench__tabs button { align-items: center; background: transparent; border: 0; border-bottom: 2px solid transparent; color: var(--color-text-muted); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); font-weight: 600; gap: 7px; margin-bottom: -1px; padding: 0 11px; } +.fleet-workbench__tabs button:hover { color: var(--color-text-primary); } +.fleet-workbench__tabs button:focus-visible { border-radius: var(--radius-sm); box-shadow: inset 0 0 0 2px var(--color-primary); outline: none; } +.fleet-workbench__tabs button.is-active { border-bottom-color: var(--color-primary); color: var(--color-text-primary); } +.fleet-workbench__tabs button span { background: var(--color-bg-tertiary); border-radius: var(--radius-full); color: var(--color-text-muted); font-family: var(--font-mono); font-size: .5625rem; min-width: 19px; padding: 2px 6px; text-align: center; } +.fleet-workbench__layout { min-width: 0; } +.fleet-workbench__fleet { min-width: 0; } +.fleet-toolbar { border-bottom: 1px solid var(--color-border-subtle); display: grid; grid-template-columns: minmax(190px, 1fr) repeat(3, minmax(112px, auto)); gap: 7px; padding: 10px 11px; } +.fleet-toolbar__search { height: 34px; min-width: 0; } +.fleet-select-wrap { display: block; min-width: 0; position: relative; } +.fleet-select { appearance: none; background: var(--color-bg-primary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); color: var(--color-text-secondary); cursor: pointer; font: inherit; font-size: var(--text-xs); height: 34px; padding: 0 28px 0 10px; width: 100%; } +.fleet-select:hover { border-color: var(--color-border-strong); } +.fleet-select:focus-visible { border-color: var(--color-primary); box-shadow: 0 0 0 3px var(--color-primary-light); outline: none; } +.fleet-select__chevron { color: var(--color-text-muted); font-size: .625rem; pointer-events: none; position: absolute; right: 10px; top: 12px; } +.fleet-bulkbar { align-items: center; background: var(--color-primary-light); border-bottom: 1px solid var(--color-border-strong); display: flex; flex-wrap: wrap; gap: var(--spacing-xs); min-height: 42px; padding: 5px 11px; } +.fleet-bulkbar strong { font-size: var(--text-xs); margin-right: var(--spacing-xs); } +.fleet-bulkbar__clear { background: none; border: 0; color: var(--color-primary); cursor: pointer; font: inherit; font-size: var(--text-xs); } +.fleet-bulkbar__count { color: var(--color-text-muted); font-size: var(--text-xs); margin-left: auto; } +.fleet-table-wrap { overflow-x: auto; } +.fleet-table { border-collapse: collapse; font-size: var(--text-xs); min-width: 820px; width: 100%; } +.fleet-table th, .fleet-table td { border-bottom: 1px solid var(--color-border-subtle); padding: 10px 12px; text-align: left; vertical-align: middle; } +.fleet-table thead th { background: var(--color-bg-secondary); color: var(--color-text-muted); font-weight: 600; } +.fleet-table__check { width: 34px; } +.fleet-table__sort { background: none; border: 0; color: inherit; cursor: pointer; font: inherit; font-weight: inherit; padding: 0; } +.fleet-table__row { cursor: pointer; height: 67px; transition: background var(--duration-fast) var(--ease-out, ease-out); } +.fleet-table__row:hover, .fleet-table__row:focus { background: var(--color-bg-hover); outline: none; } +.fleet-table__row.is-selected { background: var(--color-primary-light); box-shadow: inset 2px 0 var(--color-primary); } +.fleet-table__node { background: none; border: 0; color: var(--color-text-primary); cursor: pointer; display: block; font: inherit; font-weight: 600; padding: 0; text-align: left; } +.fleet-table__node + span, .fleet-table__subvalue { color: var(--color-text-muted); display: block; font-size: .625rem; margin-top: 1px; } +.fleet-table__approve { background: none; border: 0; color: var(--color-primary); cursor: pointer; display: block; font: inherit; font-size: .625rem; font-weight: 650; margin-top: 5px; padding: 0; } +.fleet-table__unknown { color: var(--color-text-muted); } +.fleet-table__resource { align-items: center; display: grid; gap: 5px; grid-template-columns: minmax(45px, 1fr) auto; min-width: 92px; } +.fleet-table__resource > span:last-child { color: var(--color-text-muted); font-family: var(--font-mono); font-size: .5625rem; white-space: nowrap; } +.fleet-table__resource-track { appearance: none; background: var(--color-bg-tertiary); border: 0; border-radius: var(--radius-full); display: block; height: 5px; overflow: hidden; width: 100%; } +.fleet-table__resource-track::-webkit-progress-bar { background: var(--color-bg-tertiary); } +.fleet-table__resource-track::-webkit-progress-value { background: var(--fleet-resource-color); border-radius: var(--radius-full); } +.fleet-table__resource-track::-moz-progress-bar { background: var(--fleet-resource-color); border-radius: var(--radius-full); } +.fleet-table__resource--vram { --fleet-resource-color: var(--color-success); } +.fleet-table__resource--ram { --fleet-resource-color: var(--color-primary); } +.fleet-table__capacity { display: grid; gap: 7px; min-width: 205px; } +.fleet-table__capacity-row { align-items: center; display: grid; gap: 8px; grid-template-columns: 38px minmax(0, 1fr); } +.fleet-table__capacity-row > b { color: var(--color-text-muted); font-size: .5625rem; font-weight: 500; } +.fleet-table__group th { background: var(--color-bg-tertiary); color: var(--color-text-primary); padding: 7px 10px; } +.fleet-table__group th, .fleet-table__group label { align-items: center; display: flex; gap: var(--spacing-xs); } +.fleet-table__group span { color: var(--color-text-muted); font-weight: 400; margin-left: auto; } +.fleet-table__empty { color: var(--color-text-muted); padding: var(--spacing-xl); text-align: center; } +.fleet-pagination { align-items: center; display: flex; gap: var(--spacing-xs); justify-content: flex-end; min-height: 45px; padding: 5px 11px; } +.fleet-pagination span { color: var(--color-text-muted); font-size: var(--text-xs); margin-right: var(--spacing-xs); } +.model-workbench__scope { align-items: center; background: var(--color-bg-tertiary); border-bottom: 1px solid var(--color-border-subtle); display: flex; justify-content: space-between; min-height: 48px; padding: 7px 12px; } +.model-workbench__scope > div { display: grid; gap: 1px; } +.model-workbench__scope strong { color: var(--color-text-primary); font-size: var(--text-sm); } +.model-workbench__scope span { color: var(--color-text-muted); font-size: .625rem; } +.model-toolbar { border-bottom: 1px solid var(--color-border-subtle); padding: 9px 11px; } +.model-toolbar .fleet-toolbar__search { max-width: 430px; width: 100%; } +.model-workbench__state { align-items: center; color: var(--color-text-muted); display: flex; flex-direction: column; gap: 6px; justify-content: center; min-height: 300px; padding: var(--spacing-xl); text-align: center; } +.model-workbench__state > i { color: var(--color-text-muted); font-size: var(--text-xl); } +.model-workbench__state strong { color: var(--color-text-primary); font-size: var(--text-base); } +.model-workbench__state span { font-size: var(--text-xs); max-width: 420px; } +.model-workbench__state .btn { margin-top: 5px; } +.model-workbench__state--error > i { color: var(--color-error); } +.model-fleet-table { min-width: 720px; } +.model-fleet-table th:first-child, .model-fleet-table td:first-child { width: 37%; } +.model-fleet-table .model-fleet-table__actions { padding-left: 4px; padding-right: 8px; text-align: right; width: 38px; } +.model-fleet-table .fleet-table__row:hover .action-menu__trigger, +.model-fleet-table .action-menu__trigger:focus-visible, +.model-fleet-table .action-menu__trigger.is-open { opacity: 1; } +.model-backend-list { display: flex; flex-wrap: wrap; gap: 4px; } +.model-backend-list > span:not(.fleet-table__unknown, .text-muted) { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-full); color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .5625rem; padding: 2px 6px; } +.node-inspector__scrim { display: none; } +.node-inspector { animation: node-inspector-in 180ms var(--ease-out, ease-out) both; background: var(--color-bg-primary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-xl) 0 0 var(--radius-xl); bottom: var(--spacing-md); box-shadow: -18px 0 42px rgba(0, 0, 0, .24); display: grid; grid-template-rows: auto minmax(0, 1fr) auto; overflow: hidden; position: fixed; right: var(--spacing-md); top: var(--spacing-md); width: 360px; z-index: 80; } +@keyframes node-inspector-in { from { opacity: 0; transform: translateX(18px); } to { opacity: 1; transform: translateX(0); } } +.node-inspector__header { background: linear-gradient(180deg, var(--color-bg-secondary), var(--color-bg-primary)); border-bottom: 1px solid var(--color-border-subtle); padding: 18px 22px 17px; } +.node-inspector__topbar { align-items: center; display: flex; justify-content: space-between; min-height: 30px; } +.node-inspector__topbar .fleet-kicker { margin: 0; } +.node-inspector__header h2 { font-size: 1.25rem; letter-spacing: -.025em; line-height: 1.2; margin: 9px 0 0; overflow-wrap: anywhere; } +.node-inspector__header > p, .node-inspector__identity > p { color: var(--color-text-muted); font-size: .6875rem; margin: 4px 0 9px; } +.node-inspector__identity .status-pill { margin: 0; } +.node-inspector__body { min-height: 0; overflow-y: auto; overscroll-behavior: contain; padding: 0 22px; scrollbar-gutter: stable; } +.node-inspector__section { border-bottom: 1px solid var(--color-border-subtle); padding: 15px 0; } +.node-inspector__section:last-of-type { border-bottom: 0; } +.node-inspector__section h3 { color: var(--color-text-muted); font-size: .625rem; font-weight: 650; letter-spacing: .08em; margin: 0 0 9px; text-transform: uppercase; } +.node-inspector__address { color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .625rem; overflow-wrap: anywhere; } +.node-inspector__metrics { display: grid; gap: 0; margin: 0; } +.node-inspector__metrics > div { display: grid; font-size: var(--text-xs); grid-template-columns: 92px 1fr; padding: 5px 0; } +.node-inspector__metrics dt { color: var(--color-text-muted); } +.node-inspector__metrics dd { margin: 0; text-align: right; } +.node-inspector__labels { display: flex; flex-wrap: wrap; gap: 5px; margin-top: 8px; } +.node-inspector__labels > span:not(.text-muted) { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-full); color: var(--color-text-secondary); font-family: var(--font-mono); font-size: .5625rem; padding: 2px 7px; } +.node-inspector__resource { margin: 10px 0 13px; } +.node-inspector__resource-label { align-items: baseline; display: flex; font-size: var(--text-xs); justify-content: space-between; margin-bottom: 6px; } +.node-inspector__resource-label strong span, .node-inspector__resource-label > span { color: var(--color-text-muted); font-size: .625rem; font-weight: 400; } +.node-inspector__resource-track { appearance: none; background: var(--color-bg-tertiary); border: 0; border-radius: var(--radius-full); display: block; height: 7px; overflow: hidden; width: 100%; } +.node-inspector__resource-track::-webkit-progress-bar { background: var(--color-bg-tertiary); } +.node-inspector__resource-track::-webkit-progress-value { background: var(--fleet-resource-color); border-radius: var(--radius-full); } +.node-inspector__resource-track::-moz-progress-bar { background: var(--fleet-resource-color); border-radius: var(--radius-full); } +.node-inspector__resource--vram { --fleet-resource-color: var(--color-success); } +.node-inspector__resource--ram { --fleet-resource-color: var(--color-primary); } +.node-inspector__actions { background: var(--color-bg-secondary); border-top: 1px solid var(--color-border-subtle); box-shadow: 0 -8px 18px rgba(0, 0, 0, .06); display: grid; gap: var(--spacing-xs); grid-template-columns: 1fr 1fr; padding: 14px 22px; } +.node-inspector__actions--single { grid-template-columns: 1fr; } +.node-inspector__actions .btn { justify-content: center; min-width: 0; } +.node-inspector__back { align-items: center; background: transparent; border: 0; color: var(--color-primary); cursor: pointer; display: flex; font: inherit; font-size: var(--text-xs); gap: 6px; max-width: 285px; overflow: hidden; padding: 3px 0; text-overflow: ellipsis; white-space: nowrap; } +.node-inspector__back:focus-visible { border-radius: var(--radius-sm); outline: 2px solid var(--color-primary); outline-offset: 3px; } +.model-inspector__backends { margin-top: 10px; } +.model-inspector__nodes { display: grid; gap: 9px; } +.model-inspector__node { background: var(--color-bg-tertiary); border: 1px solid var(--color-border-subtle); border-radius: var(--radius-md); padding: 10px; } +.model-inspector__node-heading { align-items: center; display: flex; gap: 7px; justify-content: space-between; } +.model-inspector__node-heading button { background: transparent; border: 0; color: var(--color-primary); cursor: pointer; font: inherit; font-size: var(--text-sm); font-weight: 650; overflow: hidden; padding: 0; text-align: left; text-overflow: ellipsis; white-space: nowrap; } +.model-inspector__node-heading strong { font-size: var(--text-sm); overflow-wrap: anywhere; } +.model-inspector__node > p { color: var(--color-text-muted); font-size: .625rem; margin: 5px 0 8px; } +.model-inspector__node ul { border-top: 1px solid var(--color-border-subtle); list-style: none; margin: 0; padding: 5px 0 0; } +.model-inspector__node li { align-items: baseline; display: grid; font-size: .625rem; gap: 7px; grid-template-columns: auto minmax(0, 1fr); padding-top: 4px; } +.model-inspector__node li span { color: var(--color-text-muted); } +.model-inspector__node code { color: var(--color-text-secondary); font-size: .5625rem; overflow: hidden; text-align: right; text-overflow: ellipsis; white-space: nowrap; } + +@container fleet-page (max-width: 760px) { + .fleet-overview { grid-template-columns: repeat(2, minmax(0, 1fr)); } + .fleet-overview__cell { min-height: 146px; } + .fleet-overview__cell + .fleet-overview__cell { border-left: 0; } + .fleet-overview__cell:nth-child(even) { border-left: 1px solid var(--color-border-subtle); } + .fleet-overview__cell:nth-child(n + 3) { border-top: 1px solid var(--color-border-subtle); } + .fleet-health { grid-column: 1 / -1; } + .fleet-health { min-height: 126px; } + .fleet-attention { align-items: flex-start; flex-direction: column; gap: 7px; } + .fleet-attention__filters { justify-content: flex-start; } + .fleet-toolbar { grid-template-columns: 1fr 1fr; } + .fleet-toolbar__search { grid-column: 1 / -1; } +} +@media (max-width: 768px) { + .node-inspector__scrim { animation: node-inspector-scrim-in 180ms ease-out both; background: rgba(7, 10, 18, .52); display: block; inset: 0; position: fixed; z-index: 79; } + .node-inspector { border-bottom: 0; border-radius: 0; border-top: 0; bottom: 0; max-width: 100vw; position: fixed; top: 0; width: min(390px, 100vw); z-index: 80; } + .node-inspector__header { padding-left: 18px; padding-right: 18px; padding-top: max(16px, env(safe-area-inset-top)); } + .node-inspector__body { padding-left: 18px; padding-right: 18px; } + .node-inspector__actions { padding-bottom: max(14px, env(safe-area-inset-bottom)); padding-left: 18px; padding-right: 18px; } +} +@keyframes node-inspector-scrim-in { from { opacity: 0; } to { opacity: 1; } } /* Rendered Markdown --------------------------------------------------------- Gallery descriptions, backend descriptions and voice notes are all diff --git a/core/http/react-ui/src/components/ActionMenu.jsx b/core/http/react-ui/src/components/ActionMenu.jsx index 55010102c..e2aea8ddd 100644 --- a/core/http/react-ui/src/components/ActionMenu.jsx +++ b/core/http/react-ui/src/components/ActionMenu.jsx @@ -1,4 +1,4 @@ -import { useRef, useState, useEffect, useCallback } from 'react' +import { useRef, useState, useEffect, useCallback, useId } from 'react' import Popover from './Popover' // ActionMenu renders a kebab (three-dot) button that opens a popover with a @@ -20,6 +20,7 @@ import Popover from './Popover' // Escape — close, return focus to trigger export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel, compact = false }) { const triggerRef = useRef(null) + const menuId = useId() const [open, setOpen] = useState(false) const [activeIdx, setActiveIdx] = useState(-1) @@ -48,7 +49,12 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel, } const handleMenuKeyDown = (e) => { - if (e.key === 'ArrowDown') { + if (e.key === 'Escape') { + // Keep the same Escape press from also closing a surrounding inspector. + e.preventDefault() + e.stopPropagation() + close() + } else if (e.key === 'ArrowDown') { e.preventDefault() setActiveIdx(i => Math.min(interactive.length - 1, (i < 0 ? -1 : i) + 1)) } else if (e.key === 'ArrowUp') { @@ -65,7 +71,7 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel, const item = interactive[activeIdx] if (item && !item.disabled) { close() - item.onClick?.() + item.onClick?.(triggerRef.current) } } } @@ -92,6 +98,7 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel,
= 0 ? `${menuId}-item-${activeIdx}` : undefined} className="action-menu" onKeyDown={handleMenuKeyDown} // Capture focus when the menu opens so arrow keys work without the @@ -118,8 +125,10 @@ export default function ActionMenu({ items, ariaLabel = 'Actions', triggerLabel, return (