diff --git a/core/http/admission_handler_test.go b/core/http/admission_handler_test.go index 89a4ce0d7..77ec65da0 100644 --- a/core/http/admission_handler_test.go +++ b/core/http/admission_handler_test.go @@ -2,6 +2,7 @@ package http import ( "errors" + "fmt" "net/http" "net/http/httptest" "testing" @@ -9,6 +10,7 @@ import ( "github.com/labstack/echo/v4" corebackend "github.com/mudler/LocalAI/core/backend" + "github.com/mudler/LocalAI/core/services/nodes" ) func TestApplyBackendAdmission(t *testing.T) { @@ -44,3 +46,38 @@ func TestApplyBackendAdmission(t *testing.T) { } }) } + +func TestApplyNoAvailableNodes(t *testing.T) { + t.Run("maps ErrNoAvailableNodes to 503", func(t *testing.T) { + // The scheduler wraps the sentinel in fmt.Errorf chains and via + // errors.Join — errors.Is must still find it. + wrapped := fmt.Errorf("routing model foo: %w", + fmt.Errorf("no available nodes: %w", + fmt.Errorf("no healthy nodes available: %w", + errors.Join(nodes.ErrEvictionBusy, nodes.ErrNoAvailableNodes)))) + + code := applyNoAvailableNodes(wrapped, http.StatusInternalServerError) + if code != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", code) + } + }) + + t.Run("maps selector-mismatch chain to 503", func(t *testing.T) { + wrapped := fmt.Errorf("routing model bar: %w", + fmt.Errorf("no available nodes: %w", + fmt.Errorf("no healthy nodes match selector for model bar: {\"gpu.vendor\":\"tpu\"}: %w", + nodes.ErrNoAvailableNodes))) + + code := applyNoAvailableNodes(wrapped, http.StatusInternalServerError) + if code != http.StatusServiceUnavailable { + t.Fatalf("expected 503, got %d", code) + } + }) + + t.Run("passes through unrelated errors unchanged", func(t *testing.T) { + code := applyNoAvailableNodes(errors.New("database timeout"), http.StatusInternalServerError) + if code != http.StatusInternalServerError { + t.Fatalf("expected 500, got %d", code) + } + }) +} diff --git a/core/http/app.go b/core/http/app.go index 9485e9b9f..9697db12b 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -88,6 +88,19 @@ func applyBackendAdmission(err error, code int, c echo.Context) int { return http.StatusTooManyRequests } +// applyNoAvailableNodes maps scheduler "no available nodes" errors to 503. +// When the cluster has no healthy node to serve a model — all are full, a +// node selector excludes every candidate, or eviction could not free a slot — +// the request is retryable, not a server bug. Without this the error fell +// through to 500, which tells clients something is broken when they just +// need to wait for a node. +func applyNoAvailableNodes(err error, code int) int { + if errors.Is(err, nodes.ErrNoAvailableNodes) { + return http.StatusServiceUnavailable + } + return code +} + // respondModelLoading answers a request whose model is still cold-loading with // 503, a Retry-After header and the live `loading` object, reporting true when // it handled the error. @@ -208,6 +221,7 @@ func API(application *application.Application) (*echo.Echo, error) { } code = applyModelLoadCooldown(err, code, c) code = applyBackendAdmission(err, code, c) + code = applyNoAvailableNodes(err, code) // Handle 404 errors: serve React SPA for HTML requests, JSON otherwise if code == http.StatusNotFound { @@ -245,6 +259,7 @@ func API(application *application.Application) (*echo.Echo, error) { // Opaque errors deliberately withhold the body, so a still-loading // model gets the status and Retry-After but no progress detail. code = applyModelLoading(err, code, c) + code = applyNoAvailableNodes(err, code) c.NoContent(code) } } diff --git a/core/services/nodes/router.go b/core/services/nodes/router.go index d024b5621..36eee4a47 100644 --- a/core/services/nodes/router.go +++ b/core/services/nodes/router.go @@ -955,7 +955,7 @@ func (r *SmartRouter) resolveSelectorCandidates(ctx context.Context, modelID str return nil, fmt.Errorf("looking up nodes for selector %s: %w", sched.NodeSelector, err) } if len(candidates) == 0 { - return nil, fmt.Errorf("no healthy nodes match selector for model %s: %s", modelID, sched.NodeSelector) + return nil, fmt.Errorf("no healthy nodes match selector for model %s: %s: %w", modelID, sched.NodeSelector, ErrNoAvailableNodes) } return extractNodeIDs(candidates), nil } @@ -1167,9 +1167,9 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID evictedNode, evictErr := r.evictLRUAndFreeNodeFrom(ctx, candidateNodeIDs) if evictErr != nil { if errors.Is(evictErr, ErrEvictionBusy) { - return nil, "", 0, fmt.Errorf("no healthy nodes available: %w", evictErr) + return nil, "", 0, fmt.Errorf("no healthy nodes available: %w", errors.Join(evictErr, ErrNoAvailableNodes)) } - return nil, "", 0, fmt.Errorf("no healthy nodes available and eviction failed: %w", evictErr) + return nil, "", 0, fmt.Errorf("no healthy nodes available and eviction failed: %w", errors.Join(evictErr, ErrNoAvailableNodes)) } node = evictedNode } @@ -2059,6 +2059,13 @@ func (r *SmartRouter) EvictLRU(ctx context.Context, nodeID string) (string, erro // and none can be evicted to make room. var ErrEvictionBusy = errors.New("all models busy, cannot evict") +// ErrNoAvailableNodes is returned when the scheduler cannot find any healthy +// node to serve a model — all nodes are full and eviction cannot free a slot, +// or a node selector excludes every candidate. The HTTP layer maps this to +// 503 so clients treat it as a transient condition rather than a server bug +// (which is what 500 would imply). +var ErrNoAvailableNodes = errors.New("no available nodes") + // evictLRUAndFreeNode finds the globally least-recently-used model with zero in-flight, // unloads it, and returns its node for reuse. If all models are busy, retries briefly. // diff --git a/core/services/nodes/router_test.go b/core/services/nodes/router_test.go index 3735ec5e4..7577e2846 100644 --- a/core/services/nodes/router_test.go +++ b/core/services/nodes/router_test.go @@ -843,6 +843,26 @@ var _ = Describe("SmartRouter", func() { Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no available nodes")) }) + + It("wraps ErrNoAvailableNodes when all nodes are full and eviction cannot help", func() { + // gorm.ErrRecordNotFound is the registry's verdict that no node + // matches — the scheduler then falls through to eviction. With + // DB nil, eviction returns ErrEvictionBusy, and the scheduler + // wraps the error with ErrNoAvailableNodes so the HTTP layer can + // map it to 503 instead of 500. + reg.findIdleErr = errors.New("no idle") + reg.findLeastLoadedErr = gorm.ErrRecordNotFound + + router := NewSmartRouter(reg, SmartRouterOptions{ + Unloader: unloader, + ClientFactory: factory, + }) + + _, err := router.Route(context.Background(), "m5", "models/m5.gguf", "llama-cpp", "", nil, false) + Expect(err).To(HaveOccurred()) + Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue()) + Expect(errors.Is(err, ErrEvictionBusy)).To(BeTrue()) + }) }) Describe("UnloadModel (mock-based)", func() { @@ -970,6 +990,7 @@ var _ = Describe("SmartRouter", func() { _, err := router.Route(context.Background(), "aliased-model", "models/aliased.gguf", "llama-cpp", "", nil, false) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no healthy nodes match selector")) + Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue()) }) It("returns error when no nodes match selector", func() { @@ -988,6 +1009,7 @@ var _ = Describe("SmartRouter", func() { _, err := router.Route(context.Background(), "no-match-model", "models/nomatch.gguf", "llama-cpp", "", nil, false) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("no healthy nodes match selector")) + Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue()) }) It("uses regular methods when model has no scheduling config", func() { diff --git a/docs/content/reference/api-errors.md b/docs/content/reference/api-errors.md index bcdc2f7f5..20f63786f 100644 --- a/docs/content/reference/api-errors.md +++ b/docs/content/reference/api-errors.md @@ -90,6 +90,7 @@ The `/v1/responses` endpoint returns errors with this structure: | 422 | Unprocessable Entity | Validation failed (e.g., invalid parameter range) | | 429 | Too Many Requests | All backends are saturated (per-model `max_concurrent` or process-wide `--max-concurrent-backend-requests` ceiling reached). Includes a `Retry-After` header and `type: "rate_limit_error"` so OpenAI-compatible clients and harnesses back off automatically | | 500 | Internal Server Error | Backend inference failure, unexpected server errors | +| 503 | Service Unavailable | No healthy node available to serve the model (cluster is full, eviction cannot free a slot, or a `node_selector` excludes all candidates). Also used during model-load cooldown and while a model is still cold-loading. Retryable | ## Global Error Handling diff --git a/docs/content/reference/runtime-errors.md b/docs/content/reference/runtime-errors.md index 3670cd708..fd5ef2bd8 100644 --- a/docs/content/reference/runtime-errors.md +++ b/docs/content/reference/runtime-errors.md @@ -21,6 +21,7 @@ The left column is the literal string as it appears in the LocalAI server log (o | `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). When a local backend has already exited, the error includes its exit code and last stderr line. | Use the included stderr diagnostic when present; otherwise check the log lines just above. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. | | `failed to load model: ...` | Returned by the load endpoints and several feature paths (voice, realtime, audio transform) when the model config could not be resolved or the backend load failed. | Confirm the model name exists (`local-ai models list`) and its YAML is valid. The trailing text carries the specific reason. | | HTTP `503` with a `Retry-After` header, after a load failed | Model-load failure cooldown. After a model fails to load, LocalAI refuses new load attempts for that model for a short window so a client that keeps polling a broken model does not respawn a crashing backend on every request. The window starts at `--model-load-failure-cooldown` (default `10s`) and doubles per consecutive failure up to 5m; it resets on the first success. | Fix the underlying load failure (see the rows above), then wait out the `Retry-After` seconds before retrying, or restart LocalAI to clear the cooldown. Set `--model-load-failure-cooldown 0` (or `LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN=0`) to disable the cooldown entirely. See {{% relref "reference/cli-reference" %}}. | +| HTTP `503` with `no available nodes` or `no healthy nodes match selector` | The scheduler could not find any healthy node to serve the model. All nodes are full and eviction cannot free a slot, or a `node_selector` in the model's scheduling config excludes every candidate. | Retry after a node becomes available or an in-flight request completes and frees a slot. In a cluster, add nodes or replicas. If a selector is set, confirm at least one healthy node matches it. | | HTTP `429` with a `Retry-After` header, under load | Per-model concurrency limit reached. When a model config sets a `MaxConcurrent` limit, extra requests are rejected with `429` and a `Retry-After` (whole seconds, floor 1) instead of queueing. | Retry after the advised delay, raise the model's concurrency limit, or run more replicas. | | HTTP `429` when backend inference is saturated | The process-wide `--max-concurrent-backend-requests` backend-execution ceiling is full. This protects inference and in-flight backend-trace memory without blocking UI or administrative endpoints. | Retry after the advised delay, reduce inference concurrency, raise the limit if the host has capacity, or add replicas. | | `invalid pitch` (with CUDA) | The prompt exceeded the model's context size. | Reduce the prompt length, or raise the model's context size (`context_size:` in the model YAML). |