fix: return 503 when scheduler has no available nodes

When the scheduler cannot find any healthy node to serve a model —
all nodes are full and eviction cannot free a slot, or a node_selector
excludes every candidate — the error fell through to 500. A 500 tells
clients something is broken when the condition is transient and
retryable.

The router now wraps these errors with a new ErrNoAvailableNodes
sentinel. The HTTP error handler maps it to 503 via applyNoAvailableNodes,
following the same pattern as applyBackendAdmission (429). Unrelated
scheduler errors (DB timeouts, registry lookups) still return 500.

Three return sites are wrapped:
- resolveSelectorCandidates: selector matches zero healthy nodes
- scheduleNewModel eviction-busy: all models have in-flight requests
- scheduleNewModel eviction-failed: eviction itself errored

The existing scheduleAndLoad wrapper ("no available nodes: %w") preserves
the sentinel through the chain via errors.Is, as does ModelRouterAdapter.

Assisted-by: AGENT:regolo/glm5.2 [TOOL]

Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/http/admission_handler_test.go

@@ -2,6 +2,7 @@ package http
 
 import (
 	"errors"
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"testing"
@@ -9,6 +10,7 @@ import (
 
 	"github.com/labstack/echo/v4"
 	corebackend "github.com/mudler/LocalAI/core/backend"
+	"github.com/mudler/LocalAI/core/services/nodes"
 )
 
 func TestApplyBackendAdmission(t *testing.T) {
@@ -44,3 +46,38 @@ func TestApplyBackendAdmission(t *testing.T) {
 		}
 	})
 }
+
+func TestApplyNoAvailableNodes(t *testing.T) {
+	t.Run("maps ErrNoAvailableNodes to 503", func(t *testing.T) {
+		// The scheduler wraps the sentinel in fmt.Errorf chains and via
+		// errors.Join — errors.Is must still find it.
+		wrapped := fmt.Errorf("routing model foo: %w",
+			fmt.Errorf("no available nodes: %w",
+				fmt.Errorf("no healthy nodes available: %w",
+						errors.Join(nodes.ErrEvictionBusy, nodes.ErrNoAvailableNodes))))
+
+		code := applyNoAvailableNodes(wrapped, http.StatusInternalServerError)
+		if code != http.StatusServiceUnavailable {
+			t.Fatalf("expected 503, got %d", code)
+		}
+	})
+
+	t.Run("maps selector-mismatch chain to 503", func(t *testing.T) {
+		wrapped := fmt.Errorf("routing model bar: %w",
+			fmt.Errorf("no available nodes: %w",
+				fmt.Errorf("no healthy nodes match selector for model bar: {\"gpu.vendor\":\"tpu\"}: %w",
+					nodes.ErrNoAvailableNodes)))
+
+		code := applyNoAvailableNodes(wrapped, http.StatusInternalServerError)
+		if code != http.StatusServiceUnavailable {
+			t.Fatalf("expected 503, got %d", code)
+		}
+	})
+
+	t.Run("passes through unrelated errors unchanged", func(t *testing.T) {
+		code := applyNoAvailableNodes(errors.New("database timeout"), http.StatusInternalServerError)
+		if code != http.StatusInternalServerError {
+			t.Fatalf("expected 500, got %d", code)
+		}
+	})
+}

core/http/app.go

@@ -88,6 +88,19 @@ func applyBackendAdmission(err error, code int, c echo.Context) int {
 	return http.StatusTooManyRequests
 }
 
+// applyNoAvailableNodes maps scheduler "no available nodes" errors to 503.
+// When the cluster has no healthy node to serve a model — all are full, a
+// node selector excludes every candidate, or eviction could not free a slot —
+// the request is retryable, not a server bug. Without this the error fell
+// through to 500, which tells clients something is broken when they just
+// need to wait for a node.
+func applyNoAvailableNodes(err error, code int) int {
+	if errors.Is(err, nodes.ErrNoAvailableNodes) {
+		return http.StatusServiceUnavailable
+	}
+	return code
+}
+
 // respondModelLoading answers a request whose model is still cold-loading with
 // 503, a Retry-After header and the live `loading` object, reporting true when
 // it handled the error.
@@ -208,6 +221,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 			}
 			code = applyModelLoadCooldown(err, code, c)
 			code = applyBackendAdmission(err, code, c)
+			code = applyNoAvailableNodes(err, code)
 
 			// Handle 404 errors: serve React SPA for HTML requests, JSON otherwise
 			if code == http.StatusNotFound {
@@ -245,6 +259,7 @@ func API(application *application.Application) (*echo.Echo, error) {
 			// Opaque errors deliberately withhold the body, so a still-loading
 			// model gets the status and Retry-After but no progress detail.
 			code = applyModelLoading(err, code, c)
+			code = applyNoAvailableNodes(err, code)
 			c.NoContent(code)
 		}
 	}

core/services/nodes/router.go

@@ -955,7 +955,7 @@ func (r *SmartRouter) resolveSelectorCandidates(ctx context.Context, modelID str
 		return nil, fmt.Errorf("looking up nodes for selector %s: %w", sched.NodeSelector, err)
 	}
 	if len(candidates) == 0 {
-		return nil, fmt.Errorf("no healthy nodes match selector for model %s: %s", modelID, sched.NodeSelector)
+		return nil, fmt.Errorf("no healthy nodes match selector for model %s: %s: %w", modelID, sched.NodeSelector, ErrNoAvailableNodes)
 	}
 	return extractNodeIDs(candidates), nil
 }
@@ -1167,9 +1167,9 @@ func (r *SmartRouter) scheduleNewModel(ctx context.Context, backendType, modelID
 		evictedNode, evictErr := r.evictLRUAndFreeNodeFrom(ctx, candidateNodeIDs)
 		if evictErr != nil {
 			if errors.Is(evictErr, ErrEvictionBusy) {
-				return nil, "", 0, fmt.Errorf("no healthy nodes available: %w", evictErr)
+				return nil, "", 0, fmt.Errorf("no healthy nodes available: %w", errors.Join(evictErr, ErrNoAvailableNodes))
 			}
-			return nil, "", 0, fmt.Errorf("no healthy nodes available and eviction failed: %w", evictErr)
+			return nil, "", 0, fmt.Errorf("no healthy nodes available and eviction failed: %w", errors.Join(evictErr, ErrNoAvailableNodes))
 		}
 		node = evictedNode
 	}
@@ -2059,6 +2059,13 @@ func (r *SmartRouter) EvictLRU(ctx context.Context, nodeID string) (string, erro
 // and none can be evicted to make room.
 var ErrEvictionBusy = errors.New("all models busy, cannot evict")
 
+// ErrNoAvailableNodes is returned when the scheduler cannot find any healthy
+// node to serve a model — all nodes are full and eviction cannot free a slot,
+// or a node selector excludes every candidate. The HTTP layer maps this to
+// 503 so clients treat it as a transient condition rather than a server bug
+// (which is what 500 would imply).
+var ErrNoAvailableNodes = errors.New("no available nodes")
+
 // evictLRUAndFreeNode finds the globally least-recently-used model with zero in-flight,
 // unloads it, and returns its node for reuse. If all models are busy, retries briefly.
 //

core/services/nodes/router_test.go

@@ -843,6 +843,26 @@ var _ = Describe("SmartRouter", func() {
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no available nodes"))
 		})
+
+		It("wraps ErrNoAvailableNodes when all nodes are full and eviction cannot help", func() {
+			// gorm.ErrRecordNotFound is the registry's verdict that no node
+			// matches — the scheduler then falls through to eviction. With
+			// DB nil, eviction returns ErrEvictionBusy, and the scheduler
+			// wraps the error with ErrNoAvailableNodes so the HTTP layer can
+			// map it to 503 instead of 500.
+			reg.findIdleErr = errors.New("no idle")
+			reg.findLeastLoadedErr = gorm.ErrRecordNotFound
+
+			router := NewSmartRouter(reg, SmartRouterOptions{
+				Unloader:      unloader,
+				ClientFactory: factory,
+			})
+
+			_, err := router.Route(context.Background(), "m5", "models/m5.gguf", "llama-cpp", "", nil, false)
+			Expect(err).To(HaveOccurred())
+			Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue())
+			Expect(errors.Is(err, ErrEvictionBusy)).To(BeTrue())
+		})
 	})
 
 	Describe("UnloadModel (mock-based)", func() {
@@ -970,6 +990,7 @@ var _ = Describe("SmartRouter", func() {
 			_, err := router.Route(context.Background(), "aliased-model", "models/aliased.gguf", "llama-cpp", "", nil, false)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no healthy nodes match selector"))
+			Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue())
 		})
 
 		It("returns error when no nodes match selector", func() {
@@ -988,6 +1009,7 @@ var _ = Describe("SmartRouter", func() {
 			_, err := router.Route(context.Background(), "no-match-model", "models/nomatch.gguf", "llama-cpp", "", nil, false)
 			Expect(err).To(HaveOccurred())
 			Expect(err.Error()).To(ContainSubstring("no healthy nodes match selector"))
+			Expect(errors.Is(err, ErrNoAvailableNodes)).To(BeTrue())
 		})
 
 		It("uses regular methods when model has no scheduling config", func() {

docs/content/reference/api-errors.md

@@ -90,6 +90,7 @@ The `/v1/responses` endpoint returns errors with this structure:
 | 422  | Unprocessable Entity     | Validation failed (e.g., invalid parameter range)      |
 | 429  | Too Many Requests        | All backends are saturated (per-model `max_concurrent` or process-wide `--max-concurrent-backend-requests` ceiling reached). Includes a `Retry-After` header and `type: "rate_limit_error"` so OpenAI-compatible clients and harnesses back off automatically |
 | 500  | Internal Server Error    | Backend inference failure, unexpected server errors    |
+| 503  | Service Unavailable       | No healthy node available to serve the model (cluster is full, eviction cannot free a slot, or a `node_selector` excludes all candidates). Also used during model-load cooldown and while a model is still cold-loading. Retryable |
 
 ## Global Error Handling
 

docs/content/reference/runtime-errors.md

@@ -21,6 +21,7 @@ The left column is the literal string as it appears in the LocalAI server log (o
 | `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). When a local backend has already exited, the error includes its exit code and last stderr line. | Use the included stderr diagnostic when present; otherwise check the log lines just above. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. |
 | `failed to load model: ...` | Returned by the load endpoints and several feature paths (voice, realtime, audio transform) when the model config could not be resolved or the backend load failed. | Confirm the model name exists (`local-ai models list`) and its YAML is valid. The trailing text carries the specific reason. |
 | HTTP `503` with a `Retry-After` header, after a load failed | Model-load failure cooldown. After a model fails to load, LocalAI refuses new load attempts for that model for a short window so a client that keeps polling a broken model does not respawn a crashing backend on every request. The window starts at `--model-load-failure-cooldown` (default `10s`) and doubles per consecutive failure up to 5m; it resets on the first success. | Fix the underlying load failure (see the rows above), then wait out the `Retry-After` seconds before retrying, or restart LocalAI to clear the cooldown. Set `--model-load-failure-cooldown 0` (or `LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN=0`) to disable the cooldown entirely. See {{% relref "reference/cli-reference" %}}. |
+| HTTP `503` with `no available nodes` or `no healthy nodes match selector` | The scheduler could not find any healthy node to serve the model. All nodes are full and eviction cannot free a slot, or a `node_selector` in the model's scheduling config excludes every candidate. | Retry after a node becomes available or an in-flight request completes and frees a slot. In a cluster, add nodes or replicas. If a selector is set, confirm at least one healthy node matches it. |
 | HTTP `429` with a `Retry-After` header, under load | Per-model concurrency limit reached. When a model config sets a `MaxConcurrent` limit, extra requests are rejected with `429` and a `Retry-After` (whole seconds, floor 1) instead of queueing. | Retry after the advised delay, raise the model's concurrency limit, or run more replicas. |
 | HTTP `429` when backend inference is saturated | The process-wide `--max-concurrent-backend-requests` backend-execution ceiling is full. This protects inference and in-flight backend-trace memory without blocking UI or administrative endpoints. | Retry after the advised delay, reduce inference concurrency, raise the limit if the host has capacity, or add replicas. |
 | `invalid pitch` (with CUDA) | The prompt exceeded the model's context size. | Reduce the prompt length, or raise the model's context size (`context_size:` in the model YAML). |