From f2cdc06781ece0627614dd2ad55f0a5dc95872d3 Mon Sep 17 00:00:00 2001 From: localai-org-maint-bot Date: Tue, 11 Aug 2026 09:54:29 +0200 Subject: [PATCH] fix(model): surface backend startup exits (#11447) * fix(model): surface backend startup exits Preserve the local backend process exit code and bounded stderr diagnostic when the process dies before its gRPC service becomes ready. Fixes #9050 Assisted-by: Codex:gpt-5 * fix(model): satisfy startup diagnostic checks Assisted-by: Codex:gpt-5.6 [Codex] --------- Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com> --- docs/content/reference/runtime-errors.md | 2 +- pkg/model/initializers.go | 57 +++++++++++++++++++- pkg/model/initializers_startup_error_test.go | 34 ++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 pkg/model/initializers_startup_error_test.go diff --git a/docs/content/reference/runtime-errors.md b/docs/content/reference/runtime-errors.md index 1adb051a4..9122a1356 100644 --- a/docs/content/reference/runtime-errors.md +++ b/docs/content/reference/runtime-errors.md @@ -18,7 +18,7 @@ The left column is the literal string as it appears in the LocalAI server log (o | `could not load model: ...` | The selected backend started but rejected the model (bad path, corrupt or truncated GGUF, wrong architecture, unsupported quantization). The `...` is the backend's own message. | Read the wrapped backend message. Re-download the model if it is truncated. Confirm the backend matches the model (for example a GGUF needs `llama-cpp`). Run with `DEBUG=true` to see the full backend output. | | `could not load model (no success): ...` | The backend replied to the load request but reported failure without a fatal error. | Same as above. The trailing message is the backend's status text; check it for the concrete reason (out of memory, unsupported option, missing file). | | `could not load model - all backends returned error: ...` | LocalAI tried every candidate backend for the model and each one failed. Usually the backend for this model type is not installed, or the model file is unusable. | Install the correct backend with `local-ai backends install ` (or from the Backends page). Confirm the model config `backend:` field names an installed backend. Inspect the concatenated per-backend messages for the real cause. | -| `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). | Check the log lines just above for the backend's stderr. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. | +| `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). When a local backend has already exited, the error includes its exit code and last stderr line. | Use the included stderr diagnostic when present; otherwise check the log lines just above. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. | | `failed to load model: ...` | Returned by the load endpoints and several feature paths (voice, realtime, audio transform) when the model config could not be resolved or the backend load failed. | Confirm the model name exists (`local-ai models list`) and its YAML is valid. The trailing text carries the specific reason. | | HTTP `503` with a `Retry-After` header, after a load failed | Model-load failure cooldown. After a model fails to load, LocalAI refuses new load attempts for that model for a short window so a client that keeps polling a broken model does not respawn a crashing backend on every request. The window starts at `--model-load-failure-cooldown` (default `10s`) and doubles per consecutive failure up to 5m; it resets on the first success. | Fix the underlying load failure (see the rows above), then wait out the `Retry-After` seconds before retrying, or restart LocalAI to clear the cooldown. Set `--model-load-failure-cooldown 0` (or `LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN=0`) to disable the cooldown entirely. See {{% relref "reference/cli-reference" %}}. | | HTTP `503` with a `Retry-After` header, under load | Per-model concurrency limit reached. When a model config sets a `MaxConcurrent` limit, extra requests are rejected with `503` and a `Retry-After` (whole seconds, floor 1) instead of queueing. | Retry after the advised delay, raise the model's concurrency limit, or run more replicas. | diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go index c7f63b5c7..8d36f1e7a 100644 --- a/pkg/model/initializers.go +++ b/pkg/model/initializers.go @@ -4,12 +4,14 @@ import ( "context" "errors" "fmt" + "io" "os" "strings" "time" grpc "github.com/mudler/LocalAI/pkg/grpc" pb "github.com/mudler/LocalAI/pkg/grpc/proto" + processManager "github.com/mudler/go-processmanager" "github.com/mudler/xlog" "github.com/phayes/freeport" "google.golang.org/protobuf/proto" @@ -161,8 +163,9 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID, if !ready { xlog.Debug("GRPC Service NOT ready") + startupErr := grpcStartupError(client.Process()) stopLoadProcess(client, modelID) - return nil, fmt.Errorf("grpc service not ready") + return nil, startupErr } // Clone before setting the per-load fields: o.gRPCOptions is shared by @@ -194,6 +197,58 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID, return client, nil } +const startupStderrTailBytes = 4096 + +func grpcStartupError(process *processManager.Process) error { + if process == nil { + return errors.New("grpc service not ready") + } + + select { + case <-process.Done(): + default: + return errors.New("grpc service not ready") + } + + exitCode, err := process.ExitCode() + if err != nil { + return errors.New("grpc service not ready: backend process exited before becoming ready") + } + + diagnostic := lastNonEmptyLine(process.StderrPath(), startupStderrTailBytes) + if diagnostic == "" { + return fmt.Errorf("grpc service not ready: backend process exited with code %s", strings.TrimSpace(exitCode)) + } + return fmt.Errorf("grpc service not ready: backend process exited with code %s: %s", strings.TrimSpace(exitCode), diagnostic) +} + +func lastNonEmptyLine(path string, maxBytes int64) string { + // #nosec G304 -- path comes from the process manager for this backend's stderr file. + f, err := os.Open(path) + if err != nil { + return "" + } + defer func() { _ = f.Close() }() + + info, err := f.Stat() + if err != nil { + return "" + } + start := max(info.Size()-maxBytes, 0) + if _, err := f.Seek(start, io.SeekStart); err != nil { + return "" + } + contents, err := io.ReadAll(f) + if err != nil { + return "" + } + lines := strings.Split(strings.TrimSpace(string(contents)), "\n") + if len(lines) == 0 { + return "" + } + return strings.TrimSpace(lines[len(lines)-1]) +} + // stopLoadProcess tears down a backend process whose load did not complete. // The stop error is only logged: the load error is what the caller reports. func stopLoadProcess(client *Model, modelID string) { diff --git a/pkg/model/initializers_startup_error_test.go b/pkg/model/initializers_startup_error_test.go new file mode 100644 index 000000000..2d10c8a31 --- /dev/null +++ b/pkg/model/initializers_startup_error_test.go @@ -0,0 +1,34 @@ +package model + +import ( + "os" + "path/filepath" + + pb "github.com/mudler/LocalAI/pkg/grpc/proto" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("gRPC backend startup errors", func() { + It("reports a backend process exit and its stderr instead of only a readiness timeout", func() { + tmpDir := GinkgoT().TempDir() + backendPath := filepath.Join(tmpDir, "failing-backend") + Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\necho 'dyld: Library not loaded: libprotobuf.33.dylib' >&2\nexit 42\n"), 0o700)).To(Succeed()) + + loader := NewModelLoader(&system.SystemState{Model: system.Model{ModelsPath: tmpDir}}) + options := NewOptions( + WithGRPCAttempts(2), + WithGRPCAttemptsDelay(1), + WithLoadGRPCLoadModelOpts(&pb.ModelOptions{}), + ) + + loaded, err := loader.spawnGRPCModel("failing", backendPath, options, "test-model", "test-model", "test.gguf") + + Expect(loaded).To(BeNil()) + Expect(err).To(MatchError(And( + ContainSubstring("backend process exited with code 42"), + ContainSubstring("dyld: Library not loaded: libprotobuf.33.dylib"), + ))) + }) +})