fix(model): surface backend startup exits (#11447)

* fix(model): surface backend startup exits

Preserve the local backend process exit code and bounded stderr diagnostic when the process dies before its gRPC service becomes ready.

Fixes #9050

Assisted-by: Codex:gpt-5

* fix(model): satisfy startup diagnostic checks

Assisted-by: Codex:gpt-5.6 [Codex]

---------

Co-authored-by: localai-org-maint-bot <306269227+localai-org-maint-bot@users.noreply.github.com>

docs/content/reference/runtime-errors.md

@@ -18,7 +18,7 @@ The left column is the literal string as it appears in the LocalAI server log (o
 | `could not load model: ...` | The selected backend started but rejected the model (bad path, corrupt or truncated GGUF, wrong architecture, unsupported quantization). The `...` is the backend's own message. | Read the wrapped backend message. Re-download the model if it is truncated. Confirm the backend matches the model (for example a GGUF needs `llama-cpp`). Run with `DEBUG=true` to see the full backend output. |
 | `could not load model (no success): ...` | The backend replied to the load request but reported failure without a fatal error. | Same as above. The trailing message is the backend's status text; check it for the concrete reason (out of memory, unsupported option, missing file). |
 | `could not load model - all backends returned error: ...` | LocalAI tried every candidate backend for the model and each one failed. Usually the backend for this model type is not installed, or the model file is unusable. | Install the correct backend with `local-ai backends install <backend>` (or from the Backends page). Confirm the model config `backend:` field names an installed backend. Inspect the concatenated per-backend messages for the real cause. |
-| `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). | Check the log lines just above for the backend's stderr. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. |
+| `grpc service not ready` | The backend process was spawned but its gRPC server did not become healthy in time (slow start, crash on startup, or the process died while loading). When a local backend has already exited, the error includes its exit code and last stderr line. | Use the included stderr diagnostic when present; otherwise check the log lines just above. A crash here often means out of memory, a missing shared library, or an incompatible CPU (see `SIGILL`). Increase available RAM/VRAM or pick a smaller quantization. |
 | `failed to load model: ...` | Returned by the load endpoints and several feature paths (voice, realtime, audio transform) when the model config could not be resolved or the backend load failed. | Confirm the model name exists (`local-ai models list`) and its YAML is valid. The trailing text carries the specific reason. |
 | HTTP `503` with a `Retry-After` header, after a load failed | Model-load failure cooldown. After a model fails to load, LocalAI refuses new load attempts for that model for a short window so a client that keeps polling a broken model does not respawn a crashing backend on every request. The window starts at `--model-load-failure-cooldown` (default `10s`) and doubles per consecutive failure up to 5m; it resets on the first success. | Fix the underlying load failure (see the rows above), then wait out the `Retry-After` seconds before retrying, or restart LocalAI to clear the cooldown. Set `--model-load-failure-cooldown 0` (or `LOCALAI_MODEL_LOAD_FAILURE_COOLDOWN=0`) to disable the cooldown entirely. See {{% relref "reference/cli-reference" %}}. |
 | HTTP `503` with a `Retry-After` header, under load | Per-model concurrency limit reached. When a model config sets a `MaxConcurrent` limit, extra requests are rejected with `503` and a `Retry-After` (whole seconds, floor 1) instead of queueing. | Retry after the advised delay, raise the model's concurrency limit, or run more replicas. |

pkg/model/initializers.go

@@ -4,12 +4,14 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"os"
 	"strings"
 	"time"
 
 	grpc "github.com/mudler/LocalAI/pkg/grpc"
 	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	processManager "github.com/mudler/go-processmanager"
 	"github.com/mudler/xlog"
 	"github.com/phayes/freeport"
 	"google.golang.org/protobuf/proto"
@@ -161,8 +163,9 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
 
 	if !ready {
 		xlog.Debug("GRPC Service NOT ready")
+		startupErr := grpcStartupError(client.Process())
 		stopLoadProcess(client, modelID)
-		return nil, fmt.Errorf("grpc service not ready")
+		return nil, startupErr
 	}
 
 	// Clone before setting the per-load fields: o.gRPCOptions is shared by
@@ -194,6 +197,58 @@ func (ml *ModelLoader) spawnGRPCModel(backend, uri string, o *Options, modelID,
 	return client, nil
 }
 
+const startupStderrTailBytes = 4096
+
+func grpcStartupError(process *processManager.Process) error {
+	if process == nil {
+		return errors.New("grpc service not ready")
+	}
+
+	select {
+	case <-process.Done():
+	default:
+		return errors.New("grpc service not ready")
+	}
+
+	exitCode, err := process.ExitCode()
+	if err != nil {
+		return errors.New("grpc service not ready: backend process exited before becoming ready")
+	}
+
+	diagnostic := lastNonEmptyLine(process.StderrPath(), startupStderrTailBytes)
+	if diagnostic == "" {
+		return fmt.Errorf("grpc service not ready: backend process exited with code %s", strings.TrimSpace(exitCode))
+	}
+	return fmt.Errorf("grpc service not ready: backend process exited with code %s: %s", strings.TrimSpace(exitCode), diagnostic)
+}
+
+func lastNonEmptyLine(path string, maxBytes int64) string {
+	// #nosec G304 -- path comes from the process manager for this backend's stderr file.
+	f, err := os.Open(path)
+	if err != nil {
+		return ""
+	}
+	defer func() { _ = f.Close() }()
+
+	info, err := f.Stat()
+	if err != nil {
+		return ""
+	}
+	start := max(info.Size()-maxBytes, 0)
+	if _, err := f.Seek(start, io.SeekStart); err != nil {
+		return ""
+	}
+	contents, err := io.ReadAll(f)
+	if err != nil {
+		return ""
+	}
+	lines := strings.Split(strings.TrimSpace(string(contents)), "\n")
+	if len(lines) == 0 {
+		return ""
+	}
+	return strings.TrimSpace(lines[len(lines)-1])
+}
+
 // stopLoadProcess tears down a backend process whose load did not complete.
 // The stop error is only logged: the load error is what the caller reports.
 func stopLoadProcess(client *Model, modelID string) {

pkg/model/initializers_startup_error_test.go

@@ -0,0 +1,34 @@
+package model
+
+import (
+	"os"
+	"path/filepath"
+
+	pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+	"github.com/mudler/LocalAI/pkg/system"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = Describe("gRPC backend startup errors", func() {
+	It("reports a backend process exit and its stderr instead of only a readiness timeout", func() {
+		tmpDir := GinkgoT().TempDir()
+		backendPath := filepath.Join(tmpDir, "failing-backend")
+		Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\necho 'dyld: Library not loaded: libprotobuf.33.dylib' >&2\nexit 42\n"), 0o700)).To(Succeed())
+
+		loader := NewModelLoader(&system.SystemState{Model: system.Model{ModelsPath: tmpDir}})
+		options := NewOptions(
+			WithGRPCAttempts(2),
+			WithGRPCAttemptsDelay(1),
+			WithLoadGRPCLoadModelOpts(&pb.ModelOptions{}),
+		)
+
+		loaded, err := loader.spawnGRPCModel("failing", backendPath, options, "test-model", "test-model", "test.gguf")
+
+		Expect(loaded).To(BeNil())
+		Expect(err).To(MatchError(And(
+			ContainSubstring("backend process exited with code 42"),
+			ContainSubstring("dyld: Library not loaded: libprotobuf.33.dylib"),
+		)))
+	})
+})