From a3d555653c0cbfd73ab59ab15beab80903e96484 Mon Sep 17 00:00:00 2001 From: Aniruddh Krovvidi Date: Wed, 7 Oct 2026 11:28:49 +0400 Subject: [PATCH] fix(model): stop listing a backend that exited on its own (#12497) Assisted-by: Claude:claude-fable-5-1 Signed-off-by: Aniruddh Krovvidi --- pkg/model/process.go | 24 ++++++++++++++++++++++++ pkg/model/process_exit_test.go | 20 ++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/pkg/model/process.go b/pkg/model/process.go index 36bff8c80..ee65de06e 100644 --- a/pkg/model/process.go +++ b/pkg/model/process.go @@ -399,11 +399,35 @@ func (ml *ModelLoader) startProcess(grpcProcess, id string, serverAddress string } runtime.cleanupScratch() close(runtime.diagnosticsDone) + if !intentional { + ml.forgetExitedProcess(id, grpcControlProcess) + } }() return grpcControlProcess, nil } +// forgetExitedProcess drops the model store entry of a backend that exited on +// its own, so it is no longer reported as loaded. It shares the lifecycle lock +// with loading and shutdown and matches the process identity, so a late exit +// notification cannot remove a replacement loaded under the same id. +func (ml *ModelLoader) forgetExitedProcess(id string, p *process.Process) { + release := ml.operations.acquire(id, true) + defer release() + ml.mu.Lock() + store := ml.store + hooks := append([]ModelUnloadHook(nil), ml.onUnloadHooks...) + ml.mu.Unlock() + if m, ok := store.Get(id); !ok || m.Process() != p { + return + } + for _, hook := range hooks { + hook(id) + } + store.Delete(id) + ml.cleanupProcessRuntime(p) +} + func (ml *ModelLoader) cleanupProcessRuntime(process *process.Process) { if process == nil { return diff --git a/pkg/model/process_exit_test.go b/pkg/model/process_exit_test.go index 1f0bd9093..1b01c38f2 100644 --- a/pkg/model/process_exit_test.go +++ b/pkg/model/process_exit_test.go @@ -43,4 +43,24 @@ var _ = Describe("backend process exit diagnostics", func() { loader.cleanupProcessRuntime(process) Eventually(process.StateDir()).ShouldNot(BeADirectory()) }) + + It("stops listing a model whose backend exited on its own, but keeps a replacement", func() { + tmpDir := GinkgoT().TempDir() + GinkgoT().Setenv(backendTempDirEnv, filepath.Join(tmpDir, "backend-runtime")) + backendPath := filepath.Join(tmpDir, "crashing-backend") + Expect(os.WriteFile(backendPath, []byte("#!/bin/sh\nsleep 0.3\nexit 42\n"), 0o700)).To(Succeed()) + + loader := NewModelLoader(&system.SystemState{Model: system.Model{ModelsPath: tmpDir}}) + crashed, err := loader.startProcess(backendPath, "crashed", "127.0.0.1:65535", nil) + Expect(err).ToNot(HaveOccurred()) + loader.store.Set("crashed", NewModel("crashed", "127.0.0.1:65535", crashed)) + replaced, err := loader.startProcess(backendPath, "replaced", "127.0.0.1:65534", nil) + Expect(err).ToNot(HaveOccurred()) + loader.store.Set("replaced", NewModel("replaced", "127.0.0.1:65533", nil)) + + Eventually(crashed.Done()).Should(BeClosed()) + Eventually(replaced.Done()).Should(BeClosed()) + Eventually(func() bool { _, ok := loader.store.Get("crashed"); return ok }).Should(BeFalse()) + Consistently(func() bool { _, ok := loader.store.Get("replaced"); return ok }, "300ms").Should(BeTrue()) + }) })