fix(distributed): make backend.stop report what it stopped

backend.stop was the one lifecycle subject a worker never answered. The controller published and returned nil as soon as the local publish succeeded, so a stop that killed nothing, and a stop that failed outright, were indistinguishable from one that worked. The unload endpoint calls model.unload and then StopBackend. Only the first is acknowledged, so the endpoint answered 200 while the backend kept running and held its VRAM, and its own "backend stop failed" branch could never run. The worker logged the failure and nobody saw it. Give the subject a reply. The worker now enumerates the process keys it terminated and reports any per-process error, so StopBackend fails when the stop failed. Resolving to nothing stays a success: stopping a backend that is not running leaves the caller in the state it asked for, and eviction paths stop already-gone models routinely. The empty list is what says nothing matched, and ReportsStoppedProcesses is what makes that emptiness trustworthy, the same way BackendDeleteReply handles it. A worker built before this reply still receives the request and still stops the backend, it only stays silent, so a timeout degrades to the old assumption rather than failing every stop on a fleet mid-upgrade. Only silence degrades: a transport error is still reported, because UnloadRemoteModel skips its registry cleanup for a node it could not reach and needs to keep hearing about that. Assisted-by: Claude:claude-opus-5 golangci-lint
2026-09-11 05:34:29 -04:00 · 2026-09-10 03:04:57 +00:00
14 changed files with 240 additions and 55 deletions
@@ -9,7 +9,7 @@
 # recipe is a make target (not a prepare.sh) so 'make purge && make' is a clean
 # rebuild and so the bump bot can see the pin.

-AUDIO_CPP_VERSION?=efb04233dab73aeee4b2912042a90e7b36329061
+AUDIO_CPP_VERSION?=05e508a70e3600b01454c647cdb122133ba8e64c
 AUDIO_CPP_REPO?=https://github.com/0xShug0/audio.cpp

 CURRENT_MAKEFILE_DIR := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))
@@ -1,5 +1,5 @@

-IK_LLAMA_VERSION?=3bb386eb68ffee0a5dc7db21da0735d594929eeb
+IK_LLAMA_VERSION?=1a2a8604a6c6c6413c06bf9adfc2f64329af4366
 LLAMA_REPO?=https://github.com/ikawrakow/ik_llama.cpp

 CMAKE_ARGS?=
@@ -1,5 +1,5 @@

-LLAMA_VERSION?=df03399b885831b2a1603b3abb0d8c156808e363
+LLAMA_VERSION?=f3f1a8f2760f28325a5ec20c05b171e5b7c83a29
 LLAMA_REPO?=https://github.com/ggerganov/llama.cpp

 CMAKE_ARGS?=
@@ -8,7 +8,7 @@ JOBS?=$(shell nproc --ignore=1)

 # stablediffusion.cpp (ggml)
 STABLEDIFFUSION_GGML_REPO?=https://github.com/leejet/stable-diffusion.cpp
-STABLEDIFFUSION_GGML_VERSION?=b68d58624d227682eb4b95ef8bcf569cd1311eb5
+STABLEDIFFUSION_GGML_VERSION?=d04e8950c1ec8d30248cbe996682b3182fb1adf6

 CMAKE_ARGS+=-DGGML_MAX_NAME=128

@@ -3,8 +3,8 @@
 # on a cu130 host. Pull the cu130-flavoured wheel from vLLM's per-tag index
 # instead — the cublas13 case in install.sh adds --index-strategy=unsafe-best-match
 # so uv consults this index alongside PyPI.
--extra-index-url https://wheels.vllm.ai/0.29.0/cu130
+--extra-index-url https://wheels.vllm.ai/0.28.0/cu130
 # VERSION COUPLING: darwin/Apple-Silicon builds use vllm-metal (see install.sh),
 # which pins this exact vLLM version. Bumping vllm here means coordinating with a
 # vllm-metal release that supports the new version, or macOS/Metal builds break.
-vllm==0.29.0
+vllm==0.28.0
@@ -24,7 +24,7 @@
        "@modelcontextprotocol/sdk": "^1.30.0",
        "dompurify": "^3.4.13",
        "highlight.js": "^11.11.1",
-        "hono": "4.13.5",
+        "hono": "4.12.34",
        "i18next": "^26.0.8",
        "i18next-browser-languagedetector": "^8.2.1",
        "i18next-http-backend": "^3.0.6",
@@ -771,9 +771,9 @@
      }
    },
    "node_modules/@istanbuljs/load-nyc-config/node_modules/js-yaml": {
-      "version": "3.15.2",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz",
-      "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==",
+      "version": "3.14.2",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz",
+      "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==",
      "dev": true,
      "license": "MIT",
      "dependencies": {
@@ -3467,9 +3467,9 @@
      }
    },
    "node_modules/hono": {
-      "version": "4.13.5",
-      "resolved": "https://registry.npmjs.org/hono/-/hono-4.13.5.tgz",
-      "integrity": "sha512-O6+/eCYRkzzzy0rPWwKLiGBR1nFuUPZynnwjxN1MBA62NNqbT0wQEzQyK2gSO5yDIDB336sXQleAhOHrzlYyKw==",
+      "version": "4.12.34",
+      "resolved": "https://registry.npmjs.org/hono/-/hono-4.12.34.tgz",
+      "integrity": "sha512-GqXJqY/xJkJmuloTrnV1ZEXG3fqte+VjkUqoRNZXcrUidiUOP4fMSIHHY4tsqZBK++kVyWmt/AAfSUuy57/eSA==",
      "license": "MIT",
      "engines": {
        "node": ">=16.9.0"
@@ -4593,21 +4593,10 @@
      "license": "MIT"
    },
    "node_modules/js-yaml": {
-      "version": "4.3.2",
-      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.3.2.tgz",
-      "integrity": "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA==",
+      "version": "4.1.1",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.1.tgz",
+      "integrity": "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA==",
      "dev": true,
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/puzrin"
-        },
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/nodeca"
-        }
-      ],
-      "license": "MIT",
      "dependencies": {
        "argparse": "^2.0.1"
      },
@@ -19,7 +19,7 @@
    "coverage:report": "nyc report"
  },
  "overrides": {
-    "hono": "4.13.5",
+    "hono": "4.12.34",
    "ip-address": "10.3.1",
    "path-to-regexp": "^8.4.0"
  },
@@ -40,7 +40,7 @@
    "@modelcontextprotocol/sdk": "^1.30.0",
    "dompurify": "^3.4.13",
    "highlight.js": "^11.11.1",
-    "hono": "4.13.5",
+    "hono": "4.12.34",
    "i18next": "^26.0.8",
    "i18next-browser-languagedetector": "^8.2.1",
    "i18next-http-backend": "^3.0.6",
@@ -287,11 +287,39 @@ type BackendStopRequest struct {
 	Force   bool   `json:"force,omitempty"`
 }

+// BackendStopReply is the worker's answer to a backend.stop request.
+//
+// backend.stop had no reply until this type existed. The controller published
+// and returned success as soon as the local publish succeeded, so a stop that
+// killed nothing, and a stop that failed outright, both looked identical to a
+// stop that worked. An operator calling the unload endpoint got HTTP 200 while
+// the backend kept running and holding its VRAM.
+type BackendStopReply struct {
+	Success bool   `json:"success"`
+	Error   string `json:"error,omitempty"`
+
+	// StoppedProcessKeys names every `modelID#replica` process the worker
+	// terminated while serving this request.
+	StoppedProcessKeys []string `json:"stopped_process_keys,omitempty"`
+
+	// ReportsStoppedProcesses distinguishes "this worker enumerates what it
+	// stopped and stopped nothing" from "this worker predates the field", the
+	// same way BackendDeleteReply does. Both send an empty list and only the
+	// first is authoritative, so a controller that cannot tell them apart would
+	// read silence as a completed stop — the exact conclusion this reply exists
+	// to prevent.
+	ReportsStoppedProcesses bool `json:"reports_stopped_processes,omitempty"`
+}
+
 // SubjectNodeBackendStop tells a worker node to stop its gRPC backend process.
 // Equivalent to the local deleteProcess(). The node will:
 // 1. Best-effort bounded Free() via gRPC (unless Force is true)
 // 2. Kill the backend process
 // 3. Can be restarted via another backend.start event.
+//
+// Request-reply, answered with a BackendStopReply. A worker that predates that
+// reply never answers, so the controller must treat a timeout as "unconfirmed"
+// rather than "failed" — see RemoteUnloaderAdapter.stopBackend.
 func SubjectNodeBackendStop(nodeID string) string {
 	return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.stop"
 }
@@ -417,6 +417,15 @@ func (a *RemoteUnloaderAdapter) ListRunningModels(nodeID string) (*messaging.Mod
 		a.nats, subject, messaging.ModelsRunningRequest{}, 10*time.Second)
 }

+// backendStopAckTimeout bounds the wait for a worker's backend.stop reply.
+//
+// A single stop is bounded by the worker's 5s best-effort Free plus the kill,
+// so this leaves comfortable headroom. It is also the stall a worker that
+// predates the reply imposes on every stop, which is why it is not generous:
+// such a worker performs the stop and simply never answers, so the controller
+// waits out the full budget before falling back to the old assumption.
+const backendStopAckTimeout = 15 * time.Second
+
 // StopBackend tells a worker node to stop a specific gRPC backend process.
 // If backend is empty, the worker stops ALL backends.
 // The node stays registered and can receive another InstallBackend later.
@@ -424,12 +433,55 @@ func (a *RemoteUnloaderAdapter) StopBackend(nodeID, backend string) error {
 	return a.stopBackend(nodeID, backend, false)
 }

+// stopBackend asks a worker to stop a backend and waits for it to say what it
+// did.
+//
+// This was a bare Publish, which returned nil as soon as the local publish
+// succeeded and so reported success for a stop that killed nothing or failed
+// outright. The unload endpoint answered HTTP 200 while the backend kept
+// running and holding its VRAM, and the endpoint's own "backend stop failed"
+// branch was unreachable.
+//
+// A silent worker is NOT treated as a failure. A worker that predates
+// BackendStopReply still receives the request and still stops the backend; it
+// only lacks the reply. Failing here would break every stop against a fleet
+// that has not been upgraded yet, so a timeout degrades to the previous
+// assumption and says so in the log.
+//
+// Only silence degrades. A transport error — the connection is closed, nothing
+// subscribes to this node's subject at all — is reported, because under the
+// old Publish it was reported too, and callers such as UnloadRemoteModel rely
+// on that to skip the registry cleanup for a node they could not reach.
 func (a *RemoteUnloaderAdapter) stopBackend(nodeID, backend string, force bool) error {
 	subject := messaging.SubjectNodeBackendStop(nodeID)
-	if backend == "" && !force {
-		return a.nats.Publish(subject, nil)
+	req := messaging.BackendStopRequest{Backend: backend, Force: force}
+
+	reply, err := messaging.RequestJSON[messaging.BackendStopRequest, messaging.BackendStopReply](
+		a.nats, subject, req, backendStopAckTimeout)
+	if err != nil {
+		if errors.Is(err, nats.ErrTimeout) {
+			xlog.Warn("Worker did not acknowledge backend.stop; assuming an older worker delivered it",
+				"nodeID", nodeID, "backend", backend, "force", force)
+			return nil
+		}
+		return fmt.Errorf("backend.stop on node %s: %w", nodeID, err)
 	}
-	return a.nats.Publish(subject, messaging.BackendStopRequest{Backend: backend, Force: force})
+	if !reply.Success {
+		return fmt.Errorf("backend.stop on node %s: %s", nodeID, reply.Error)
+	}
+	// An empty list from a worker that enumerates what it stopped is the answer
+	// to "was anything actually running under that name" — and the answer is
+	// no. That is not an error: stopping a backend that is not running leaves
+	// the caller in the state it asked for. It is worth saying out loud,
+	// because a caller that expected to reclaim VRAM did not.
+	if reply.ReportsStoppedProcesses && len(reply.StoppedProcessKeys) == 0 {
+		xlog.Warn("backend.stop matched no running process on the worker",
+			"nodeID", nodeID, "backend", backend)
+		return nil
+	}
+	xlog.Info("Worker stopped backend processes",
+		"nodeID", nodeID, "backend", backend, "stopped", reply.StoppedProcessKeys)
+	return nil
 }

 // DeleteBackend tells a worker node to delete a backend (stop + remove files).
@@ -124,6 +124,12 @@ type fakeSubscription struct{}

 func (f *fakeSubscription) Unsubscribe() error { return nil }

+func mustJSON(v any) []byte {
+	data, err := json.Marshal(v)
+	Expect(err).ToNot(HaveOccurred())
+	return data
+}
+
 // --- Tests ---

 var _ = Describe("RemoteUnloaderAdapter", func() {
@@ -136,6 +142,14 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 	BeforeEach(func() {
 		locator = &fakeModelLocator{}
 		mc = &fakeMessagingClient{}
+		// backend.stop is request-reply, so the default fake must answer the
+		// way a current worker does. Specs that care about the reply override
+		// requestReply themselves.
+		mc.requestReply = mustJSON(messaging.BackendStopReply{
+			Success:                 true,
+			StoppedProcessKeys:      []string{"llama#0"},
+			ReportsStoppedProcesses: true,
+		})
 		adapter = NewRemoteUnloaderAdapter(locator, mc, 3*time.Minute, 15*time.Minute)
 	})

@@ -179,7 +193,7 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 			// tests/e2e/distributed/node_lifecycle_test.go — keep them in step.
 			locator.nodes = nil
 			Expect(adapter.UnloadRemoteModel("my-model")).To(Succeed())
-			Expect(mc.published).To(BeEmpty())
+			Expect(mc.requestCalls).To(BeEmpty())
 		})

 		It("broadcasts to all nodes with model", func() {
@@ -189,10 +203,10 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 			}
 			Expect(adapter.UnloadRemoteModel("llama")).To(Succeed())

-			// Should have published a StopBackend for each node.
-			Expect(mc.published).To(HaveLen(2))
-			Expect(mc.published[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1")))
-			Expect(mc.published[1].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-2")))
+			// Should have asked each node to stop the backend.
+			Expect(mc.requestCalls).To(HaveLen(2))
+			Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1")))
+			Expect(mc.requestCalls[1].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-2")))

 			// Should have removed the model from each node in the registry.
 			Expect(locator.removedPairs).To(HaveLen(2))
@@ -205,7 +219,7 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 				{ID: "node-fail", Name: "worker-fail"},
 				{ID: "node-ok", Name: "worker-ok"},
 			}
-			// Use a messaging client that fails the first Publish call only.
+			// Use a messaging client that fails the first Request call only.
 			failOnce := &failOnceMessagingClient{inner: mc, failOn: 0}
 			adapter = NewRemoteUnloaderAdapter(locator, failOnce, 3*time.Minute, 15*time.Minute)

@@ -223,25 +237,70 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 			Expect(adapter.UnloadRemoteModelContext(context.Background(), "llama", true)).To(Succeed())

 			var payload messaging.BackendStopRequest
-			Expect(json.Unmarshal(mc.published[0].Data, &payload)).To(Succeed())
+			Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed())
 			Expect(payload).To(Equal(messaging.BackendStopRequest{Backend: "llama", Force: true}))
 		})
 	})

 	Describe("StopBackend", func() {
-		It("with empty backend publishes nil payload", func() {
+		It("with empty backend asks the worker to stop everything", func() {
 			Expect(adapter.StopBackend("node-1", "")).To(Succeed())
-			Expect(mc.published).To(HaveLen(1))
-			Expect(mc.published[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1")))
-			Expect(mc.published[0].Data).To(BeNil())
+			Expect(mc.requestCalls).To(HaveLen(1))
+			Expect(mc.requestCalls[0].Subject).To(Equal(messaging.SubjectNodeBackendStop("node-1")))
+
+			// An empty Backend is the wire signal for "stop all"; the worker's
+			// decodeBackendStopRequest reads it the same way it read the bare
+			// nil payload this replaced.
+			var payload messaging.BackendStopRequest
+			Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed())
+			Expect(payload.Backend).To(BeEmpty())
 		})

-		It("with backend name publishes JSON", func() {
+		// The bug this reply exists for: the worker could not stop what was
+		// asked, and the caller was told everything was fine.
+		It("reports a stop the worker could not carry out", func() {
+			mc.requestReply = mustJSON(messaging.BackendStopReply{
+				Success:                 false,
+				Error:                   "llama#0: process refused to die",
+				ReportsStoppedProcesses: true,
+			})
+			err := adapter.StopBackend("node-1", "llama-backend")
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("process refused to die"))
+		})
+
+		// Nothing running under that name is the state the caller asked for, so
+		// it stays a success — eviction and cleanup paths stop models that are
+		// already gone all the time.
+		It("succeeds when the worker matched no running process", func() {
+			mc.requestReply = mustJSON(messaging.BackendStopReply{
+				Success:                 true,
+				ReportsStoppedProcesses: true,
+			})
 			Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed())
-			Expect(mc.published).To(HaveLen(1))
+		})
+
+		// A worker built before BackendStopReply performs the stop and never
+		// answers. Failing here would break every stop on a fleet mid-upgrade.
+		It("assumes delivery when an older worker never answers", func() {
+			mc.requestErr = nats.ErrTimeout
+			Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed())
+		})
+
+		// A closed connection is not an old worker, and callers depend on
+		// hearing about it: UnloadRemoteModel skips the registry cleanup for a
+		// node it could not reach.
+		It("reports a transport failure rather than assuming delivery", func() {
+			mc.requestErr = nats.ErrConnectionClosed
+			Expect(adapter.StopBackend("node-1", "llama-backend")).To(HaveOccurred())
+		})
+
+		It("with backend name sends JSON", func() {
+			Expect(adapter.StopBackend("node-1", "llama-backend")).To(Succeed())
+			Expect(mc.requestCalls).To(HaveLen(1))

 			var payload messaging.BackendStopRequest
-			Expect(json.Unmarshal(mc.published[0].Data, &payload)).To(Succeed())
+			Expect(json.Unmarshal(mc.requestCalls[0].Data, &payload)).To(Succeed())
 			Expect(payload.Backend).To(Equal("llama-backend"))
 			Expect(payload.Force).To(BeFalse())
 		})
@@ -335,6 +394,13 @@ func (f *failOnceMessagingClient) SubscribeReply(subject string, handler func(da
 }

 func (f *failOnceMessagingClient) Request(subject string, data []byte, timeout time.Duration) ([]byte, error) {
+	f.mu.Lock()
+	idx := f.callIdx
+	f.callIdx++
+	f.mu.Unlock()
+	if idx == f.failOn {
+		return nil, fmt.Errorf("simulated failure")
+	}
 	return f.inner.Request(subject, data, timeout)
 }

@@ -7,6 +7,7 @@ import (
 	"maps"
 	"net"
 	"slices"
+	"strings"
 	"syscall"

 	"github.com/mudler/LocalAI/core/gallery"
@@ -27,7 +28,7 @@ func (s *backendSupervisor) subscribeLifecycleEvents() error {
 	if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendUpgrade(s.nodeID), s.handleBackendUpgrade); err != nil {
 		return fmt.Errorf("subscribing to backend upgrade events: %w", err)
 	}
-	if _, err := s.nats.Subscribe(messaging.SubjectNodeBackendStop(s.nodeID), s.handleBackendStop); err != nil {
+	if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendStop(s.nodeID), s.handleBackendStop); err != nil {
 		return fmt.Errorf("subscribing to backend stop events: %w", err)
 	}
 	if _, err := s.nats.SubscribeReply(messaging.SubjectNodeBackendDelete(s.nodeID), s.handleBackendDelete); err != nil {
@@ -154,27 +155,58 @@ func (s *backendSupervisor) handleBackendUpgrade(data []byte, reply func([]byte)
 }

 // handleBackendStop is the NATS callback for backend.stop — stop a specific
-// backend process (fire-and-forget, no reply expected).
-func (s *backendSupervisor) handleBackendStop(data []byte) {
+// backend process and report what it terminated.
+//
+// The reply is what lets the controller tell a stop that worked from one that
+// matched nothing or failed. Callers that publish without a reply subject (an
+// older controller) still work: SubscribeReply drops the response.
+func (s *backendSupervisor) handleBackendStop(data []byte, reply func([]byte)) {
 	req, stopAll, err := decodeBackendStopRequest(data)
 	if err != nil {
 		xlog.Error("Ignoring malformed NATS backend.stop event", "error", err)
+		replyJSON(reply, messaging.BackendStopReply{
+			Error:                   fmt.Sprintf("invalid request: %v", err),
+			ReportsStoppedProcesses: true,
+		})
 		return
 	}
 	if stopAll {
 		xlog.Info("Received NATS backend.stop event (all)", "force", req.Force)
-		s.stopAllBackends(req.Force)
+		stopped := s.stopAllBackends(req.Force)
+		replyJSON(reply, messaging.BackendStopReply{
+			Success:                 true,
+			StoppedProcessKeys:      stopped,
+			ReportsStoppedProcesses: true,
+		})
 		return
 	}
 	xlog.Info("Received NATS backend.stop event", "backend", req.Backend, "force", req.Force)
 	// The identifier may be a backend name, a model name, or an exact
 	// modelID#replica key depending on the publisher; resolveStopTargets
 	// handles all three. stopBackend alone resolves only the model meanings.
+	var stopped []string
+	var failures []string
 	for _, key := range s.resolveStopTargets(req.Backend) {
 		if err := s.stopBackendExact(key, req.Force); err != nil {
 			xlog.Error("Failed to stop backend process", "backend", req.Backend, "processKey", key, "error", err)
+			failures = append(failures, fmt.Sprintf("%s: %v", key, err))
+			continue
 		}
+		stopped = append(stopped, key)
 	}
+	// Resolving to nothing is reported as success with an empty list, not as a
+	// failure: stopping a backend that is not running is the state the caller
+	// asked for. The empty list is what tells the caller nothing matched, and
+	// ReportsStoppedProcesses is what makes that emptiness trustworthy.
+	res := messaging.BackendStopReply{
+		Success:                 len(failures) == 0,
+		StoppedProcessKeys:      stopped,
+		ReportsStoppedProcesses: true,
+	}
+	if len(failures) > 0 {
+		res.Error = strings.Join(failures, "; ")
+	}
+	replyJSON(reply, res)
 }

 func decodeBackendStopRequest(data []byte) (messaging.BackendStopRequest, bool, error) {
@@ -966,8 +966,10 @@ func (s *backendSupervisor) cleanupProcessRuntime(proc *process.Process) {
 	}
 }

-// stopAllBackends stops all running backend processes.
-func (s *backendSupervisor) stopAllBackends(force bool) {
+// stopAllBackends stops all running backend processes and returns the process
+// keys it attempted, so a caller answering a backend.stop request can report
+// what it acted on.
+func (s *backendSupervisor) stopAllBackends(force bool) []string {
 	s.mu.Lock()
 	backends := slices.Collect(maps.Keys(s.processes))
 	s.mu.Unlock()
@@ -975,6 +977,7 @@ func (s *backendSupervisor) stopAllBackends(force bool) {
 	for _, b := range backends {
 		s.stopBackend(b, force)
 	}
+	return backends
 }

 // isRunning returns whether at least one backend process matching the given
@@ -568,6 +568,21 @@ The responses from `GET /api/node/:id/models` and `GET /api/nodes/:id/models` in

 `model.unload` releases model memory inside a running backend. It does not replace the exact process stop that configuration cleanup requires. The `backend.stop` operation remains an administrative backend operation.

+#### `backend.stop` is acknowledged
+
+`backend.stop` is request-reply. The worker answers with what it terminated, so the controller can tell a stop that worked from one that matched nothing or failed outright.
+
+This matters for `POST /api/nodes/:id/models/unload`, which stops the backend after unloading the model. The stop used to be fire-and-forget, so the endpoint answered `200` as soon as the message left the frontend — including when the backend was still running and still holding its VRAM. It now returns an error when the worker reports that the stop failed.
+
+Two outcomes are deliberately **not** errors:
+
+- **Nothing matched.** The worker reports an empty stopped-process list, logged as `backend.stop matched no running process`. Stopping a backend that is not running leaves the caller in the state it asked for, and eviction and cleanup paths stop already-gone models routinely.
+- **No answer.** A worker built before this reply performs the stop and never responds. The controller waits 15 seconds, logs `Worker did not acknowledge backend.stop`, and assumes delivery, so a fleet mid-upgrade keeps working. A transport failure is reported rather than assumed.
+
+{{% notice note %}}
+On a mixed fleet, every stop against a worker that predates the reply costs the full 15-second wait before falling back. Upgrading the workers removes the delay.
+{{% /notice %}}
+
 ### Per-node VRAM budget

 Each worker advertises its detected VRAM, and the SmartRouter uses that number when picking a node with enough free memory. You can cap the VRAM a node offers for placement so it never gets scheduled beyond a chosen limit, leaving headroom for other workloads on that machine.
@@ -49,11 +49,11 @@
      use_tokenizer_template: true
  files:
    - filename: llama-cpp/models/Qwopus3.8-27B-Flash-MTP-Q4_K_M/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf
+      sha256: 2b9a335bd33bb977d3127062061c12cc94f37ceef1850d372317fbf4ee89d9c5
      uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/Qwopus3.8-27B-Flash-MTP-Q4_K_M.gguf
-      sha256: 1b76ad1596d6afc0b6e38a2b45b404462920739bd3087960b7a18a74ca938a74
    - filename: llama-cpp/mmproj/Qwopus3.8-27B-Flash-MTP-Q4_K_M/mmproj-F32.gguf
+      sha256: be3f444ecaedf084fd19399ba18cefcbbe3789bb81f4ad5ed67c91a1a3f48a67
      uri: https://huggingface.co/Jackrong/Qwopus3.8-27B-Flash-GGUF/resolve/main/mmproj-F32.gguf
-      sha256: 52e6818e4d18eea010c50e5245eaa10a8cc3dcc30efea4ff60cbad8abf5669e1
 - name: "qwen3.8-27b"
  url: "github:mudler/LocalAI/gallery/virtual.yaml@master"
  urls:
@@ -2511,7 +2511,7 @@
  files:
    - filename: llama-cpp/models/tiel-coder-35b-a3b-mtp/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf
      uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/Tiel-Coder-35B-A3B-MTP-UD-Q4_K_XL.gguf
-      sha256: 54f46c4ce544c225122b0f066c2336f10404be7bc53b0cc94d1dbbc5e826bdc1
+      sha256: 10960d1d6477b08ed36a0e542e571b473022023c25b8315b0cf8c33c57e98ccd
    - filename: llama-cpp/mmproj/tiel-coder-35b-a3b-mtp/mmproj-BF16.gguf
      uri: huggingface://peculiar-ragdoll/Tiel-Coder-35B-A3B-GGUF-MTP/mmproj-BF16.gguf
      sha256: d9ce31026d1cb1f3f8d5152e2e2a014d9d2b302b6c93a7dc07bb0a0487f52837