feat(distributed): give agent workers a tunnel of their own

Phase 2 gated agent nodes out of tunnel credentials at the mint site.
That was right while nothing dialled into an agent worker: a credential
would have replaced nothing, and the gate was structural rather than a
second check that could drift. It is wrong now that the frontend needs to
reach an agent worker by RPC.

attachTunnelToken mints for backend and agent nodes and CLEARS for
anything else, through one tunnelEligible predicate rather than two
conditions that can be widened separately. ConnectHandler still never
reads NodeType, so an empty hash is still what refuses an ineligible
node.

An agent worker now starts a loopback control server behind the same
bearer check a backend worker uses, and holds one tunnel whose only
stream tag is http: it runs no backend processes, so the grpc tag has
nothing to route to and is not offered. Its MCP tool, MCP discovery and
backend.stop verbs are served from ONE implementation reached by both the
bus and the tunnel, so a frontend cannot get different bytes depending on
which carrier delivered.

The tunnel is an ADDITION. --nats-url is still required, and agent jobs,
MCP execution, MCP CI jobs and nodes.<id>.backend.stop all still travel
on the bus.

Absence semantics are unchanged. An agent node now has a real
node_connections row whose departure ages past the grace, so the node
type check in HealthMonitor.tunnelDeparted stopped being an optimisation
and became the rule; its comment says so, and the spec that pins it is
shown red under a mutation that deletes the check. The scheduler needed
no change: every placement query already filters node_type = backend, so
an agent node never reaches nodeMayTakeWork.

Shared rules moved to one site each. The request bounds, the POST-only
check and the unknown-path 404 live in workerctl and are called by both
worker packages; the bearer check that guards every extra route is one
function in core/services/nodes used by both server constructors.
workerctl.AllPaths splits into BackendPaths and AgentPaths, with AllPaths
as their deduped union, because a backend worker does not mount the agent
verbs and asserting otherwise would fail a correct worker.

Assisted-by: Claude Opus 5 [claude-code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/cli/agent_worker.go

@@ -16,6 +16,7 @@ import (
 	"github.com/mudler/LocalAI/core/config"
 	mcpTools "github.com/mudler/LocalAI/core/http/endpoints/mcp"
 	"github.com/mudler/LocalAI/core/services/agents"
+	"github.com/mudler/LocalAI/core/services/agentworker"
 	"github.com/mudler/LocalAI/core/services/jobs"
 	mcpRemote "github.com/mudler/LocalAI/core/services/mcp"
 	"github.com/mudler/LocalAI/core/services/messaging"
@@ -31,6 +32,11 @@ import (
 // receives the full agent config and skills in the NATS job payload, so it
 // does not need direct database access.
 //
+// It also holds one tunnel to the frontend, so the frontend can reach its MCP
+// verbs by RPC without the worker opening an inbound port. The tunnel is an
+// ADDITION: --nats-url is still required, and every job, every fan-out event
+// and the node backend.stop subject still travel on the bus.
+//
 // Usage:
 //
 //	localai agent-worker --nats-url nats://... --register-to http://localai:8080
@@ -170,6 +176,48 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 	}
 	defer natsClient.Close()
 
+	// The tunnel, and the loopback control plane behind it.
+	//
+	// ADDED to this worker rather than swapping anything out: every verb below
+	// still arrives on NATS, and will until the tasks that move them land. What
+	// this buys today is that the frontend can reach an agent worker by RPC at
+	// all, on the same carrier and with the same failure vocabulary a backend
+	// worker already uses, without the agent worker opening an inbound port.
+	//
+	// The credential is read through credMgr rather than captured from res,
+	// because every re-registration the manager performs ROTATES it and a
+	// captured value would lock this worker out of its own tunnel at the first
+	// JWT refresh.
+	//
+	// It is started AFTER registration, which is what supplies both the node
+	// identity the dial names and the credential it presents, and BEFORE the
+	// NATS subscriptions, so that a frontend that reaches this worker over the
+	// tunnel finds its verbs mounted rather than a 404 it would read as a
+	// version skew.
+	agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{
+		FrontendURL:  cmd.RegisterTo,
+		NodeID:       nodeID,
+		TunnelToken:  credMgr.TunnelToken,
+		ControlToken: cmd.RegistrationToken,
+		Handlers: agentworker.Config{
+			MCPTool:      serveMCPToolRequest,
+			MCPDiscovery: serveMCPDiscoveryRequest,
+			// The same cleanup the nodes.<id>.backend.stop subscription below
+			// performs, reachable over the tunnel on the path a backend worker
+			// already serves. Both are live: the subject is what the frontend
+			// still publishes on, and it is a later task that removes it.
+			BackendStop: dropMCPSessionsForBackend,
+		},
+	})
+	if err != nil {
+		return fmt.Errorf("starting the agent worker control plane: %w", err)
+	}
+	defer func() {
+		if err := agentCtl.Close(); err != nil {
+			xlog.Warn("Closing the agent worker tunnel failed", "error", err)
+		}
+	}()
+
 	// Create event bridge for publishing results back via NATS
 	eventBridge := agents.NewEventBridge(natsClient, nil, "agent-worker-"+nodeID)
 
@@ -198,16 +246,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 
 	// Subscribe to MCP tool execution requests (load-balanced across workers).
 	// The frontend routes model-level MCP tool calls here via NATS request-reply.
-	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) {
-		handleMCPToolRequest(data, reply)
-	}); err != nil {
+	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers,
+		replyOverNATS(messaging.SubjectMCPToolExecute, serveMCPToolRequest)); err != nil {
 		return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPToolExecute, err)
 	}
 
 	// Subscribe to MCP discovery requests (load-balanced across workers).
-	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers, func(data []byte, reply func([]byte)) {
-		handleMCPDiscoveryRequest(data, reply)
-	}); err != nil {
+	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers,
+		replyOverNATS(messaging.SubjectMCPDiscovery, serveMCPDiscoveryRequest)); err != nil {
 		return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPDiscovery, err)
 	}
 
@@ -229,12 +275,19 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 	// Subscribe to backend stop events to clean up cached MCP sessions.
 	// In the main application this is done via ml.OnModelUnload, but the agent
 	// worker has no model loader — we listen for the NATS stop event instead.
+	//
+	// It runs BESIDE the tunnel route mounted above, not instead of it, and
+	// both call dropMCPSessionsForBackend. The subject is still what the
+	// frontend publishes on; a later task is what moves it. Two carriers, one
+	// implementation, so which one delivered cannot change what happened.
 	if _, err := natsClient.Subscribe(messaging.SubjectNodeBackendStop(nodeID), func(data []byte) {
-		var req struct {
-			Backend string `json:"backend"`
+		var req messaging.BackendStopRequest
+		if err := json.Unmarshal(data, &req); err != nil {
+			xlog.Warn("Agent worker could not decode a backend stop event", "error", err)
+			return
 		}
-		if json.Unmarshal(data, &req) == nil && req.Backend != "" {
-			mcpTools.CloseMCPSessions(req.Backend)
+		if err := dropMCPSessionsForBackend(context.Background(), req); err != nil {
+			xlog.Warn("Agent worker could not drop the MCP sessions of a stopped backend", "error", err)
 		}
 	}); err != nil {
 		return fmt.Errorf("subscribing to %s: %w", messaging.SubjectNodeBackendStop(nodeID), err)
@@ -263,72 +316,106 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 	return runErr
 }
 
-// handleMCPToolRequest handles a NATS request-reply for MCP tool execution.
-// The worker creates/caches MCP sessions from the serialized config and executes the tool.
-func handleMCPToolRequest(data []byte, reply func([]byte)) {
+// The MCP verbs, written ONCE and served on two carriers.
+//
+// The bus subscription and the tunnel's control route both call the same
+// serve* function and both send the same bytes, so a worker reached either way
+// answers identically. Two implementations of one verb is the shape that lets a
+// deployment behave differently depending on which carrier a frontend happened
+// to pick, and there is no version of this migration in which that is
+// acceptable: for the whole of it, both carriers are live at once.
+//
+// The distinction the return type carries: an MCP tool that RAN and failed is
+// this worker's own answer and travels as bytes with an error field set, on a
+// 200. A returned error is this worker failing to serve the verb at all, which
+// becomes a non-2xx over the tunnel and nothing the frontend may act on.
+
+// dropMCPSessionsForBackend closes the MCP sessions this worker cached for a
+// backend that is going away.
+//
+// It is the agent worker's whole implementation of backend.stop, and it is
+// deliberately nothing like the backend worker's, which kills the process and
+// recycles its port. An agent worker runs no backend processes; what it holds
+// are sessions that were created against one.
+//
+// A backend nobody named is a no-op rather than an error. The event carries the
+// name, and a request without one asks this worker to forget nothing in
+// particular; failing it would put a malformed publish into the bucket the
+// frontend reads as a worker that could not be reached.
+func dropMCPSessionsForBackend(_ context.Context, req messaging.BackendStopRequest) error {
+	if req.Backend == "" {
+		return nil
+	}
+	mcpTools.CloseMCPSessions(req.Backend)
+	return nil
+}
+
+// serveMCPToolRequest answers an MCP tool execution request.
+func serveMCPToolRequest(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) {
+	return encodeMCPReply(runMCPTool(ctx, raw))
+}
+
+// runMCPTool creates or reuses the named MCP sessions from the request's config
+// and executes the named tool against them.
+//
+// Every failure inside it is an answer rather than an error, because every one
+// of them is something this worker LEARNED by trying: a config it could not
+// build sessions from, a discovery that failed, a tool that returned an error.
+func runMCPTool(ctx context.Context, raw json.RawMessage) mcpRemote.MCPToolResponse {
 	var req mcpRemote.MCPToolRequest
-	if err := json.Unmarshal(data, &req); err != nil {
-		sendMCPToolReply(reply, "", fmt.Sprintf("unmarshal error: %v", err))
-		return
+	if err := json.Unmarshal(raw, &req); err != nil {
+		return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("unmarshal error: %v", err)}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), config.DefaultMCPToolTimeout)
+	// Bounded here rather than by the caller, so the bus path and the tunnel
+	// path give a stuck MCP server the same budget.
+	ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPToolTimeout)
 	defer cancel()
 
-	// Create/cache named MCP sessions from the provided config
 	namedSessions, err := mcpTools.NamedSessionsFromMCPConfig(req.ModelName, req.RemoteServers, req.StdioServers, nil)
 	if err != nil {
-		sendMCPToolReply(reply, "", fmt.Sprintf("session error: %v", err))
-		return
+		return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("session error: %v", err)}
 	}
 
 	// Discover tools to find the right session
 	tools, err := mcpTools.DiscoverMCPTools(ctx, namedSessions)
 	if err != nil {
-		sendMCPToolReply(reply, "", fmt.Sprintf("discovery error: %v", err))
-		return
+		return mcpRemote.MCPToolResponse{Error: fmt.Sprintf("discovery error: %v", err)}
 	}
 
-	// Execute the tool
 	argsJSON, _ := json.Marshal(req.Arguments)
 	result, err := mcpTools.ExecuteMCPToolCall(ctx, tools, req.ToolName, string(argsJSON))
 	if err != nil {
-		sendMCPToolReply(reply, "", err.Error())
-		return
+		return mcpRemote.MCPToolResponse{Error: err.Error()}
 	}
-
-	sendMCPToolReply(reply, result, "")
+	return mcpRemote.MCPToolResponse{Result: result}
 }
 
-func sendMCPToolReply(reply func([]byte), result, errMsg string) {
-	resp := mcpRemote.MCPToolResponse{Result: result, Error: errMsg}
-	data, _ := json.Marshal(resp)
-	reply(data)
+// serveMCPDiscoveryRequest answers an MCP tool/prompt/resource discovery
+// request.
+func serveMCPDiscoveryRequest(ctx context.Context, raw json.RawMessage) (json.RawMessage, error) {
+	return encodeMCPReply(runMCPDiscovery(ctx, raw))
 }
 
-// handleMCPDiscoveryRequest handles a NATS request-reply for MCP tool/prompt/resource discovery.
-func handleMCPDiscoveryRequest(data []byte, reply func([]byte)) {
+// runMCPDiscovery lists the servers this worker can reach for a model, with
+// their tools, prompts and resources.
+func runMCPDiscovery(ctx context.Context, raw json.RawMessage) mcpRemote.MCPDiscoveryResponse {
 	var req mcpRemote.MCPDiscoveryRequest
-	if err := json.Unmarshal(data, &req); err != nil {
-		sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("unmarshal error: %v", err))
-		return
+	if err := json.Unmarshal(raw, &req); err != nil {
+		return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("unmarshal error: %v", err)}
 	}
 
-	ctx, cancel := context.WithTimeout(context.Background(), config.DefaultMCPDiscoveryTimeout)
+	ctx, cancel := context.WithTimeout(ctx, config.DefaultMCPDiscoveryTimeout)
 	defer cancel()
 
-	// Create/cache named MCP sessions
 	namedSessions, err := mcpTools.NamedSessionsFromMCPConfig(req.ModelName, req.RemoteServers, req.StdioServers, nil)
 	if err != nil {
-		sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("session error: %v", err))
-		return
+		return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("session error: %v", err)}
 	}
 
-	// List servers with their tools/prompts/resources
 	serverInfos, err := mcpTools.ListMCPServers(ctx, namedSessions)
 	if err != nil {
-		sendMCPDiscoveryReply(reply, nil, nil, fmt.Sprintf("list error: %v", err))
-		return
+		return mcpRemote.MCPDiscoveryResponse{Error: fmt.Sprintf("list error: %v", err)}
 	}
 
 	// Also get tool function schemas for the frontend
@@ -342,26 +429,48 @@ func handleMCPDiscoveryRequest(data []byte, reply func([]byte)) {
 		})
 	}
 
-	// Convert server infos
 	var servers []mcpRemote.MCPServerInfo
-	for _, s := range serverInfos {
+	for _, srv := range serverInfos {
 		servers = append(servers, mcpRemote.MCPServerInfo{
-			Name:      s.Name,
-			Type:      s.Type,
-			Tools:     s.Tools,
-			Prompts:   s.Prompts,
-			Resources: s.Resources,
-			Error:     s.Error,
+			Name:      srv.Name,
+			Type:      srv.Type,
+			Tools:     srv.Tools,
+			Prompts:   srv.Prompts,
+			Resources: srv.Resources,
+			Error:     srv.Error,
 		})
 	}
-
-	sendMCPDiscoveryReply(reply, servers, toolDefs, "")
+	return mcpRemote.MCPDiscoveryResponse{Servers: servers, Tools: toolDefs}
 }
 
-func sendMCPDiscoveryReply(reply func([]byte), servers []mcpRemote.MCPServerInfo, tools []mcpRemote.MCPToolDef, errMsg string) {
-	resp := mcpRemote.MCPDiscoveryResponse{Servers: servers, Tools: tools, Error: errMsg}
-	data, _ := json.Marshal(resp)
-	reply(data)
+// encodeMCPReply turns a verb's answer into the bytes both carriers send.
+//
+// A marshalling failure is the one thing here that is NOT an answer: this
+// worker has said nothing about the request, so it is returned as an error and
+// becomes a non-2xx over the tunnel rather than an empty 200.
+func encodeMCPReply(resp any) (json.RawMessage, error) {
+	out, err := json.Marshal(resp)
+	if err != nil {
+		return nil, fmt.Errorf("encoding the reply: %w", err)
+	}
+	return out, nil
+}
+
+// replyOverNATS adapts one of the serve* functions to a NATS request-reply
+// subscription, so the bus carries exactly the bytes the tunnel does.
+func replyOverNATS(subject string, serve func(context.Context, json.RawMessage) (json.RawMessage, error)) func([]byte, func([]byte)) {
+	return func(data []byte, reply func([]byte)) {
+		out, err := serve(context.Background(), data)
+		if err != nil {
+			// Nothing is sent. A requester on the bus reads that as a timeout,
+			// which is the closest the carrier has to "this worker did not
+			// answer"; inventing a reply body here would put a failure to serve
+			// into the bucket reserved for the worker's own verdict.
+			xlog.Error("Agent worker could not serve a bus request", "subject", subject, "error", err)
+			return
+		}
+		reply(out)
+	}
 }
 
 // handleMCPCIJob processes an MCP CI job on the agent worker.

core/cli/agent_worker_test.go

@@ -0,0 +1,100 @@
+package cli
+
+import (
+	"context"
+	"encoding/json"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	mcpRemote "github.com/mudler/LocalAI/core/services/mcp"
+	"github.com/mudler/LocalAI/core/services/messaging"
+)
+
+// The agent worker answers its MCP verbs on two carriers at once: the NATS
+// subject it has always answered, and the control route on the tunnel it now
+// holds. These specs pin the two properties that stops those drifting apart.
+//
+// The first is that there is ONE implementation. A frontend that reaches a
+// worker over the bus and one that reaches the same worker over its tunnel must
+// get the same bytes, because during this migration both are live and which one
+// is used is not a decision anybody makes deliberately.
+//
+// The second is the split between an ANSWER and a FAILURE TO SERVE. A tool that
+// ran and failed is the worker's own verdict and travels inside the reply; a
+// verb this worker could not serve at all is a Go error, which becomes a
+// non-2xx over the tunnel and silence on the bus, and which nothing may read as
+// evidence about anything.
+var _ = Describe("The agent worker's MCP verbs", func() {
+	It("answers a tool request it could not decode, rather than failing to serve it", func() {
+		// The decode happened on this worker and its outcome is something the
+		// worker LEARNED, so it belongs in the reply. Returned as an error it
+		// would become "no route to that worker" at the frontend.
+		raw, err := serveMCPToolRequest(context.Background(), json.RawMessage(`{"tool_name":`))
+		Expect(err).ToNot(HaveOccurred())
+
+		var resp mcpRemote.MCPToolResponse
+		Expect(json.Unmarshal(raw, &resp)).To(Succeed())
+		Expect(resp.Error).To(ContainSubstring("unmarshal error"))
+		Expect(resp.Result).To(BeEmpty())
+	})
+
+	It("answers a discovery request it could not decode the same way", func() {
+		raw, err := serveMCPDiscoveryRequest(context.Background(), json.RawMessage(`not json`))
+		Expect(err).ToNot(HaveOccurred())
+
+		var resp mcpRemote.MCPDiscoveryResponse
+		Expect(json.Unmarshal(raw, &resp)).To(Succeed())
+		Expect(resp.Error).To(ContainSubstring("unmarshal error"))
+	})
+
+	It("puts on the bus exactly the bytes the tunnel route returns", func() {
+		// The one property that keeps the two carriers honest. A second
+		// implementation for the bus is how a deployment ends up behaving
+		// differently depending on which one a frontend happened to pick.
+		request := json.RawMessage(`{"tool_name":`)
+		overTunnel, err := serveMCPToolRequest(context.Background(), request)
+		Expect(err).ToNot(HaveOccurred())
+
+		sent := make(chan []byte, 1)
+		replyOverNATS("mcp.tools.execute", serveMCPToolRequest)(request, func(b []byte) { sent <- b })
+
+		var overBus []byte
+		Eventually(sent).Should(Receive(&overBus))
+		Expect(string(overBus)).To(Equal(string(overTunnel)))
+	})
+
+	It("sends nothing on the bus when the verb could not be served", func() {
+		// A requester reads the silence as a timeout, which is the closest the
+		// bus has to "this worker did not answer". Inventing a reply body would
+		// put a failure to serve into the bucket reserved for the worker's own
+		// verdict, which is the collapse this whole phase exists to prevent.
+		sent := make(chan []byte, 1)
+		failing := func(context.Context, json.RawMessage) (json.RawMessage, error) {
+			return nil, context.DeadlineExceeded
+		}
+
+		replyOverNATS("mcp.tools.execute", failing)(json.RawMessage(`{}`), func(b []byte) { sent <- b })
+
+		Expect(sent).ToNot(Receive())
+	})
+})
+
+var _ = Describe("The agent worker's backend stop", func() {
+	// One implementation behind both carriers, for the same reason: the bus
+	// subscription and the tunnel's control route both call this.
+
+	It("treats a stop that names no backend as a no-op rather than a failure", func() {
+		// A malformed publish must not become a non-2xx the frontend reads as
+		// a worker it could not reach.
+		Expect(dropMCPSessionsForBackend(context.Background(), messaging.BackendStopRequest{})).To(Succeed())
+	})
+
+	It("succeeds for a backend it holds no sessions for", func() {
+		// The ordinary case on a worker that never touched that backend. An
+		// error here would be reported as this worker failing to serve the
+		// verb, on every stop of every backend it does not know about.
+		Expect(dropMCPSessionsForBackend(context.Background(),
+			messaging.BackendStopRequest{Backend: "a-backend-this-worker-never-saw"})).To(Succeed())
+	})
+})

core/http/endpoints/localai/nodes.go

@@ -302,30 +302,39 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr
 // path in core/services/worker/worker.go does), because approval alone does not
 // prompt a re-registration and nothing else can hand it the secret.
 //
-// Only BACKEND nodes get one, and that is a decision rather than an oversight.
-// An agent worker serves no gRPC backends and no file staging; nothing dials
-// into it at all, so a tunnel replaces nothing for it and there is no client on
-// the agent side that would ever open one. Minting anyway would hand out a
-// working credential for a pipe nobody drives, which is surface without a
-// feature, and it would contradict every comment in this change that says
-// "backend workers, the ones that tunnel".
+// BACKEND and AGENT nodes get one; anything else has its credential CLEARED.
+//
+// Both kinds of worker now dial out and hold one tunnel, and neither opens an
+// inbound port. A backend worker carries its gRPC backends and its file staging
+// on it; an agent worker carries only its own HTTP server, which is where its
+// MCP control verbs live. The gate was once backend-only, and correctly so:
+// nothing dialled into an agent worker, so a credential for it would have been
+// surface with no feature behind it. That stopped being true when the frontend
+// gained a reason to reach an agent worker by RPC.
 //
 // The gate lives HERE and not in ConnectHandler, which never looks at NodeType.
 // It does not need to, PROVIDED an ineligible node ends up with no credential
 // rather than merely being handed no new one, because the handler's empty-hash
 // branch is what does the refusing. So this CLEARS the column instead of
 // returning early, and the difference is not theoretical: Register upserts by
-// NAME, so a backend node re-registering as an agent keeps its ID, and
+// NAME, so a node re-registering under a different type keeps its ID, and
 // Register's struct Updates zero-skips TunnelTokenHash while writing the new
-// node_type. An early return left a live credential on a row that had become an
-// agent. Clearing is what makes "enforcement is structural" true.
+// node_type. An early return left a live credential on a row whose type had
+// changed. Clearing is what makes "enforcement is structural" true, and it is
+// what the eligibility list widening rather than disappearing preserves: the
+// invariant is that a node's stored hash always matches the credential its
+// CURRENT type is entitled to, which for an ineligible type is none.
 //
 // It clears unconditionally rather than only when something is there, so the
 // invariant holds without depending on what the row happened to contain. The
-// cost is one UPDATE per agent registration.
+// cost is one UPDATE per registration of an ineligible node.
 //
-// The day agent workers want a tunnel, relaxing the eligibility condition is
-// the whole change, and it has to be a deliberate one.
+// No node type reaches the clearing branch through RegisterNodeEndpoint today,
+// which rejects any node_type that is neither backend nor agent. That is a
+// reason to keep the branch rather than to drop it: a row's node_type is also
+// written by older builds and will be written by future types, and the branch
+// is what makes adding one a decision about eligibility rather than a silent
+// grant.
 //
 // A failure to mint or to store is logged and the response goes out without the
 // token. Registration is what gets a worker into the cluster at all, and
@@ -337,12 +346,12 @@ func attachTunnelToken(ctx context.Context, response map[string]any, registry *n
 	if node == nil {
 		return
 	}
-	if node.NodeType != nodes.NodeTypeBackend {
+	if !tunnelEligible(node.NodeType) {
 		// Cleared, not skipped. SetTunnelTokenHash writes the single column
 		// directly rather than through a struct update, so unlike Register it
 		// can write an empty value; see its doc.
 		if err := registry.SetTunnelTokenHash(ctx, node.ID, ""); err != nil {
-			xlog.Error("Failed to clear the tunnel credential of a node that is not a backend worker",
+			xlog.Error("Failed to clear the tunnel credential of a node whose type holds none",
 				"node", node.Name, "type", node.NodeType, "error", err)
 		}
 		return
@@ -358,6 +367,18 @@ func attachTunnelToken(ctx context.Context, response map[string]any, registry *n
 	response["tunnel_token"] = plaintext
 }
 
+// tunnelEligible reports whether a node of this type holds a tunnel credential.
+//
+// One predicate rather than a condition written into attachTunnelToken, because
+// the mint branch and the clear branch are the two halves of ONE rule and must
+// not be able to disagree: written as two conditions, widening the mint without
+// widening the clear leaves a node type that is granted a credential and never
+// stripped of one, and widening the clear without the mint strips a node type
+// on every registration it makes. Neither has a symptom until a tunnel dial.
+func tunnelEligible(nodeType string) bool {
+	return nodeType == nodes.NodeTypeBackend || nodeType == nodes.NodeTypeAgent
+}
+
 // attachNatsJWT adds a per-node NATS user JWT to a register/approve response when minting is enabled.
 func attachNatsJWT(response map[string]any, node *nodes.BackendNode, natsCfg natsauth.Config) {
 	if !natsCfg.CanMintWorkers() || node == nil || node.Status == nodes.StatusPending {
@@ -704,12 +725,12 @@ func DeleteBackendOnNodeEndpoint(unloader nodes.NodeCommandSender) echo.HandlerF
 func ListBackendsOnNodeEndpoint(unloader nodes.NodeCommandSender, registry *nodes.NodeRegistry) echo.HandlerFunc {
 	return func(c echo.Context) error {
 		nodeID := c.Param("id")
-		// Agent-type workers don't run backends and never subscribe to the
-		// nodes.<id>.backend.list NATS subject, so the request would hang
-		// until timeout with "no responders". Their backend list is simply
-		// empty. Mirror the aggregate-list guard in managers_distributed.go
-		// (skip nodes whose NodeType is set and not "backend") so the
-		// single-node and cluster-wide views stay consistent.
+		// Agent-type workers don't run backends and mount no backend.list
+		// route on the tunnel they hold, so asking one can only 404. Their
+		// backend list is simply empty. Mirror the aggregate-list guard in
+		// managers_distributed.go (skip nodes whose NodeType is set and not
+		// "backend") so the single-node and cluster-wide views stay
+		// consistent.
 		if node, err := registry.Get(c.Request().Context(), nodeID); err == nil {
 			if node.NodeType != "" && node.NodeType != nodes.NodeTypeBackend {
 				return c.JSON(http.StatusOK, []messaging.NodeBackendInfo{})

core/http/endpoints/localai/nodes_test.go

@@ -181,50 +181,98 @@ var _ = Describe("Node HTTP handlers", func() {
 			Expect(second["tunnel_token"]).ToNot(Equal(plaintext))
 		})
 
-		It("does not issue a tunnel credential to an agent node", func() {
-			// An agent worker serves no gRPC backends and no file staging;
-			// nothing dials into it, so a tunnel replaces nothing for it and no
-			// client on its side would open one. Minting anyway would be
-			// credential surface with no feature behind it.
+		It("issues a tunnel credential to an agent node", func() {
+			// An agent worker holds a tunnel too. It runs no backends and
+			// stages no files, so what its tunnel carries is only its own HTTP
+			// server, but the frontend reaches its control verbs over it and
+			// therefore has to be able to dial it at all.
 			//
-			// Enforcement is structural rather than a second check: with no
-			// credential minted, the node's hash stays empty and the tunnel
-			// route refuses it like any other node without one.
+			// The gate that used to refuse this was correct while nothing
+			// dialled into an agent worker. Reopening it is deliberate, and the
+			// spec below is what keeps its other half honest.
 			resp := register(`{"name":"agent-1","node_type":"agent"}`, "", true)
 			Expect(resp["node_type"]).To(Equal(nodes.NodeTypeAgent))
-			Expect(resp).ToNot(HaveKey("tunnel_token"))
+
+			plaintext, _ := resp["tunnel_token"].(string)
+			Expect(plaintext).ToNot(BeEmpty())
 
 			node, err := registry.Get(context.Background(), resp["id"].(string))
 			Expect(err).ToNot(HaveOccurred())
-			Expect(node.TunnelTokenHash).To(BeEmpty())
+			// Stored as a hash, never as the secret, exactly as a backend
+			// node's is; ConnectHandler compares against this column and does
+			// not look at node_type at all.
+			Expect(node.TunnelTokenHash).To(Equal(hashOf(plaintext)))
+			Expect(node.TunnelTokenHash).ToNot(Equal(plaintext))
+
+			// Per-node, not derived from anything shared. A second agent
+			// registering gets a different credential.
+			other := register(`{"name":"agent-2","node_type":"agent"}`, "", true)
+			Expect(other["tunnel_token"]).ToNot(Equal(plaintext))
 		})
 
-		It("clears a tunnel credential when a node stops being a backend node", func() {
-			// Register upserts BY NAME, so a node can change node_type in place.
-			// Skipping the mint on the way through leaves the credential the
-			// node earned as a backend sitting on a row that is now an agent:
-			// Register's struct Updates zero-skips the column while writing the
-			// new node_type, so nothing else clears it. ConnectHandler never
-			// looks at node_type, so that stale hash is a usable tunnel
-			// credential for a node type that is not supposed to hold one.
+		It("rewrites the hash, rather than leaving a stale one, when a node changes type", func() {
+			// Register upserts BY NAME, so a node can change node_type in
+			// place, and Register's struct Updates zero-skips TunnelTokenHash
+			// while writing the new node_type. The invariant is that the stored
+			// hash always matches the node's CURRENT credential: a node whose
+			// type changed must not be left holding the one it was handed
+			// under its old type, because that is a secret the worker still
+			// knows and nothing would ever retire.
 			//
 			// This is the same shape as the Register-upserts-by-name hazard
 			// already carried forward: a name is not an identity.
 			backend := register(`{"name":"shifty","address":"10.0.0.7:50051"}`, "", true)
-			Expect(backend["tunnel_token"]).ToNot(BeEmpty())
+			// Read with a comma-ok rather than a bare assertion: a build that
+			// issues no credential must fail this spec on the assertion below,
+			// naming what it is missing, rather than panic on a nil interface.
+			backendToken, _ := backend["tunnel_token"].(string)
+			Expect(backendToken).ToNot(BeEmpty())
 
 			agent := register(`{"name":"shifty","node_type":"agent"}`, "", true)
 			Expect(agent["id"]).To(Equal(backend["id"]), "re-registration must keep the node identity")
 			Expect(agent["node_type"]).To(Equal(nodes.NodeTypeAgent))
-			Expect(agent).ToNot(HaveKey("tunnel_token"))
+			agentToken, _ := agent["tunnel_token"].(string)
+			Expect(agentToken).ToNot(BeEmpty(),
+				"the node changed type and was handed no credential, so its stored hash is whatever its previous type left behind")
 
 			node, err := registry.Get(context.Background(), backend["id"].(string))
 			Expect(err).ToNot(HaveOccurred())
-			// The claim the gate makes is that an ineligible node HAS no
-			// credential, not merely that it was not handed a new one. Only
-			// then is the empty-hash refusal in ConnectHandler the enforcement.
-			Expect(node.TunnelTokenHash).To(BeEmpty(),
-				"the node kept the credential it earned as a backend, so the mint-site gate is not structural")
+			Expect(node.TunnelTokenHash).To(Equal(hashOf(agentToken)))
+			Expect(node.TunnelTokenHash).ToNot(Equal(hashOf(backendToken)),
+				"the node kept the credential it earned under its previous type")
+		})
+
+		It("clears the credential of a node whose type is entitled to none", func() {
+			// The other half of the gate, and the half that makes enforcement
+			// STRUCTURAL: ConnectHandler never looks at node_type, so what
+			// refuses an ineligible node is its empty hash. Skipping the mint
+			// would leave a live credential on the row.
+			//
+			// Driven through the registry rather than through
+			// RegisterNodeEndpoint, which rejects any node_type that is neither
+			// backend nor agent. That validation is exactly why the branch
+			// cannot be reached from the wire today, and exactly why the branch
+			// has to stay: a row's node_type is also written by other builds,
+			// and adding a type must be a decision about eligibility rather
+			// than a silent grant.
+			node := &nodes.BackendNode{Name: "shifty-future", NodeType: nodes.NodeTypeBackend}
+			Expect(registry.Register(context.Background(), node, true)).To(Succeed())
+			response := map[string]any{}
+			attachTunnelToken(context.Background(), response, registry, node)
+			Expect(response).To(HaveKey("tunnel_token"))
+			stored, err := registry.Get(context.Background(), node.ID)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stored.TunnelTokenHash).ToNot(BeEmpty())
+
+			node.NodeType = "some-future-worker-kind"
+			response = map[string]any{}
+			attachTunnelToken(context.Background(), response, registry, node)
+
+			Expect(response).ToNot(HaveKey("tunnel_token"))
+			stored, err = registry.Get(context.Background(), node.ID)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(stored.TunnelTokenHash).To(BeEmpty(),
+				"an ineligible node kept a usable tunnel credential, so the mint-site gate is not structural")
 		})
 
 		It("returns nats_jwt when account seed is configured", func() {

core/services/agentworker/agentworker_suite_test.go

@@ -0,0 +1,13 @@
+package agentworker_test
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestAgentWorker(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Agent worker control plane")
+}

core/services/agentworker/handlers.go

@@ -0,0 +1,111 @@
+// Package agentworker serves an agent worker's control plane over the tunnel it
+// holds to the frontend.
+//
+// An agent worker runs no backend processes and stages no files. What it does
+// serve is MCP: it creates the sessions a frontend cannot (stdio servers under
+// docker), executes tools against them, and answers discovery. Those verbs
+// travelled on NATS request-reply, which meant the frontend picked a worker by
+// letting the bus's queue group pick one, and neither side could say which
+// worker had answered.
+//
+// This package is the worker half of moving them onto the same HTTP control
+// plane a backend worker already serves. The frontend half, which chooses WHICH
+// agent worker to ask, is not here: this package answers, it does not select.
+package agentworker
+
+import (
+	"context"
+	"encoding/json"
+	"net/http"
+
+	"github.com/mudler/LocalAI/core/services/messaging"
+	"github.com/mudler/LocalAI/core/services/workerctl"
+)
+
+// UnaryHandler answers a verb with exactly one reply and no progress. The
+// handler's bytes become the whole response body.
+//
+// An error returned here is this worker FAILING TO SERVE the verb, never the
+// verb's own bad news. The distinction is the one the whole phase is built on:
+// a body on a 200 is the WORKER'S OWN ANSWER, which a reap guard may act on,
+// and everything else is a failure the frontend must not read as evidence
+// about anything. An MCP tool that ran and errored is an answer, and it travels
+// as bytes with an error field set, exactly as it did over the bus.
+type UnaryHandler func(ctx context.Context, raw json.RawMessage) (json.RawMessage, error)
+
+// StreamHandler answers a verb that emits progress lines BEFORE its reply.
+//
+// It exists from this task even though nothing sets one yet, because the
+// alternative found in review was a later task discovering that a Config of
+// UnaryHandlers cannot express a verb that streams, and inventing a second
+// handler shape under time pressure.
+//
+// pub is how a handler writes those lines. It is messaging.Publisher and not
+// *agents.StreamPublisher so that this package does not import
+// core/services/agents: the concrete writer is constructed by the caller that
+// owns the response body, which is the server in this package, and the handler
+// only ever needs Publish(subject string, data any) error.
+type StreamHandler func(ctx context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error)
+
+// Config is what the agent worker's control plane needs to serve its verbs.
+//
+// A nil field means this worker does not serve that verb, and Register mounts
+// nothing for it, so the catch-all under workerctl.Prefix answers 404. That is
+// the same 404 an older build gives, which is what the frontend already reads
+// as "this worker does not serve that verb" (nodes.ErrWorkerControlUnsupported)
+// rather than as absence. It is why a later task can add two handlers without
+// any other file changing shape.
+//
+// workerctl.PathAgentCancel is named in the path table with no field here. It
+// gets the same 404 for the same reason, and the path is fixed now so the two
+// sides cannot disagree about it later.
+type Config struct {
+	// MCPTool answers workerctl.PathMCPToolExecute.
+	MCPTool UnaryHandler
+	// MCPDiscovery answers workerctl.PathMCPDiscovery.
+	MCPDiscovery UnaryHandler
+	// BackendStop drops the MCP sessions cached for a backend that is going
+	// away. It answers workerctl.PathBackendStop, the SAME path a backend
+	// worker serves with a completely different implementation: that one kills
+	// the process and recycles its port. One path, two implementations, one
+	// caller, which is what lets the frontend stop branching on node type to
+	// pick a carrier.
+	BackendStop func(ctx context.Context, req messaging.BackendStopRequest) error
+
+	// AgentExecute answers workerctl.PathAgentExecute and MCPCIRun answers
+	// workerctl.PathMCPCIRun. Both are nil in this task and both are set later,
+	// which is where the queue subjects they replace become claim rows. They are
+	// declared here so the streaming shape is decided once, with the rest of the
+	// control plane, rather than twice.
+	AgentExecute StreamHandler
+	MCPCIRun     StreamHandler
+}
+
+// Register mounts every agent control verb whose handler is non-nil on mux.
+//
+// It is the Register half of a nodes.AuthenticatedRoutes, so the mux it is
+// handed is private and reachable only through that route set's bearer check.
+// Nothing here does its own authentication, and nothing here may: a second
+// check beside the first is the one that gets forgotten.
+func (c Config) Register(mux *http.ServeMux) {
+	if c.MCPTool != nil {
+		mux.HandleFunc(workerctl.PathMCPToolExecute, serveUnary(workerctl.PathMCPToolExecute, c.MCPTool))
+	}
+	if c.MCPDiscovery != nil {
+		mux.HandleFunc(workerctl.PathMCPDiscovery, serveUnary(workerctl.PathMCPDiscovery, c.MCPDiscovery))
+	}
+	if c.BackendStop != nil {
+		mux.HandleFunc(workerctl.PathBackendStop, serveBackendStop(c.BackendStop))
+	}
+	if c.AgentExecute != nil {
+		mux.HandleFunc(workerctl.PathAgentExecute, serveStream(workerctl.PathAgentExecute, c.AgentExecute))
+	}
+	if c.MCPCIRun != nil {
+		mux.HandleFunc(workerctl.PathMCPCIRun, serveStream(workerctl.PathMCPCIRun, c.MCPCIRun))
+	}
+
+	// The catch-all, mounted unconditionally and last. A path under the control
+	// prefix that no verb claims is a frontend newer than this worker, or a
+	// verb this worker does not implement, and both are the same answer.
+	mux.HandleFunc(workerctl.Prefix, workerctl.WriteUnknownPath)
+}

core/services/agentworker/server.go

@@ -0,0 +1,337 @@
+package agentworker
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net"
+	"net/http"
+	"sync"
+
+	"github.com/mudler/xlog"
+
+	"github.com/mudler/LocalAI/core/services/messaging"
+	"github.com/mudler/LocalAI/core/services/nodes"
+	"github.com/mudler/LocalAI/core/services/worker"
+	"github.com/mudler/LocalAI/core/services/workerctl"
+)
+
+// How a verb's failure reaches the frontend, written once.
+//
+// Phase 3 found this same rule written at three sites in one file and pinned at
+// one, so every exit below goes through writeFailure and every verb has its own
+// spec for it. The rule: a non-2xx means this worker FAILED TO SERVE the
+// request, which the frontend maps onto "no route to that worker" and nobody
+// may act on; a 2xx body is the WORKER'S OWN ANSWER, which a reap guard may.
+// A handler's error is the first, always, and never the second.
+
+// writeFailure is the ONE place a verb's failure becomes a status.
+func writeFailure(w http.ResponseWriter, status int, verb string, err error) {
+	http.Error(w, "the "+verb+" verb could not be served: "+err.Error(), status)
+}
+
+// failToServe answers a verb this worker could not serve. 500 rather than 404,
+// which the frontend reads as a verb the worker does not implement, and rather
+// than any 2xx, which it reads as an answer.
+func failToServe(w http.ResponseWriter, verb string, err error) {
+	writeFailure(w, http.StatusInternalServerError, verb, err)
+}
+
+// refuseRequest answers a request this worker could not read. 400 because the
+// fault is the caller's, and non-2xx for the same reason as above: a body this
+// worker could not decode is not an answer about anything.
+func refuseRequest(w http.ResponseWriter, verb string, err error) {
+	writeFailure(w, http.StatusBadRequest, verb, err)
+}
+
+// serveUnary answers one unary verb: the handler's bytes, whole, on a 200.
+func serveUnary(verb string, h UnaryHandler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, ok := workerctl.ReadRequestBody(w, r)
+		if !ok {
+			return
+		}
+		reply, err := h(r.Context(), json.RawMessage(body))
+		if err != nil {
+			failToServe(w, verb, err)
+			return
+		}
+		if len(reply) == 0 {
+			// The shape a verb with nothing to say takes, and the same one the
+			// backend worker's fire-and-forget verbs take, so one frontend
+			// client handles both without knowing which kind of worker answered.
+			w.WriteHeader(http.StatusNoContent)
+			return
+		}
+		w.Header().Set("Content-Type", "application/json")
+		if _, err := w.Write(reply); err != nil {
+			xlog.Debug("agent worker control reply could not be written", "verb", verb, "error", err)
+		}
+	}
+}
+
+// serveBackendStop answers backend.stop on the agent worker.
+//
+// It answers 204 with no body, which is exactly what a BACKEND worker answers
+// on this same path. The two implementations share nothing else, and that is
+// the point: the frontend issues one RPC and reads one answer without knowing
+// which kind of worker it reached.
+func serveBackendStop(h func(ctx context.Context, req messaging.BackendStopRequest) error) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, ok := workerctl.ReadRequestBody(w, r)
+		if !ok {
+			return
+		}
+		var req messaging.BackendStopRequest
+		if err := json.Unmarshal(body, &req); err != nil {
+			refuseRequest(w, workerctl.PathBackendStop, err)
+			return
+		}
+		// The caller's context is dropped, for the reason the backend worker's
+		// backend.stop drops it: this is fire-and-forget, so there is no answer
+		// the caller is still waiting on, and abandoning the cleanup half way
+		// would leave sessions cached for a backend that is gone.
+		if err := h(context.WithoutCancel(r.Context()), req); err != nil {
+			failToServe(w, workerctl.PathBackendStop, err)
+			return
+		}
+		w.WriteHeader(http.StatusNoContent)
+	}
+}
+
+// serveStream answers one streaming verb: zero or more progress lines followed
+// by exactly one reply line, NDJSON, the reply last.
+//
+// The status is written on the FIRST line rather than at entry, and that is
+// what lets a handler that fails before publishing anything answer a non-2xx
+// like every other verb here. Once a line is on the wire the status is spent:
+// such a handler's failure ends the body with NO reply line, which the frontend
+// reads as no answer rather than as a failed one. Both are "this worker did not
+// answer", which is the only honest thing to say and the only thing nobody may
+// act on.
+func serveStream(verb string, h StreamHandler) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		body, ok := workerctl.ReadRequestBody(w, r)
+		if !ok {
+			return
+		}
+		stream := &ndjsonStream{w: w}
+		reply, err := h(r.Context(), json.RawMessage(body), stream)
+		if err != nil {
+			if !stream.committed() {
+				failToServe(w, verb, err)
+				return
+			}
+			xlog.Warn("An agent worker control verb failed after it had streamed progress, so its response ends with no reply line",
+				"verb", verb, "error", err)
+			return
+		}
+		stream.writeReply(verb, reply)
+	}
+}
+
+// ndjsonStream writes the Envelope lines of one streaming control response.
+//
+// The mutex is not optional: a handler may publish progress from a goroutine of
+// its own, so without serialization a progress write can interleave with the
+// terminal reply write and put a torn line on the wire. done is what enforces
+// the other half of the contract: once the reply is written, a late progress
+// line is dropped rather than appended after it.
+type ndjsonStream struct {
+	mu     sync.Mutex
+	w      http.ResponseWriter
+	header bool
+	done   bool
+}
+
+// Publish writes one progress line. It is messaging.Publisher, which is what
+// lets a handler written against the bus keep its shape.
+//
+// The subject is DISCARDED, and that is a statement about the carrier rather
+// than a shortcut. On the bus a subject was how a progress message found the
+// one caller waiting for it; here the response body the caller is already
+// reading IS that correlation, so there is nothing left for a subject to
+// select. It is kept in the signature because it is the interface the handlers
+// implement.
+func (n *ndjsonStream) Publish(_ string, data any) error {
+	raw, err := json.Marshal(data)
+	if err != nil {
+		return fmt.Errorf("encoding a progress line: %w", err)
+	}
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.done {
+		return nil
+	}
+	n.write(workerctl.Envelope{Progress: raw})
+	return nil
+}
+
+// committed reports whether anything has been written, which is whether the
+// status is still this handler's to choose.
+func (n *ndjsonStream) committed() bool {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	return n.header
+}
+
+// writeReply writes the single terminal reply line and closes the stream to any
+// further progress.
+func (n *ndjsonStream) writeReply(verb string, reply json.RawMessage) {
+	n.mu.Lock()
+	defer n.mu.Unlock()
+	if n.done {
+		return
+	}
+	if len(reply) == 0 {
+		// A handler that answers nothing still owes the caller a terminal line,
+		// or the caller reads to EOF without one and cannot tell that from a
+		// truncated response.
+		reply = json.RawMessage(`null`)
+	}
+	n.write(workerctl.Envelope{Reply: reply})
+	n.done = true
+	xlog.Debug("agent worker control stream finished", "verb", verb)
+}
+
+// write encodes one envelope and flushes it. Callers hold n.mu.
+func (n *ndjsonStream) write(env workerctl.Envelope) {
+	if !n.header {
+		n.w.Header().Set("Content-Type", workerctl.ContentTypeStream)
+		// A streaming body must not be buffered into a guessed content type:
+		// the caller reads line by line and the first line may be minutes
+		// before the last.
+		n.w.Header().Set("X-Content-Type-Options", "nosniff")
+		n.w.WriteHeader(http.StatusOK)
+		n.header = true
+	}
+	if err := json.NewEncoder(n.w).Encode(env); err != nil {
+		xlog.Debug("agent worker control stream line could not be written", "error", err)
+		return
+	}
+	if f, ok := n.w.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+// Options is everything Start needs to put an agent worker on the tunnel.
+type Options struct {
+	// FrontendURL is the value the worker registered against
+	// (LOCALAI_REGISTER_TO).
+	FrontendURL string
+	// NodeID is the identity registration assigned this worker.
+	NodeID string
+	// TunnelToken supplies the node's own tunnel credential. A function because
+	// the credential rotates on every re-registration and is read at DIAL time;
+	// see worker.TunnelConfig.Token.
+	TunnelToken func() string
+	// ControlToken is the bearer token guarding the loopback control server.
+	// It is the deployment's registration token, the same one a backend worker
+	// puts in front of its control plane and the same one the frontend's
+	// control client presents.
+	ControlToken string
+	// Handlers are the verbs this worker serves.
+	Handlers Config
+}
+
+// Runtime is a running agent worker control plane: one loopback HTTP server and
+// one tunnel to the frontend.
+type Runtime struct {
+	server *http.Server
+	tunnel *worker.Tunnel
+	addr   string
+}
+
+// Addr is the loopback address the control server bound.
+//
+// It exists for specs and for a log line. It is deliberately NOT registered
+// anywhere: nothing outside this process may learn it, because the only way in
+// is meant to be the tunnel.
+func (r *Runtime) Addr() string { return r.addr }
+
+// Connected reports whether the tunnel currently holds a live session. It is a
+// statement about REACHABILITY and nothing else; nothing may read it as the
+// worker being gone.
+func (r *Runtime) Connected() bool { return r != nil && r.tunnel.Connected() }
+
+// Close stops the tunnel and the control server. It is safe on a nil receiver
+// so a caller can defer it beside a failed Start.
+func (r *Runtime) Close() error {
+	if r == nil {
+		return nil
+	}
+	err := r.tunnel.Close()
+	nodes.ShutdownFileTransferServer(r.server)
+	return err
+}
+
+// Start binds the control server on loopback, mounts the agent's verbs behind
+// one bearer check, and holds a tunnel to the frontend that carries them.
+//
+// One function rather than a run of statements in a CLI command, and that is
+// the point rather than tidiness. The pieces are one fact: a control server
+// nothing can reach is a worker that looks healthy and answers nothing, and a
+// tunnel with no server behind it refuses every stream. Started as separate
+// lines in a Run body, each is a line whose loss has no symptom and which no
+// spec can reach without starting a whole worker process.
+//
+// The listener binds 127.0.0.1:0 and the port is never advertised: an agent
+// worker needs no inbound port, which is the whole reason it is being moved off
+// a bus it had to be able to dial.
+//
+// A failure to START the tunnel is returned, unlike a failure to CONNECT: the
+// first means the frontend URL or this node's identity is unusable, and the
+// second is a frontend that is down or an admin who has not approved this node
+// yet, which the tunnel retries through with backoff.
+func Start(ctx context.Context, opts Options) (*Runtime, error) {
+	lis, err := net.Listen("tcp", loopbackBind)
+	if err != nil {
+		return nil, fmt.Errorf("binding the agent worker control server: %w", err)
+	}
+	addr := lis.Addr().String()
+
+	// Created here and armed only once the tunnel exists, below. Until then
+	// /readyz reports ready, which is correct: reaching this line means the
+	// worker has already registered with the frontend, so it is mid-startup
+	// rather than broken.
+	readiness := &nodes.WorkerReadiness{}
+	server, err := nodes.StartControlOnlyServer(lis, opts.ControlToken, readiness, &nodes.AuthenticatedRoutes{
+		Prefix:   workerctl.Prefix,
+		Register: opts.Handlers.Register,
+	})
+	if err != nil {
+		_ = lis.Close()
+		return nil, fmt.Errorf("starting the agent worker control server: %w", err)
+	}
+
+	tunnel, err := worker.StartTunnel(ctx, worker.TunnelConfig{
+		FrontendURL: opts.FrontendURL,
+		NodeID:      opts.NodeID,
+		Token:       opts.TunnelToken,
+		// Built by worker.HTTPOnlyServices rather than inline, so the routing
+		// table, which is this feature's security boundary, is reachable from a
+		// spec without starting a worker. An agent worker runs no backend
+		// processes, so the grpc tag is not offered at all.
+		Services: worker.HTTPOnlyServices(addr),
+	})
+	if err != nil {
+		nodes.ShutdownFileTransferServer(server)
+		return nil, fmt.Errorf("starting the agent worker tunnel: %w", err)
+	}
+	// Armed here rather than at the call site. /readyz means "the frontend can
+	// reach me", and a live tunnel session is the only thing that makes that
+	// true for a process that binds loopback and advertises nothing. As a
+	// separate statement elsewhere its loss has no symptom: the gate fails
+	// open, so the worker answers 200 forever with no session.
+	readiness.Set(nodes.TunnelReadiness(tunnel))
+
+	xlog.Info("Agent worker control plane serving over its tunnel", "node", opts.NodeID, "addr", addr)
+	return &Runtime{server: server, tunnel: tunnel, addr: addr}, nil
+}
+
+// loopbackBind is where the control server binds.
+//
+// A constant so that "an agent worker opens no inbound port" is a fact about
+// the code rather than a claim about its configuration: there is no flag, no
+// environment variable and no argument that moves it.
+const loopbackBind = "127.0.0.1:0"

core/services/agentworker/server_test.go

@@ -0,0 +1,572 @@
+package agentworker_test
+
+import (
+	"bufio"
+	"context"
+	"encoding/json"
+	"errors"
+	"io"
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+
+	"github.com/gorilla/websocket"
+	"github.com/libp2p/go-yamux/v5"
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	"github.com/mudler/LocalAI/core/services/agentworker"
+	"github.com/mudler/LocalAI/core/services/cluster"
+	"github.com/mudler/LocalAI/core/services/messaging"
+	"github.com/mudler/LocalAI/core/services/nodes"
+	"github.com/mudler/LocalAI/core/services/workerctl"
+)
+
+// The agent worker's control plane is exercised over a REAL listener with a
+// real http.Client, never against a bare ServeMux.
+//
+// The bearer check, the 404 for an unmounted verb and the status a failing
+// handler answers with are all things the transport carries, and a double that
+// calls a handler function directly cannot fail the way production fails: it
+// never goes through the mount that applies the check, so a spec written that
+// way stays green with the check deleted.
+
+const controlToken = "agent-control-token"
+
+// serveConfig mounts cfg the way the agent worker mounts it, on loopback behind
+// the bearer check, and returns the base URL.
+func serveConfig(cfg agentworker.Config) string {
+	GinkgoHelper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	Expect(err).ToNot(HaveOccurred())
+	srv, err := nodes.StartControlOnlyServer(lis, controlToken, nil, &nodes.AuthenticatedRoutes{
+		Prefix:   workerctl.Prefix,
+		Register: cfg.Register,
+	})
+	Expect(err).ToNot(HaveOccurred())
+	DeferCleanup(func() { nodes.ShutdownFileTransferServer(srv) })
+	return "http://" + lis.Addr().String()
+}
+
+// post issues one control request with the bearer token.
+func post(base, path, body string) *http.Response {
+	GinkgoHelper()
+	return postWithToken(base, path, body, controlToken)
+}
+
+func postWithToken(base, path, body, token string) *http.Response {
+	GinkgoHelper()
+	req, err := http.NewRequest(http.MethodPost, base+path, strings.NewReader(body))
+	Expect(err).ToNot(HaveOccurred())
+	if token != "" {
+		req.Header.Set("Authorization", "Bearer "+token)
+	}
+	resp, err := http.DefaultClient.Do(req)
+	Expect(err).ToNot(HaveOccurred())
+	DeferCleanup(func() { _ = resp.Body.Close() })
+	return resp
+}
+
+func bodyOf(resp *http.Response) string {
+	GinkgoHelper()
+	raw, err := io.ReadAll(resp.Body)
+	Expect(err).ToNot(HaveOccurred())
+	return string(raw)
+}
+
+// echoHandler answers with the bytes it was given, so a spec can tell the
+// handler's own answer apart from anything the server invented.
+func echoHandler(reply string) agentworker.UnaryHandler {
+	return func(_ context.Context, raw json.RawMessage) (json.RawMessage, error) {
+		return json.RawMessage(reply), nil
+	}
+}
+
+// failingHandler is a verb this worker could not serve.
+func failingHandler(err error) agentworker.UnaryHandler {
+	return func(context.Context, json.RawMessage) (json.RawMessage, error) { return nil, err }
+}
+
+// fullConfig serves every verb this task implements.
+func fullConfig() agentworker.Config {
+	return agentworker.Config{
+		MCPTool:      echoHandler(`{"result":"tool-ran"}`),
+		MCPDiscovery: echoHandler(`{"servers":[]}`),
+		BackendStop:  func(context.Context, messaging.BackendStopRequest) error { return nil },
+	}
+}
+
+var _ = Describe("The agent worker's control verbs", func() {
+	It("answers each verb with the handler's own bytes, on a 200", func() {
+		base := serveConfig(fullConfig())
+
+		tool := post(base, workerctl.PathMCPToolExecute, `{"tool_name":"x"}`)
+		Expect(tool.StatusCode).To(Equal(http.StatusOK))
+		Expect(bodyOf(tool)).To(Equal(`{"result":"tool-ran"}`))
+
+		disc := post(base, workerctl.PathMCPDiscovery, `{"model_name":"m"}`)
+		Expect(disc.StatusCode).To(Equal(http.StatusOK))
+		Expect(bodyOf(disc)).To(Equal(`{"servers":[]}`))
+	})
+
+	It("answers backend.stop with the 204 a BACKEND worker answers on that same path", func() {
+		// One path, two implementations, one caller. The frontend must not have
+		// to know which kind of worker it reached to read the answer, which is
+		// what lets its carrier split for this verb die.
+		stopped := make(chan messaging.BackendStopRequest, 1)
+		cfg := fullConfig()
+		cfg.BackendStop = func(_ context.Context, req messaging.BackendStopRequest) error {
+			stopped <- req
+			return nil
+		}
+		base := serveConfig(cfg)
+
+		resp := post(base, workerctl.PathBackendStop, `{"backend":"llama-cpp","force":true}`)
+		Expect(resp.StatusCode).To(Equal(http.StatusNoContent))
+		Expect(bodyOf(resp)).To(BeEmpty())
+		var got messaging.BackendStopRequest
+		Eventually(stopped).Should(Receive(&got))
+		// The DTO reaches the handler decoded, so the agent's implementation
+		// reads the same request the backend worker's does.
+		Expect(got.Backend).To(Equal("llama-cpp"))
+		Expect(got.Force).To(BeTrue())
+	})
+
+	It("refuses a request with no bearer token", func() {
+		base := serveConfig(fullConfig())
+		resp := postWithToken(base, workerctl.PathMCPToolExecute, `{}`, "")
+		Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))
+	})
+
+	It("refuses a request with the wrong bearer token", func() {
+		// Non-empty and wrong, which a check that merely tested for the
+		// header's presence would let through.
+		base := serveConfig(fullConfig())
+		resp := postWithToken(base, workerctl.PathMCPToolExecute, `{}`, "not-the-token")
+		Expect(resp.StatusCode).To(Equal(http.StatusUnauthorized))
+	})
+
+	DescribeTable("answers a verb it does not serve with the catch-all 404",
+		// 404 and NOT 500. The frontend reads exactly this status as "this
+		// worker does not serve that verb" (nodes.ErrWorkerControlUnsupported)
+		// rather than as absence, which is what makes adding a handler later a
+		// purely additive change: an older worker and a newer one differ only
+		// in which paths are mounted.
+		func(path string) {
+			base := serveConfig(fullConfig())
+			resp := post(base, path, `{}`)
+			Expect(resp.StatusCode).To(Equal(http.StatusNotFound))
+			Expect(bodyOf(resp)).To(ContainSubstring("unknown worker control path"))
+		},
+		Entry("agent execute, whose handler is nil until it is wired", workerctl.PathAgentExecute),
+		Entry("mcp ci run, whose handler is nil until it is wired", workerctl.PathMCPCIRun),
+		Entry("agent cancel, which has no handler field at all yet", workerctl.PathAgentCancel),
+		Entry("a backend worker's verb, which an agent worker never serves", workerctl.PathBackendInstall),
+		Entry("a path no build has ever named", workerctl.Prefix+"nothing/here"),
+	)
+
+	DescribeTable("mounts nothing for a nil handler, so the catch-all answers",
+		// The mirror of the table above: it is the NIL FIELD that produces the
+		// 404, not the path being unknown to the package.
+		func(path string, blank func(*agentworker.Config)) {
+			cfg := fullConfig()
+			blank(&cfg)
+			base := serveConfig(cfg)
+			Expect(post(base, path, `{}`).StatusCode).To(Equal(http.StatusNotFound))
+		},
+		Entry("mcp tool execute", workerctl.PathMCPToolExecute, func(c *agentworker.Config) { c.MCPTool = nil }),
+		Entry("mcp discovery", workerctl.PathMCPDiscovery, func(c *agentworker.Config) { c.MCPDiscovery = nil }),
+		Entry("backend stop", workerctl.PathBackendStop, func(c *agentworker.Config) { c.BackendStop = nil }),
+	)
+
+	// A rule stated at every verb has to be pinned at every verb. The exit
+	// helper is one function, and that is exactly why one spec here would not
+	// prove the others use it.
+	DescribeTable("answers a handler's failure with a non-2xx and never a 200",
+		func(path string, breaks func(*agentworker.Config)) {
+			cfg := fullConfig()
+			breaks(&cfg)
+			base := serveConfig(cfg)
+
+			resp := post(base, path, `{}`)
+			// The property, stated as the frontend reads it: a 2xx body is the
+			// WORKER'S OWN ANSWER, which a reap guard may act on, and a failure
+			// to serve is not an answer about anything. A 200 carrying an error
+			// field would put this in the wrong bucket.
+			Expect(resp.StatusCode).ToNot(Equal(http.StatusOK))
+			Expect(resp.StatusCode).ToNot(Equal(http.StatusNoContent))
+			Expect(resp.StatusCode).To(BeNumerically(">=", 400))
+			// And not the 404 that means "older worker", which would send the
+			// frontend down a version-skew fallback for a transient failure.
+			Expect(resp.StatusCode).ToNot(Equal(http.StatusNotFound))
+			Expect(bodyOf(resp)).To(ContainSubstring("could not be served"))
+		},
+		Entry("mcp tool execute", workerctl.PathMCPToolExecute, func(c *agentworker.Config) {
+			c.MCPTool = failingHandler(errors.New("the tool plane is down"))
+		}),
+		Entry("mcp discovery", workerctl.PathMCPDiscovery, func(c *agentworker.Config) {
+			c.MCPDiscovery = failingHandler(errors.New("discovery is down"))
+		}),
+		Entry("backend stop", workerctl.PathBackendStop, func(c *agentworker.Config) {
+			c.BackendStop = func(context.Context, messaging.BackendStopRequest) error {
+				return errors.New("the session cache is wedged")
+			}
+		}),
+		Entry("agent execute, before it has published anything", workerctl.PathAgentExecute, func(c *agentworker.Config) {
+			c.AgentExecute = func(context.Context, json.RawMessage, messaging.Publisher) (json.RawMessage, error) {
+				return nil, errors.New("no agent to run")
+			}
+		}),
+	)
+
+	It("refuses a GET on a mounted verb, because a control verb is a command", func() {
+		base := serveConfig(fullConfig())
+		req, err := http.NewRequest(http.MethodGet, base+workerctl.PathMCPToolExecute, nil)
+		Expect(err).ToNot(HaveOccurred())
+		req.Header.Set("Authorization", "Bearer "+controlToken)
+		resp, err := http.DefaultClient.Do(req)
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { _ = resp.Body.Close() })
+		Expect(resp.StatusCode).To(Equal(http.StatusMethodNotAllowed))
+	})
+
+	It("refuses a body larger than the shared control cap", func() {
+		base := serveConfig(fullConfig())
+		oversized := `{"tool_name":"` + strings.Repeat("a", workerctl.MaxRequestBytes) + `"}`
+		resp := post(base, workerctl.PathMCPToolExecute, oversized)
+		Expect(resp.StatusCode).To(Equal(http.StatusBadRequest))
+	})
+
+	It("answers a body the verb could not decode with a non-2xx", func() {
+		base := serveConfig(fullConfig())
+		resp := post(base, workerctl.PathBackendStop, `{"backend":`)
+		Expect(resp.StatusCode).To(Equal(http.StatusBadRequest))
+	})
+
+	It("mounts no file-staging routes, because an agent worker stages nothing", func() {
+		// StartControlOnlyServer exists so that a worker with no staging
+		// directories cannot end up serving upload and download handlers
+		// against a path that resolves to its working directory.
+		base := serveConfig(fullConfig())
+		req, err := http.NewRequest(http.MethodGet, base+"/v1/files/anything", nil)
+		Expect(err).ToNot(HaveOccurred())
+		req.Header.Set("Authorization", "Bearer "+controlToken)
+		resp, err := http.DefaultClient.Do(req)
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { _ = resp.Body.Close() })
+		Expect(resp.StatusCode).To(Equal(http.StatusNotFound))
+	})
+})
+
+var _ = Describe("The agent worker's streaming verbs", func() {
+	// Nothing sets a StreamHandler in this task. These specs exist so the shape
+	// is decided and pinned now: a later task adds two handlers and changes
+	// nothing else, and it can only do that if what a stream looks like on the
+	// wire is already fixed.
+
+	// lines reads an NDJSON body into its envelopes.
+	lines := func(resp *http.Response) []workerctl.Envelope {
+		GinkgoHelper()
+		var out []workerctl.Envelope
+		sc := bufio.NewScanner(resp.Body)
+		for sc.Scan() {
+			if strings.TrimSpace(sc.Text()) == "" {
+				continue
+			}
+			var env workerctl.Envelope
+			Expect(json.Unmarshal(sc.Bytes(), &env)).To(Succeed())
+			out = append(out, env)
+		}
+		Expect(sc.Err()).ToNot(HaveOccurred())
+		return out
+	}
+
+	It("puts every progress line before the one reply, and the reply last", func() {
+		cfg := fullConfig()
+		cfg.AgentExecute = func(_ context.Context, raw json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) {
+			Expect(pub.Publish("agent.progress", map[string]string{"step": "one"})).To(Succeed())
+			Expect(pub.Publish("agent.progress", map[string]string{"step": "two"})).To(Succeed())
+			return json.RawMessage(`{"done":true}`), nil
+		}
+		base := serveConfig(cfg)
+
+		resp := post(base, workerctl.PathAgentExecute, `{}`)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		Expect(resp.Header.Get("Content-Type")).To(Equal(workerctl.ContentTypeStream))
+
+		got := lines(resp)
+		Expect(got).To(HaveLen(3))
+		Expect(string(got[0].Progress)).To(ContainSubstring(`"one"`))
+		Expect(string(got[1].Progress)).To(ContainSubstring(`"two"`))
+		// The contract the caller stops reading on: exactly one reply, and it
+		// is the last thing on the body.
+		Expect(got[2].Reply).ToNot(BeEmpty())
+		Expect(string(got[2].Reply)).To(Equal(`{"done":true}`))
+	})
+
+	It("drops a progress line published after the reply, rather than appending it", func() {
+		// Without this the reply is not the last line, and a caller that stops
+		// reading at the reply leaves bytes in the stream that the next read on
+		// a pooled connection would see.
+		late := make(chan error, 1)
+		cfg := fullConfig()
+		cfg.MCPCIRun = func(_ context.Context, _ json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) {
+			go func() {
+				// Published from another goroutine, which is what a handler
+				// with a debounce timer does, and what the mutex is for.
+				<-late
+				_ = pub.Publish("job.progress", map[string]string{"step": "too-late"})
+				late <- nil
+			}()
+			return json.RawMessage(`{"status":"completed"}`), nil
+		}
+		base := serveConfig(cfg)
+
+		resp := post(base, workerctl.PathMCPCIRun, `{}`)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		got := lines(resp)
+		// The body ended after the reply, so the late publish had nowhere to
+		// go. Released only now, so the ordering is scripted rather than raced.
+		late <- nil
+		Eventually(late).Should(Receive())
+		Expect(got).To(HaveLen(1))
+		Expect(string(got[0].Reply)).To(Equal(`{"status":"completed"}`))
+	})
+
+	It("ends the body with NO reply line when a handler fails after streaming", func() {
+		// The status is already on the wire by then, so the failure cannot be a
+		// non-2xx. It must also not be a reply, because a reply is the worker's
+		// own answer. A body that ends without one is "this worker did not
+		// answer", which is the only honest thing left to say.
+		cfg := fullConfig()
+		cfg.AgentExecute = func(_ context.Context, _ json.RawMessage, pub messaging.Publisher) (json.RawMessage, error) {
+			Expect(pub.Publish("agent.progress", map[string]string{"step": "started"})).To(Succeed())
+			return nil, errors.New("the agent died mid-run")
+		}
+		base := serveConfig(cfg)
+
+		resp := post(base, workerctl.PathAgentExecute, `{}`)
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		got := lines(resp)
+		Expect(got).To(HaveLen(1))
+		Expect(got[0].Reply).To(BeEmpty())
+		Expect(string(got[0].Progress)).To(ContainSubstring("started"))
+	})
+})
+
+// fakeFrontend is the far side of an agent worker's tunnel: the real WebSocket
+// upgrade and the real yamux server handshake, so these specs exercise the
+// wire rather than a mock of it.
+type fakeFrontend struct {
+	srv      *httptest.Server
+	sessions chan *yamux.Session
+}
+
+func newFakeFrontend() *fakeFrontend {
+	f := &fakeFrontend{sessions: make(chan *yamux.Session, 8)}
+	upgrader := websocket.Upgrader{}
+	f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != cluster.ConnectPath {
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		ws, err := upgrader.Upgrade(w, r, nil)
+		if err != nil {
+			return
+		}
+		sess, err := yamux.Server(cluster.WebsocketConn(ws), nil, nil)
+		if err != nil {
+			_ = ws.Close()
+			return
+		}
+		select {
+		case f.sessions <- sess:
+		default:
+			_ = sess.Close()
+		}
+	}))
+	return f
+}
+
+func (f *fakeFrontend) close() {
+	for {
+		select {
+		case sess := <-f.sessions:
+			_ = sess.Close()
+		default:
+			f.srv.Close()
+			return
+		}
+	}
+}
+
+// awaitErr runs fn on its own goroutine and reports its result on a channel.
+//
+// Every blocking read below goes through it. Asserting "the worker refused this
+// stream" with a read deadline is satisfied just as well by a stream the worker
+// never answered at all; reading with NO deadline and requiring the channel to
+// deliver inverts that.
+func awaitErr(fn func() error) <-chan error {
+	ch := make(chan error, 1)
+	go func() { ch <- fn() }()
+	return ch
+}
+
+var _ = Describe("An agent worker on the tunnel", func() {
+	var (
+		ctx      context.Context
+		cancel   context.CancelFunc
+		frontend *fakeFrontend
+		rt       *agentworker.Runtime
+	)
+
+	BeforeEach(func() {
+		ctx, cancel = context.WithCancel(context.Background())
+		frontend = newFakeFrontend()
+	})
+
+	AfterEach(func() {
+		if rt != nil {
+			Expect(rt.Close()).To(Succeed())
+			rt = nil
+		}
+		cancel()
+		frontend.close()
+	})
+
+	start := func(url string) {
+		GinkgoHelper()
+		var err error
+		rt, err = agentworker.Start(ctx, agentworker.Options{
+			FrontendURL:  url,
+			NodeID:       "agent-node-1",
+			TunnelToken:  func() string { return "tunnel-secret" },
+			ControlToken: controlToken,
+			Handlers:     fullConfig(),
+		})
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	session := func() *yamux.Session {
+		GinkgoHelper()
+		var sess *yamux.Session
+		Eventually(frontend.sessions, "10s").Should(Receive(&sess))
+		return sess
+	}
+
+	It("binds only loopback, so it opens no inbound port", func() {
+		start(frontend.srv.URL)
+		host, _, err := net.SplitHostPort(rt.Addr())
+		Expect(err).ToNot(HaveOccurred())
+		Expect(host).To(Equal("127.0.0.1"))
+	})
+
+	It("serves its control verbs through the tunnel and nothing else", func() {
+		start(frontend.srv.URL)
+		sess := session()
+
+		// The http tag, which is the one an agent worker offers. Driven as a
+		// real HTTP request over a real yamux stream, so what this proves is
+		// that the frontend can reach the verbs, not that a map has a key.
+		stream, err := sess.OpenStream(ctx)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "ignored:1")).To(Succeed())
+		accepted := awaitErr(func() error { return cluster.ReadStreamReply(stream) })
+		Eventually(accepted, "10s").Should(Receive(BeNil()))
+
+		req, err := http.NewRequest(http.MethodPost, "http://agent-node-1"+workerctl.PathMCPToolExecute,
+			strings.NewReader(`{"tool_name":"x"}`))
+		Expect(err).ToNot(HaveOccurred())
+		req.Header.Set("Authorization", "Bearer "+controlToken)
+		Expect(req.Write(stream)).To(Succeed())
+
+		read := make(chan *http.Response, 1)
+		go func() {
+			resp, rerr := http.ReadResponse(bufio.NewReader(stream), req)
+			if rerr == nil {
+				read <- resp
+			}
+			close(read)
+		}()
+		var resp *http.Response
+		Eventually(read, "10s").Should(Receive(&resp))
+		Expect(resp.StatusCode).To(Equal(http.StatusOK))
+		defer func() { _ = resp.Body.Close() }()
+		raw, err := io.ReadAll(resp.Body)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(string(raw)).To(Equal(`{"result":"tool-ran"}`))
+	})
+
+	It("refuses a stream tagged for gRPC, which it runs nothing to serve", func() {
+		start(frontend.srv.URL)
+		sess := session()
+
+		stream, err := sess.OpenStream(ctx)
+		Expect(err).ToNot(HaveOccurred())
+		Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:50051")).To(Succeed())
+
+		reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) })
+		var got error
+		Eventually(reply, "10s").Should(Receive(&got))
+		// The refusal names the tag, which tells a frontend a retry is
+		// pointless, rather than "unavailable", which tells it to retry
+		// something that can never work on this kind of worker.
+		Expect(got).To(MatchError(cluster.ErrStreamTagUnknown))
+		Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable))
+	})
+
+	It("reports itself not ready while it holds no tunnel session", func() {
+		// The gate FAILS OPEN when nothing arms it, so a worker with a dead
+		// tunnel would answer 200 forever and nothing in the process would say
+		// a word. That is the whole symptom of the arming line going missing.
+		start("http://127.0.0.1:1")
+
+		resp, err := http.Get("http://" + rt.Addr() + "/readyz") //nolint:noctx // loopback probe in a spec
+		Expect(err).ToNot(HaveOccurred())
+		DeferCleanup(func() { _ = resp.Body.Close() })
+		Expect(resp.StatusCode).To(Equal(http.StatusServiceUnavailable))
+		Expect(bodyOf(resp)).To(ContainSubstring("tunnel"))
+		Expect(rt.Connected()).To(BeFalse())
+	})
+
+	It("reports itself ready once its tunnel is up", func() {
+		// The negative control for the spec above: a gate wired to a probe that
+		// always failed would satisfy it.
+		start(frontend.srv.URL)
+		session()
+
+		Eventually(func() int {
+			resp, err := http.Get("http://" + rt.Addr() + "/readyz") //nolint:noctx // loopback probe in a spec
+			if err != nil {
+				return 0
+			}
+			defer func() { _ = resp.Body.Close() }()
+			return resp.StatusCode
+		}, "10s").Should(Equal(http.StatusOK))
+	})
+
+	It("refuses to start with no routes to serve", func() {
+		_, err := nodes.StartControlOnlyServer(mustListen(), controlToken, nil, nil)
+		Expect(err).To(HaveOccurred())
+		Expect(err.Error()).To(ContainSubstring("no routes"))
+	})
+
+	DescribeTable("refuses to start on a route set it cannot mount",
+		func(routes *nodes.AuthenticatedRoutes, want string) {
+			_, err := nodes.StartControlOnlyServer(mustListen(), controlToken, nil, routes)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring(want))
+		},
+		Entry("no prefix", &nodes.AuthenticatedRoutes{Register: func(*http.ServeMux) {}}, "no prefix"),
+		Entry("no registrar", &nodes.AuthenticatedRoutes{Prefix: workerctl.Prefix}, "no registrar"),
+	)
+})
+
+func mustListen() net.Listener {
+	GinkgoHelper()
+	lis, err := net.Listen("tcp", "127.0.0.1:0")
+	Expect(err).ToNot(HaveOccurred())
+	DeferCleanup(func() { _ = lis.Close() })
+	return lis
+}

core/services/messaging/subjects.go

@@ -149,11 +149,13 @@ func SubjectResponseCancel(responseID string) string {
 // through its tunnel, so a subject builder for any of them would be a subject
 // nothing publishes and nothing subscribes to.
 //
-// ONE survives: backend.stop, and only for AGENT workers. They hold no tunnel,
-// so they have no control plane to serve, and they subscribe to it to drop the
-// MCP sessions cached for a backend that is going away. See
-// nodes.RemoteUnloaderAdapter.stopBackend for the split, and
-// core/cli/agent_worker.go for the subscriber.
+// ONE survives: backend.stop, and only for AGENT workers. They now hold a
+// tunnel and mount that verb on it, so what keeps this subject alive is the
+// PUBLISHER: nodes.RemoteUnloaderAdapter.stopBackend still sends an agent
+// node's stop here rather than over its control route. The agent worker
+// subscribes to it to drop the MCP sessions cached for a backend that is going
+// away; see core/cli/agent_worker.go for the subscriber, which shares one
+// implementation with the control route so the two carriers cannot diverge.
 //
 // The request and reply types below are UNCHANGED and still live here: they are
 // the wire format of the control routes, byte for byte what the subjects
@@ -285,7 +287,9 @@ type BackendStopRequest struct {
 // It is the one node subject left, and it is addressed only to agent nodes. A
 // BACKEND worker takes its stop on workerctl.PathBackendStop over its tunnel,
 // where it also kills the process and recycles the port; an agent worker runs
-// no backend processes and only needs to hear that one went.
+// no backend processes and only needs to hear that one went. An agent worker
+// serves that same path over its own tunnel as well, so this subject is what
+// the publisher has not moved off yet rather than the only way to reach one.
 func SubjectNodeBackendStop(nodeID string) string {
 	return subjectNodePrefix + sanitizeSubjectToken(nodeID) + ".backend.stop"
 }

core/services/nodes/file_transfer_server.go

@@ -90,18 +90,9 @@ func StartFileTransferServerWithReadiness(lis net.Listener, stagingDir, modelsDi
 // StartFileTransferServerWithRoutes is StartFileTransferServerWithReadiness
 // plus an extra authenticated route set. See AuthenticatedRoutes.
 func StartFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir, dataDir, token string, maxUploadSize int64, readiness *WorkerReadiness, extra *AuthenticatedRoutes, logStore ...*model.BackendLogStore) (*http.Server, error) {
-	// Checked before anything is created. A route set that names no prefix or
-	// no registrar is a caller bug, and mounting nothing for it would be the
-	// worst possible answer: the server comes up healthy and every route the
-	// caller believes it registered answers 404, which through a tunnel is
-	// indistinguishable from a version skew.
-	if extra != nil {
-		if extra.Prefix == "" {
-			return nil, fmt.Errorf("extra routes were given no prefix to mount under")
-		}
-		if extra.Register == nil {
-			return nil, fmt.Errorf("extra routes under %q were given no registrar", extra.Prefix)
-		}
+	// Checked before anything is created; see validateAuthenticatedRoutes.
+	if err := validateAuthenticatedRoutes(extra); err != nil {
+		return nil, err
 	}
 	if err := os.MkdirAll(stagingDir, 0750); err != nil {
 		return nil, fmt.Errorf("creating staging dir %s: %w", stagingDir, err)
@@ -175,8 +166,101 @@ func StartFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir,
 		registerBackendLogHandlers(mux, token, ls)
 	}
 
-	// Liveness/readiness probes — unauthenticated so container orchestrators
-	// (Docker HEALTHCHECK, k8s probes) can hit them without the bearer token.
+	mountProbes(mux, readiness)
+	mountAuthenticatedRoutes(mux, token, extra)
+
+	return serveWorkerHTTP(lis, mux,
+		"stagingDir", stagingDir, "modelsDir", modelsDir, "dataDir", dataDir), nil
+}
+
+// StartControlOnlyServer starts a worker's loopback HTTP server carrying ONLY
+// the routes in extra, behind the same bearer check the file-transfer server
+// applies, plus the same two probes.
+//
+// Separate from StartFileTransferServer rather than a flag on it: an agent
+// worker stages no files, and a constructor that took empty directory arguments
+// would mount the upload and download handlers against a path that resolves to
+// the process working directory. The two servers share every rule they have in
+// common through the helpers below rather than through a parameter that means
+// "skip half of me".
+//
+// extra is required here, unlike on the file-transfer server where it is
+// optional. A control-only server with no routes is a listener that answers
+// nothing, and the failure mode is the one the route validation below already
+// exists to prevent: the process comes up healthy and every verb 404s, which
+// through a tunnel is indistinguishable from a version skew.
+func StartControlOnlyServer(lis net.Listener, token string, readiness *WorkerReadiness, extra *AuthenticatedRoutes) (*http.Server, error) {
+	if extra == nil {
+		return nil, fmt.Errorf("a control-only worker server was given no routes to serve")
+	}
+	if err := validateAuthenticatedRoutes(extra); err != nil {
+		return nil, err
+	}
+	// The same warning the file-transfer server gives, for a smaller blast
+	// radius: an empty token makes checkBearerToken fail open, so every control
+	// verb this worker serves is reachable unauthenticated by anything that can
+	// reach this port. It binds loopback only, which is why this is a warning
+	// rather than a refusal.
+	if token == "" {
+		xlog.Warn("Worker control server starting WITHOUT a registration token: its control verbs are unauthenticated for anyone who can reach this port; set LOCALAI_REGISTRATION_TOKEN")
+	}
+
+	mux := http.NewServeMux()
+	mountProbes(mux, readiness)
+	mountAuthenticatedRoutes(mux, token, extra)
+
+	return serveWorkerHTTP(lis, mux, "routes", extra.Prefix), nil
+}
+
+// validateAuthenticatedRoutes rejects a route set that names no prefix or no
+// registrar.
+//
+// Mounting nothing for one would be the worst possible answer: the server comes
+// up healthy and every route the caller believes it registered answers 404,
+// which through a tunnel is indistinguishable from a version skew.
+func validateAuthenticatedRoutes(extra *AuthenticatedRoutes) error {
+	if extra == nil {
+		return nil
+	}
+	if extra.Prefix == "" {
+		return fmt.Errorf("extra routes were given no prefix to mount under")
+	}
+	if extra.Register == nil {
+		return fmt.Errorf("extra routes under %q were given no registrar", extra.Prefix)
+	}
+	return nil
+}
+
+// mountAuthenticatedRoutes puts extra behind the bearer check on its own mux.
+//
+// One site, and that is the point rather than a convenience. This is the check
+// that stands between an unauthenticated caller and every verb a worker serves,
+// and there are now two kinds of worker server mounting routes through it. A
+// second copy is a second place to forget it, which would not fail any spec
+// written against the first.
+func mountAuthenticatedRoutes(mux *http.ServeMux, token string, extra *AuthenticatedRoutes) {
+	if extra == nil {
+		return
+	}
+	extraMux := http.NewServeMux()
+	extra.Register(extraMux)
+	mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if !checkBearerToken(r, token) {
+			xlog.Debug("worker HTTP server: unauthorized request on an extra route",
+				"method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr)
+			http.Error(w, "unauthorized", http.StatusUnauthorized)
+			return
+		}
+		extraMux.ServeHTTP(w, r)
+	}))
+}
+
+// mountProbes adds the liveness and readiness probes every worker server
+// carries.
+//
+// They are unauthenticated so container orchestrators (Docker HEALTHCHECK, k8s
+// probes) can reach them without the bearer token.
+func mountProbes(mux *http.ServeMux, readiness *WorkerReadiness) {
 	probe := func(check func() error) http.HandlerFunc {
 		return func(w http.ResponseWriter, r *http.Request) {
 			if r.Method != http.MethodGet && r.Method != http.MethodHead {
@@ -195,27 +279,19 @@ func StartFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir,
 	}
 
 	// Liveness: reaching this point means the listener is bound and the mux is
-	// serving. Deliberately independent of readiness — a worker whose NATS link
-	// is momentarily down must not be restarted, or a NATS blip becomes a
-	// cluster-wide restart storm.
+	// serving. Deliberately independent of readiness: a worker whose link to
+	// the frontend is momentarily down must not be restarted, or a blip becomes
+	// a cluster-wide restart storm.
 	mux.HandleFunc("/healthz", probe(func() error { return nil }))
 	// Readiness: "can this worker actually accept work?" See WorkerReadiness.
+	// A 503 here is THIS CONTAINER'S OWN REPORT that it cannot serve right now,
+	// never a claim that the node is gone; the frontend decides that from the
+	// tunnel session it holds, aged against LOCALAI_WORKER_RECONNECT_GRACE.
 	mux.HandleFunc("/readyz", probe(readiness.Check))
+}
 
-	if extra != nil {
-		extraMux := http.NewServeMux()
-		extra.Register(extraMux)
-		mux.Handle(extra.Prefix, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			if !checkBearerToken(r, token) {
-				xlog.Debug("worker HTTP server: unauthorized request on an extra route",
-					"method", r.Method, "path", r.URL.Path, "remote", r.RemoteAddr)
-				http.Error(w, "unauthorized", http.StatusUnauthorized)
-				return
-			}
-			extraMux.ServeHTTP(w, r)
-		}))
-	}
-
+// serveWorkerHTTP starts serving mux on lis and returns the server.
+func serveWorkerHTTP(lis net.Listener, mux *http.ServeMux, logFields ...any) *http.Server {
 	addr := lis.Addr().String()
 	server := &http.Server{
 		// Addr is informational here: Serve takes the listener, not this
@@ -228,13 +304,13 @@ func StartFileTransferServerWithRoutes(lis net.Listener, stagingDir, modelsDir,
 	}
 
 	go func() {
-		xlog.Info("HTTP file transfer server started", "addr", addr, "stagingDir", stagingDir, "modelsDir", modelsDir, "dataDir", dataDir)
+		xlog.Info("Worker HTTP server started", append([]any{"addr", addr}, logFields...)...)
 		if err := server.Serve(lis); err != nil && err != http.ErrServerClosed {
-			xlog.Error("HTTP file transfer server error", "error", err)
+			xlog.Error("Worker HTTP server error", "error", err)
 		}
 	}()
 
-	return server, nil
+	return server
 }
 
 func handleHead(w http.ResponseWriter, r *http.Request, stagingDir, modelsDir, dataDir, key string) {

core/services/nodes/health.go

@@ -111,9 +111,15 @@ func (hm *HealthMonitor) ReadsAbsence() bool { return hm != nil && hm.presence !
 // Acting on any of those would demote a fleet for a reason that has nothing to
 // do with any worker, which is the collapse this whole mechanism replaced.
 //
-// Backend workers only. An agent worker holds no tunnel at all, so it has no
-// departure to measure and would answer PresenceUnknown anyway; the type check
-// is here to save the query rather than to add a second rule.
+// Backend workers only, and since agent workers hold tunnels too that check is
+// now the RULE rather than an optimisation. It used to save a query: an agent
+// worker dialled nothing, so it had no departure to measure and would have
+// answered PresenceUnknown regardless. It now has a real node_connections row
+// that really does age past the grace, and demoting an agent node for it would
+// be a verdict about a route the agent's actual work does not travel on: an
+// agent worker still takes its jobs and its verbs over the bus. Deleting this
+// check would silently make every agent worker whose tunnel dropped go
+// unhealthy, and its next heartbeat would make it healthy again.
 func (hm *HealthMonitor) tunnelDeparted(ctx context.Context, node *BackendNode) bool {
 	if hm.presence == nil || node == nil {
 		return false

core/services/nodes/health_test.go

@@ -563,8 +563,11 @@ var _ = Describe("HealthMonitor and a worker whose tunnel is gone", func() {
 	})
 
 	It("leaves an AGENT node alone even when its tunnel would read as gone", func() {
-		// Agent workers hold no tunnel and still take their one verb over the
-		// bus. A departure row for one is not a fact about it.
+		// Agent workers hold a tunnel of their own now, and still take their
+		// jobs and their one remaining node verb over the bus. So a departure
+		// row for one is real and is still not a fact about whether the agent
+		// worker can work. This spec is what stops a bug in the agent tunnel
+		// client from demoting a fleet of perfectly healthy agent workers.
 		node := &BackendNode{Name: "agent-worker", NodeType: NodeTypeAgent}
 		Expect(registry.Register(ctx, node, true)).To(Succeed())
 		departTunnel(node.ID, grace+5*time.Second)

core/services/nodes/interfaces.go

@@ -103,10 +103,11 @@ type NodeHealthStore interface {
 // ModelLocator is used by RemoteUnloaderAdapter for model discovery.
 //
 // Get is here for one reason: backend.stop is the only control verb whose
-// carrier depends on what KIND of worker it is addressed to, because agent
-// workers hold no tunnel and still take it over the bus. The callers that come
-// through NodeCommandSender carry a node id and nothing else, so the type is
-// read here rather than threaded through every one of them.
+// carrier depends on what KIND of worker it is addressed to, because an agent
+// node's stop is still published on the bus rather than issued over the tunnel
+// it now holds. The callers that come through NodeCommandSender carry a node id
+// and nothing else, so the type is read here rather than threaded through every
+// one of them.
 type ModelLocator interface {
 	FindNodesWithModel(ctx context.Context, modelName string) ([]BackendNode, error)
 	RemoveNodeModel(ctx context.Context, nodeID, modelName string, replicaIndex int) error

core/services/nodes/managers_agent_node_test.go

@@ -14,12 +14,12 @@ import (
 	"github.com/mudler/LocalAI/core/services/workerctl"
 )
 
-// Agent workers hold no tunnel and serve no control plane, so asking one to
-// list its backends can only fail. ListBackends read that failure as a node
-// that had gone away and marked it unhealthy; the node's next heartbeat marked
-// it healthy again. Every poll of the backends view therefore flapped every
-// agent node in the cluster, and while it was unhealthy the router would not
-// schedule onto it.
+// Agent workers run no backend processes and mount no backend.list route on
+// the tunnel they hold, so asking one to list its backends can only fail.
+// ListBackends read that failure as a node that had gone away and marked it
+// unhealthy; the node's next heartbeat marked it healthy again. Every poll of
+// the backends view therefore flapped every agent node in the cluster, and
+// while it was unhealthy the router would not schedule onto it.
 var _ = Describe("Backend listing across mixed node types", func() {
 	var (
 		db       *gorm.DB

core/services/nodes/managers_distributed.go

@@ -168,9 +168,11 @@ func (d *DistributedBackendManager) enqueueAndDrainBackendOp(ctx context.Context
 		if node.Status == StatusPending {
 			continue
 		}
-		// Backend lifecycle ops only make sense on backend-type workers.
-		// Agent workers hold no tunnel and serve no control plane, so
-		// enqueueing for them guarantees a forever-retrying row that the
+		// Backend lifecycle ops only make sense on backend-type workers. An
+		// agent worker holds a tunnel and serves a control plane, but not
+		// THESE verbs: it runs no backend processes, so it mounts none of the
+		// install/upgrade/delete routes and answers the catch-all 404 for
+		// them. Enqueueing for one guarantees a forever-retrying row that the
 		// reconciler can never drain. Silently skip - they aren't consumers.
 		if node.NodeType != "" && node.NodeType != NodeTypeBackend {
 			continue
@@ -349,11 +351,12 @@ func (d *DistributedBackendManager) ListBackends() (gallery.SystemBackends, erro
 		if node.Status == StatusPending || node.Status == StatusOffline || node.Status == StatusDraining {
 			continue
 		}
-		// Only backend workers serve backend.list. An agent worker holds no
-		// tunnel, so asking one can only fail, and the failure handling used to
-		// read that as a node that had gone away: every poll of this view
-		// marked every agent node unhealthy and its next heartbeat marked it
-		// healthy again. The backend-op fan-out skips them for the same reason.
+		// Only backend workers serve backend.list. An agent worker holds a
+		// tunnel but runs no backend processes, so it mounts no such route and
+		// asking one can only 404, and the failure handling used to read that
+		// as a node that had gone away: every poll of this view marked every
+		// agent node unhealthy and its next heartbeat marked it healthy again.
+		// The backend-op fan-out skips them for the same reason.
 		if node.NodeType != "" && node.NodeType != NodeTypeBackend {
 			continue
 		}

core/services/nodes/unloader.go

@@ -46,10 +46,11 @@ type NodeCommandSender interface {
 // remote nodes.
 //
 // One verb is still carried by the bus, and only for one KIND of node:
-// backend.stop to an AGENT node. Agent workers hold no tunnel yet, so they have
-// nothing to serve a control route on, and they subscribe to
-// nodes.<id>.backend.stop to drop cached MCP sessions. Removing that publish
-// would strand them; see stopBackend.
+// backend.stop to an AGENT node. An agent worker now holds a tunnel and mounts
+// that route on it, but this side has not been moved onto it yet, and until it
+// is, the agent worker's subscription to nodes.<id>.backend.stop is what
+// actually drops its cached MCP sessions. Removing that publish would strand
+// them; see stopBackend.
 type RemoteUnloaderAdapter struct {
 	registry       ModelLocator
 	nats           messaging.MessagingClient
@@ -384,11 +385,12 @@ func (a *RemoteUnloaderAdapter) nodeTypeOf(ctx context.Context, nodeID string) s
 // stopBackend sends one backend.stop, over the carrier that kind of worker
 // listens on.
 //
-// An AGENT node keeps the bus. It holds no tunnel, so it has no control route
-// to serve, and it subscribes to nodes.<id>.backend.stop to drop the MCP
-// sessions cached for a backend that is going away. This is the one verb of the
-// ten that is split rather than moved, and the split is the honest intermediate
-// state until agent workers hold tunnels too.
+// An AGENT node keeps the bus, and that is now a statement about THIS side
+// rather than about the worker. The agent worker mounts backend.stop on the
+// same control path a backend worker serves it on, so the tunnel could carry
+// it; what has not moved is this publish. Splitting on node type is the one
+// verb of the ten that is split rather than moved, and it stays split until the
+// change that switches this call onto the control route retires the subject.
 func (a *RemoteUnloaderAdapter) stopBackend(ctx context.Context, nodeID, nodeType, backend string, force bool) error {
 	if nodeType == NodeTypeAgent {
 		subject := messaging.SubjectNodeBackendStop(nodeID)

core/services/nodes/unloader_test.go

@@ -317,7 +317,7 @@ var _ = Describe("RemoteUnloaderAdapter", func() {
 			Expect(bus.publishedSubjects()).To(BeEmpty())
 		})
 
-		It("sends a backend stop to an AGENT node over the bus, because agent workers hold no tunnel", func() {
+		It("sends a backend stop to an AGENT node over the bus, because that publisher has not moved onto the tunnel", func() {
 			locator.nodes = []BackendNode{{ID: "agent-1", Name: "agent", NodeType: NodeTypeAgent}}
 
 			Expect(adapter.StopBackend("agent-1", "llama-backend")).To(Succeed())

core/services/worker/control_routes.go

@@ -4,10 +4,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
 	"net/http"
 	"sync"
-	"unicode/utf8"
 
 	"github.com/mudler/LocalAI/core/services/messaging"
 	"github.com/mudler/LocalAI/core/services/workerctl"
@@ -35,19 +33,16 @@ import (
 // the bucket reserved for a broken link. Only a failure to read or route the
 // request is a non-2xx.
 
-// maxControlRequestBytes bounds a control request body.
-//
-// The largest real body is BackendInstallRequest.BackendGalleries, a serialized
-// gallery list of a few hundred kilobytes. Eight megabytes is therefore not a
-// size the protocol needs: it is a defence against a body that never ends,
-// arriving on a boundary this worker now serves.
-const maxControlRequestBytes = 8 << 20
-
-// maxEchoedPathBytes bounds how much of an unknown control path the 404 body
-// repeats back. The path is caller-controlled and the answer exists to be read
-// in a log line, so a caller cannot make this worker echo a request-sized
-// string into one.
-const maxEchoedPathBytes = 128
+// The request bounds and the unknown-path answer are workerctl's, not this
+// package's. An agent worker mounts verbs under the same prefix behind the same
+// bearer check, and two workers with private copies of "how big may a body be"
+// and "what does an unmounted verb answer" is the rule-at-N-sites-pinned-at-one
+// shape phase 3 kept finding. Aliased rather than re-typed at every use so the
+// specs that reason about the cap keep naming it locally.
+const (
+	maxControlRequestBytes = workerctl.MaxRequestBytes
+	maxEchoedPathBytes     = workerctl.MaxEchoedPathBytes
+)
 
 // installFunc and upgradeFunc are the shapes of the two long-running verbs.
 // They exist as named types so the fields that override them below read as one
@@ -169,7 +164,7 @@ func (s *backendSupervisor) RegisterControlRoutes(mux *http.ServeMux) {
 		// The catch-all. A path under the control prefix that no verb claims is
 		// a frontend newer than this worker, and the body says so, because a
 		// bare 404 through a tunnel is indistinguishable from a proxy fault.
-		http.Error(w, "unknown worker control path "+truncate(r.URL.Path, maxEchoedPathBytes), http.StatusNotFound)
+		workerctl.WriteUnknownPath(w, r)
 	})
 }
 
@@ -213,45 +208,14 @@ func postControlVerb(mux *http.ServeMux, path string, h controlVerb) {
 	})
 }
 
-// readControlBody enforces the two things every control verb requires of a
-// request: that it is a POST, and that its body is bounded.
+// readControlBody is workerctl.ReadRequestBody under this package's own name.
 //
-// A GET is refused rather than served because a control verb is a command, and
-// a liveness probe, a link prefetch or a browser address bar must not be able
-// to stop a node.
+// The indirection is one line and it earns it: every verb in this file reads
+// its body through this name, so the shared rule has exactly one call shape
+// here rather than fifteen import-qualified ones that a future edit could
+// replace individually.
 func readControlBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
-	if r.Method != http.MethodPost {
-		http.Error(w, "control verbs are POST only", http.StatusMethodNotAllowed)
-		return nil, false
-	}
-	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, maxControlRequestBytes))
-	if err != nil {
-		// A body this worker could not READ is not an answer about any
-		// backend, so it must not look like one: 400 is what the frontend maps
-		// onto "the request was rejected", never onto "that model is gone".
-		http.Error(w, "reading the control request body: "+err.Error(), http.StatusBadRequest)
-		return nil, false
-	}
-	return body, true
-}
-
-// truncate bounds a caller-controlled string that is about to be echoed.
-//
-// It cuts on a rune boundary. A byte-wise cut can split a multi-byte rune, and
-// the half rune then travels as a replacement character through every log and
-// UI that reads it; phase 2 shipped exactly that defect on a refusal reason and
-// pinned the rule afterwards. utf8.RuneStart is the same predicate the cluster
-// package uses for it, so the two are one rule rather than two hand-rolled
-// copies that can drift.
-func truncate(s string, max int) string {
-	if len(s) <= max {
-		return s
-	}
-	cut := max
-	for cut > 0 && !utf8.RuneStart(s[cut]) {
-		cut--
-	}
-	return s[:cut] + "…"
+	return workerctl.ReadRequestBody(w, r)
 }
 
 // ndjsonStream writes the Envelope lines of one streaming control response.

core/services/worker/control_routes_test.go

@@ -515,8 +515,11 @@ var _ = Describe("the worker's HTTP server", func() {
 		}
 	})
 
-	It("mounts every control verb, not just the one this spec reads", func() {
-		for _, p := range workerctl.AllPaths() {
+	It("mounts every control verb a BACKEND worker serves, not just the one this spec reads", func() {
+		// BackendPaths and not AllPaths. The union now includes the agent
+		// worker's verbs, which this worker deliberately does not mount and
+		// answers the catch-all 404 for.
+		for _, p := range workerctl.BackendPaths() {
 			if p == workerctl.PathNodeStop {
 				// Firing it would tear down the worker under the other specs.
 				continue

core/services/worker/tunnel.go

@@ -808,6 +808,28 @@ func tunnelServices(cfg *Config, httpBindAddr string) map[string]LocalService {
 	}
 }
 
+// HTTPOnlyServices is the stream-tag table for a worker that serves only its
+// own HTTP server.
+//
+// An agent worker runs no backend processes, so the grpc tag has nothing to
+// route to and is not offered. That is a security statement and not a tidiness
+// one: the table IS the tunnel's boundary, and the grpc tag is the only entry
+// whose target the frontend gets to influence at all. A worker that cannot
+// serve it should not carry the code that would.
+//
+// It exists as its own function for the reason tunnelServices does: the table
+// is the security boundary of the tunnel, and building it inline in a CLI Run
+// left it reachable only by starting a worker, which meant it was covered by
+// nothing.
+//
+// httpBindAddr is this worker's own listener, and fixedService ignores whatever
+// the frontend names, so nothing derived from the wire reaches a dialler here.
+func HTTPOnlyServices(httpBindAddr string) map[string]LocalService {
+	return map[string]LocalService{
+		cluster.StreamTagHTTP: fixedService(loopbackAddr(httpBindAddr)),
+	}
+}
+
 // fixedService routes a tagged stream to one address on this worker, ignoring
 // whatever the frontend named.
 //

core/services/worker/tunnel_test.go

@@ -279,6 +279,84 @@ var _ = Describe("Worker tunnel client", func() {
 		})
 	})
 
+	Describe("the HTTP-only routing table", func() {
+		// The table an AGENT worker installs. It is the security boundary of
+		// that worker's tunnel, so it is asserted directly and then over the
+		// wire, rather than only through whatever starts an agent worker.
+
+		It("offers exactly one tag, the worker's own HTTP server", func() {
+			// The grpc tag is the only entry whose target the frontend gets to
+			// influence at all, and an agent worker runs no backend processes
+			// for it to reach. Offering it would be reachable surface with
+			// nothing behind it.
+			table := HTTPOnlyServices("127.0.0.1:9999")
+			Expect(table).To(HaveLen(1))
+			Expect(table).To(HaveKey(cluster.StreamTagHTTP))
+			Expect(table).ToNot(HaveKey(cluster.StreamTagGRPC))
+		})
+
+		It("routes the http tag to the worker's own server, ignoring the address the frontend names", func() {
+			own := echoListener()
+			DeferCleanup(func() { _ = own.Close() })
+
+			frontend = newFakeFrontend(false)
+			start(func(c *TunnelConfig) { c.Services = HTTPOnlyServices(own.Addr().String()) })
+
+			stream, err := session().OpenStream(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			// A target this worker must not dial. fixedService ignores it.
+			Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "attacker.invalid:1")).To(Succeed())
+
+			reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) })
+			Eventually(reply, "10s").Should(Receive(BeNil()))
+
+			_, err = stream.Write([]byte("own-server"))
+			Expect(err).ToNot(HaveOccurred())
+			buf := make([]byte, len("own-server"))
+			read := awaitErr(func() error {
+				_, rerr := io.ReadFull(stream, buf)
+				return rerr
+			})
+			Eventually(read, "10s").Should(Receive(BeNil()))
+			Expect(string(buf)).To(Equal("own-server"))
+		})
+
+		It("refuses a stream tagged for gRPC as an unknown tag, over the wire", func() {
+			frontend = newFakeFrontend(false)
+			start(func(c *TunnelConfig) { c.Services = HTTPOnlyServices("127.0.0.1:1") })
+
+			stream, err := session().OpenStream(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagGRPC, "127.0.0.1:50051")).To(Succeed())
+
+			reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) })
+			var got error
+			Eventually(reply, "10s").Should(Receive(&got))
+			// The tag, not "unavailable". A frontend gives up on the first and
+			// retries the second, and retrying a tag this worker will never
+			// serve is a request that can never succeed.
+			Expect(got).To(MatchError(cluster.ErrStreamTagUnknown))
+			Expect(got).ToNot(MatchError(cluster.ErrStreamTargetUnavailable))
+			Expect(got).ToNot(MatchError(cluster.ErrStreamRequestInvalid))
+		})
+
+		It("rewrites a wildcard bind onto loopback, so the dial reaches the same listener", func() {
+			own := echoListenerOn("127.0.0.1:0")
+			DeferCleanup(func() { _ = own.Close() })
+
+			frontend = newFakeFrontend(false)
+			start(func(c *TunnelConfig) {
+				c.Services = HTTPOnlyServices(fmt.Sprintf("0.0.0.0:%d", portOf(own)))
+			})
+
+			stream, err := session().OpenStream(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(cluster.WriteStreamRequest(stream, cluster.StreamTagHTTP, "")).To(Succeed())
+			reply := awaitErr(func() error { return cluster.ReadStreamReply(stream) })
+			Eventually(reply, "10s").Should(Receive(BeNil()))
+		})
+	})
+
 	Describe("refusing a stream it cannot serve", func() {
 		// The refusal specs all read with NO deadline, on another goroutine.
 		// See awaitErr: a deadline would be satisfied by a stream that was

core/services/workerctl/paths.go

@@ -42,11 +42,26 @@ const (
 	PathFilesListDir = "/v1/control/files/listdir"
 )
 
-// AllPaths returns every control verb's path.
+// The verbs an AGENT worker serves.
+//
+// They live in the same package, under the same prefix and behind the same
+// bearer check as the backend worker's, because a worker is a worker to the
+// frontend: one control client, one path table, one set of failure meanings.
+// What differs is which of them a given worker MOUNTS, which is why the two
+// sets are named separately below.
+const (
+	PathMCPToolExecute = "/v1/control/mcp/tools/execute"
+	PathMCPDiscovery   = "/v1/control/mcp/discovery"
+	PathAgentExecute   = "/v1/control/agent/execute"
+	PathAgentCancel    = "/v1/control/agent/cancel"
+	PathMCPCIRun       = "/v1/control/mcp/ci/run"
+)
+
+// BackendPaths returns every control verb a BACKEND worker serves.
 //
 // It exists so a spec can assert a property of the whole set rather than of a
 // list it re-types, which would go stale the moment a verb is added.
-func AllPaths() []string {
+func BackendPaths() []string {
 	return []string{
 		PathBackendInstall,
 		PathBackendUpgrade,
@@ -65,6 +80,49 @@ func AllPaths() []string {
 	}
 }
 
+// AgentPaths returns every control verb an AGENT worker serves.
+//
+// PathBackendStop is in BOTH sets and that is the point rather than an
+// oversight: one path, two implementations, one caller. A backend worker kills
+// the process and recycles its port; an agent worker drops the MCP sessions it
+// cached for that backend. The frontend issues the same RPC to either and does
+// not branch on the node's type to pick a carrier.
+//
+// PathAgentCancel is named here with nothing mounting it yet. The frontend
+// therefore gets the catch-all's 404, which it already reads as "this worker
+// does not serve that verb" rather than as absence, and the path is fixed now
+// so the two sides cannot disagree about it later.
+func AgentPaths() []string {
+	return []string{
+		PathMCPToolExecute,
+		PathMCPDiscovery,
+		PathAgentExecute,
+		PathAgentCancel,
+		PathMCPCIRun,
+		PathBackendStop,
+	}
+}
+
+// AllPaths returns every control verb's path, each once.
+//
+// It is the union of the two sets above and is what package-wide properties
+// (every path under the prefix, no two verbs sharing a path) are asserted
+// over. A spec about what a PARTICULAR worker mounts must use BackendPaths or
+// AgentPaths instead: asserting mounting over the union would require every
+// worker to serve every verb, which is the opposite of what the split is for.
+func AllPaths() []string {
+	seen := make(map[string]bool)
+	out := make([]string, 0, len(BackendPaths())+len(AgentPaths()))
+	for _, p := range append(BackendPaths(), AgentPaths()...) {
+		if seen[p] {
+			continue
+		}
+		seen[p] = true
+		out = append(out, p)
+	}
+	return out
+}
+
 // Envelope is one line of a streaming control response.
 //
 // Exactly one of the two is set. Zero or more Progress lines are followed by

core/services/workerctl/paths_test.go

@@ -32,6 +32,11 @@ var _ = Describe("control plane paths on the wire", func() {
 		Entry("files stage", workerctl.PathFilesStage, "/v1/control/files/stage"),
 		Entry("files temp", workerctl.PathFilesTemp, "/v1/control/files/temp"),
 		Entry("files listdir", workerctl.PathFilesListDir, "/v1/control/files/listdir"),
+		Entry("mcp tool execute", workerctl.PathMCPToolExecute, "/v1/control/mcp/tools/execute"),
+		Entry("mcp discovery", workerctl.PathMCPDiscovery, "/v1/control/mcp/discovery"),
+		Entry("agent execute", workerctl.PathAgentExecute, "/v1/control/agent/execute"),
+		Entry("agent cancel", workerctl.PathAgentCancel, "/v1/control/agent/cancel"),
+		Entry("mcp ci run", workerctl.PathMCPCIRun, "/v1/control/mcp/ci/run"),
 	)
 
 	It("names the prefix exactly, since the worker mounts its whole control plane behind it", func() {
@@ -44,13 +49,13 @@ var _ = Describe("control plane paths on the wire", func() {
 		}
 	})
 
-	It("lists every verb this package names, so none can be dropped from the set", func() {
+	It("lists every verb a backend worker serves, so none can be dropped from the set", func() {
 		// The claim is bounded on purpose. Go constants are not enumerable, so
-		// nothing here can see a NEW constant that was never added to AllPaths;
+		// nothing here can see a NEW constant that was never added to the set;
 		// what this catches is an EXISTING verb going missing from it, which
 		// matters because the prefix check above and the worker's mounting spec
-		// both iterate AllPaths and would silently stop covering it.
-		Expect(workerctl.AllPaths()).To(ConsistOf(
+		// both iterate these and would silently stop covering it.
+		Expect(workerctl.BackendPaths()).To(ConsistOf(
 			workerctl.PathBackendInstall,
 			workerctl.PathBackendUpgrade,
 			workerctl.PathBackendList,
@@ -68,6 +73,37 @@ var _ = Describe("control plane paths on the wire", func() {
 		))
 	})
 
+	It("lists every verb an agent worker serves", func() {
+		Expect(workerctl.AgentPaths()).To(ConsistOf(
+			workerctl.PathMCPToolExecute,
+			workerctl.PathMCPDiscovery,
+			workerctl.PathAgentExecute,
+			workerctl.PathAgentCancel,
+			workerctl.PathMCPCIRun,
+			workerctl.PathBackendStop,
+		))
+	})
+
+	It("is the union of the two worker kinds, with the shared verb counted once", func() {
+		// backend.stop is in both sets, and AllPaths deduping it is what makes
+		// the distinctness check below a statement about the path table rather
+		// than about which set a verb happened to be typed into. A union that
+		// repeated it would fail that check for a table that is perfectly
+		// correct.
+		Expect(workerctl.AllPaths()).To(ContainElements(workerctl.BackendPaths()))
+		Expect(workerctl.AllPaths()).To(ContainElements(workerctl.AgentPaths()))
+		Expect(workerctl.AllPaths()).To(HaveLen(
+			len(workerctl.BackendPaths()) + len(workerctl.AgentPaths()) - 1))
+	})
+
+	It("serves the agent worker's backend.stop on the SAME path the backend worker's is on", func() {
+		// One path, two implementations, one caller. The frontend's carrier
+		// split for backend.stop dies on this: it issues the same RPC to either
+		// kind of worker without branching on the node's type.
+		Expect(workerctl.AgentPaths()).To(ContainElement(workerctl.PathBackendStop))
+		Expect(workerctl.BackendPaths()).To(ContainElement(workerctl.PathBackendStop))
+	})
+
 	It("gives each verb a distinct path", func() {
 		seen := map[string]bool{}
 		for _, p := range workerctl.AllPaths() {

core/services/workerctl/request.go

@@ -0,0 +1,85 @@
+package workerctl
+
+import (
+	"io"
+	"net/http"
+	"unicode/utf8"
+)
+
+// The rules every worker control verb enforces on a request, and the one answer
+// every worker gives for a path it does not serve.
+//
+// They live here, beside the path table both sides read, because there are now
+// TWO kinds of worker mounting verbs under Prefix: a backend worker and an
+// agent worker. Phase 3 found the same rule written at three sites in one file
+// and pinned at one, which is how a verb ends up reading an unbounded body or
+// answering a 500 where the frontend expects a 404. A second worker package
+// with its own copy of these rules would be that finding again, across a
+// package boundary where it is harder to see.
+
+// MaxRequestBytes bounds a control request body.
+//
+// The largest real body is BackendInstallRequest.BackendGalleries, a serialized
+// gallery list of a few hundred kilobytes. Eight megabytes is therefore not a
+// size the protocol needs: it is a defence against a body that never ends,
+// arriving on a boundary a worker now serves.
+const MaxRequestBytes = 8 << 20
+
+// MaxEchoedPathBytes bounds how much of an unknown control path the 404 body
+// repeats back. The path is caller-controlled and the answer exists to be read
+// in a log line, so a caller cannot make a worker echo a request-sized string
+// into one.
+const MaxEchoedPathBytes = 128
+
+// ReadRequestBody enforces the two things every control verb requires of a
+// request: that it is a POST, and that its body is bounded. It writes the
+// refusal itself and reports false when it did.
+//
+// A GET is refused rather than served because a control verb is a command, and
+// a liveness probe, a link prefetch or a browser address bar must not be able
+// to stop a node.
+func ReadRequestBody(w http.ResponseWriter, r *http.Request) ([]byte, bool) {
+	if r.Method != http.MethodPost {
+		http.Error(w, "control verbs are POST only", http.StatusMethodNotAllowed)
+		return nil, false
+	}
+	body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, MaxRequestBytes))
+	if err != nil {
+		// A body a worker could not READ is not an answer about any backend, so
+		// it must not look like one: 400 is what the frontend maps onto "the
+		// request was rejected", never onto "that model is gone".
+		http.Error(w, "reading the control request body: "+err.Error(), http.StatusBadRequest)
+		return nil, false
+	}
+	return body, true
+}
+
+// WriteUnknownPath answers a path under Prefix that this worker mounts no
+// handler for.
+//
+// The body says what happened, because a bare 404 through a tunnel is
+// indistinguishable from a proxy fault, and the frontend reads this exact
+// status as "this worker does not serve that verb"
+// (nodes.ErrWorkerControlUnsupported) rather than as the worker being absent.
+func WriteUnknownPath(w http.ResponseWriter, r *http.Request) {
+	http.Error(w, "unknown worker control path "+truncate(r.URL.Path, MaxEchoedPathBytes), http.StatusNotFound)
+}
+
+// truncate bounds a caller-controlled string that is about to be echoed.
+//
+// It cuts on a rune boundary. A byte-wise cut can split a multi-byte rune, and
+// the half rune then travels as a replacement character through every log and
+// UI that reads it; phase 2 shipped exactly that defect on a refusal reason and
+// pinned the rule afterwards. utf8.RuneStart is the same predicate the cluster
+// package uses for it, so the two are one rule rather than two hand-rolled
+// copies that can drift.
+func truncate(s string, max int) string {
+	if len(s) <= max {
+		return s
+	}
+	cut := max
+	for cut > 0 && !utf8.RuneStart(s[cut]) {
+		cut--
+	}
+	return s[:cut] + "…"
+}

docs/content/features/distributed-mode.md

@@ -139,7 +139,11 @@ Unlike the agent worker's API key and its NATS credential, a tunnel credential *
 
 A tunnel credential does not replace `LOCALAI_REGISTRATION_TOKEN`. Without one, node registration itself is unauthenticated, so anyone who can reach the frontend can register a worker and be issued a tunnel credential for it. How far that gets them depends on auto-approve: with auto-approve on the node is healthy at once and the credential works immediately; with it off the node is pending and the credential is inert until an admin approves, so approval is the real gate. LocalAI warns about the missing token at startup.
 
-Only **backend** nodes are issued one. An agent worker has no inbound surface for the tunnel to replace and no client for it, so minting one would widen the credential surface for nothing; its row keeps an empty tunnel credential and the tunnel route refuses it like any other node without one.
+Both **backend** and **agent** nodes are issued one. Earlier releases minted a credential only for backend nodes, because nothing dialled into an agent worker; the frontend now reaches an agent worker's MCP control verbs over a tunnel of its own, so an agent worker dials one too. A node whose type is neither has its tunnel credential cleared on every registration rather than merely not renewed, so what refuses it is an empty credential and not a second check that could drift from this one.
+
+An agent worker's tunnel carries only the `http` tag: it runs no backend processes, so it does not offer the `grpc` tag at all. Its control server binds `127.0.0.1` on a port chosen by the kernel and advertises it nowhere, so an agent worker still opens no inbound port.
+
+**An agent worker still requires `--nats-url`.** The tunnel is an addition, not a replacement: agent jobs, MCP execution, MCP CI jobs and `nodes.<id>.backend.stop` all still travel on NATS. What the tunnel changes today is that the frontend *can* reach an agent worker directly, which is what later releases move those verbs onto.
 
 The tunnel lands on exactly one frontend replica, and that replica records itself as the owner of the worker's connection in the `node_connections` table. When the socket dies the claim is dropped, but the row stays behind with no owner and a `disconnected_at` stamp, so a worker that is re-dialling the load balancer can be told from one that has never connected. The row is deleted once that departure is older than ten liveness windows (five minutes). If the replica stalls long enough for its peers to reap it, it re-claims the tunnels it still holds on a live session as soon as it re-registers, skipping any whose socket has already gone. That re-claim needs the replica to have an advertised address: without one it never had an instance row to begin with, and its tunnels are usable only by the replica holding them.
 
@@ -156,7 +160,9 @@ The worker holds the tunnel with one goroutine: it dials, serves the frontend's
 | Tag | Goes to | Target |
 |-----|---------|--------|
 | `grpc` | a backend process on this worker | the port; the host is discarded and only `127.0.0.1` is dialled, within the worker's own backend port range |
-| `http` | the worker's own file-transfer and backend-log server | ignored; there is one such server and only the worker knows where it bound |
+| `http` | the worker's own file-transfer and backend-log server (an agent worker's control server) | ignored; there is one such server and only the worker knows where it bound |
+
+An **agent worker** offers only the `http` row. It runs no backend processes, so the `grpc` tag has nothing to route to and a stream that names it is refused as an unknown tag.
 
 The `grpc` row is the security boundary of the tunnel, and it is worth being explicit about it. A tunnel terminates inside the worker process, so a stream arriving on it can reach anything the worker can reach; if the frontend could name the host, whoever holds the frontend end could make every worker in the fleet dial arbitrary addresses on its private network. The worker therefore builds the dial address from a constant `127.0.0.1` and a port it has validated, and the string from the wire never reaches the dialler at all. The port range is the one the worker's own allocator hands to backend processes, which by default runs to 65535; setting `LOCALAI_GRPC_MAX_PORT` narrows the allocator and this range together, and a worker with a known backend count should set it.
 
@@ -309,7 +315,7 @@ Registering against an upgraded frontend **clears** a node's `address` and `http
 
 A worker on this release opens **no inbound listener on a routable interface**. Its backend gRPC processes and its HTTP file-transfer server all bind loopback, and the frontend reaches both through the tunnel. Concretely:
 
-- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker also needs outbound access to `LOCALAI_NATS_URL`.
+- **No inbound firewall rule, published port, Service or Ingress is needed for a worker.** A serve-backend worker needs outbound access to the frontend URL (`LOCALAI_REGISTER_TO`), and nothing else - not even to NATS. An agent worker binds only loopback too, and needs outbound access to both `LOCALAI_REGISTER_TO` (registration, heartbeats and now its tunnel) and `LOCALAI_NATS_URL`.
 - **`LOCALAI_ADVERTISE_ADDR` and `LOCALAI_ADVERTISE_HTTP_ADDR` are gone.** There is nothing to advertise. Both are ignored if still set; remove them.
 - **`LOCALAI_ADDR` and `LOCALAI_SERVE_ADDR` are read for their port only.** The port is the base of the backend port range, and `port-1` is the HTTP file-transfer port. The host half names an interface nothing binds.
 - The node's `address` and `http_address` fields in `GET /api/nodes` are empty, and are cleared for nodes that reported them before the upgrade.
@@ -320,6 +326,8 @@ When a worker's tunnel goes, the frontend does **not** forget the worker. It rec
 
 That distinction exists because a worker loses its tunnel for entirely ordinary reasons. A frontend replica restarting during a rolling upgrade drops every tunnel it held, and each of those workers immediately re-dials the load balancer and lands on another replica. Treating that as "the worker is gone" would evict models mid-upgrade for a fleet that never actually went anywhere.
 
+This applies to **backend** nodes only. An agent worker holds a tunnel and therefore has a `node_connections` row that ages exactly like a backend worker's, and nothing acts on it: an agent worker's real work travels on NATS, so a lost tunnel says nothing about whether it can run a job. The scheduler never sees agent nodes at all (every placement query selects `node_type = 'backend'`), and the health monitor skips them explicitly.
+
 Three things read this. The **scheduler** reads it before it places a cold load: a worker whose departure has outlived the grace is skipped and marked unhealthy, so every other frontend replica stops choosing it too. **LRU eviction** reads it before it hands back the node it freed capacity on, because a node full enough to be an eviction target is exactly the node the placement selectors never offer, so the scheduler's own check never sees it. The **health monitor** reads it on every cycle, which is what covers the case the heartbeat cannot see. A worker's heartbeat says its supervisor is alive; it says nothing about whether anything here can reach that worker's backends, because those are reached over the tunnel. A worker that heartbeats with a permanently dead tunnel (a proxy that stopped upgrading WebSockets, a rotated registration credential, a reconnect loop longer than the grace) is therefore marked unhealthy too, rather than staying listed healthy while every request for a model loaded on it fails "no route to that worker".
 
 The log lines are `Scheduled node has no tunnel and its departure outlived the reconnect grace, marking unhealthy and re-scheduling`, `Eviction target has no tunnel and its departure outlived the reconnect grace, marking unhealthy and evicting again`, and `Node is heartbeating but its tunnel has been gone longer than the reconnect grace; marking unhealthy`. Each names the grace it used.
@@ -425,7 +433,7 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr
 
 ### NATS JWT authentication (recommended for production)
 
-**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`.
+**This section is about agent workers and the frontend.** A serve-backend worker opens no NATS connection, so none of it applies to one; its own credential is the tunnel token it gets at registration, and its control plane is authenticated by `LOCALAI_REGISTRATION_TOKEN`. An agent worker now has both: a NATS credential for everything still on the bus, and a tunnel token plus the same `LOCALAI_REGISTRATION_TOKEN` bearer check in front of its control server.
 
 By default, NATS connections are anonymous: any client that can reach port `4222` may publish the subjects still carried on it. Those are the agent-worker job subjects, MCP, and the frontend's own cross-replica events. `nodes.<id>.backend.install` and its nine siblings are **not** among them - they are HTTP routes on the worker's tunnel, see [The worker control plane](#the-worker-control-plane). Enable JWT auth to scope agent workers to their own subjects and give the frontend a dedicated service credential.
 

tests/e2e/distributed/cluster/cluster.go

@@ -48,11 +48,13 @@ type Options struct {
 	// alongside the backend workers.
 	//
 	// They exist so a spec can hold the two kinds of worker side by side in one
-	// cluster. An agent worker still speaks NATS and holds no tunnel at all,
-	// which is exactly the shape the tunnel-departure rules must not act on: it
-	// has no node_connections row, so its presence is PresenceUnknown forever.
-	// A spec that asserted only on backend workers could not tell "agent
-	// workers are unaffected" from "nothing here looks at them".
+	// cluster. An agent worker still speaks NATS, and now holds a tunnel of its
+	// own as well, which is exactly the shape the tunnel-departure rules must
+	// not act on: it gets a real node_connections row whose departure really
+	// does age past the grace, and demoting it for that would take out a fleet
+	// whose actual work travels on the bus. A spec that asserted only on
+	// backend workers could not tell "agent workers are unaffected" from
+	// "nothing here looks at them".
 	//
 	// They register through the same WorkerFrontendURL hook as backend workers,
 	// so a spec that puts a balancer in front of the fleet gets one for its
@@ -394,11 +396,12 @@ func (c *Cluster) startWorker(i int) (*Process, error) {
 // startAgentWorker starts agent worker i.
 //
 // It is `local-ai agent-worker`, not `local-ai worker`, and the difference is
-// the whole point of having it here: an agent worker REQUIRES a NATS URL, dials
-// no tunnel, and runs no backend, so it is the control for every rule this
-// phase added about a worker whose tunnel is gone. It binds nothing, so there
-// is no port to reserve and no readiness endpoint to wait on; a spec learns it
-// is up by finding it in the roster.
+// the whole point of having it here: an agent worker REQUIRES a NATS URL and
+// runs no backend, so it is the control for every rule this phase added about a
+// worker whose tunnel is gone. It dials a tunnel of its own, but binds only
+// loopback for the control plane behind it, so there is still no port to
+// reserve and no reachable readiness endpoint to wait on; a spec learns it is
+// up by finding it in the roster.
 func (c *Cluster) startAgentWorker(i int) (*Process, error) {
 	name := agentWorkerName(i)
 	cmd := exec.Command(c.opts.Binary, "agent-worker")

tests/e2e/distributed/cluster_control_test.go

@@ -382,8 +382,8 @@ func withReconnectGrace(d time.Duration) func(*cluster.Options) {
 	return func(o *cluster.Options) { o.ReconnectGrace = d }
 }
 
-// withAgentWorkers adds agent workers, which still speak NATS and hold no
-// tunnel, to a cluster.
+// withAgentWorkers adds agent workers, which still speak NATS for everything
+// this phase has not moved, to a cluster.
 func withAgentWorkers(n int) func(*cluster.Options) {
 	return func(o *cluster.Options) { o.AgentWorkers = n }
 }
@@ -653,9 +653,11 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L
 	// every reaper keyed on the heartbeat and the heartbeat was fine.
 	//
 	// The agent worker in the same cluster is the control for the other
-	// direction. It still speaks NATS, holds no tunnel and never will, so a
-	// rule that read "no tunnel" as "gone" would take the whole agent fleet
-	// down with it.
+	// direction. It holds a tunnel too now, and the balancer below blocks its
+	// dial exactly as it blocks the backend worker's, so its departure ages
+	// past the same grace. Its real work still travels on the bus, so a rule
+	// that read "no tunnel" as "gone" would take the whole agent fleet down
+	// with it.
 	It("stops reporting a heartbeating worker healthy once its tunnel is gone past the grace, and leaves agent workers alone", func() {
 		var balancer *frontendBalancer
 		c, dsn := startClusterOnFreshDB(2, 1, withBalancer(&balancer),
@@ -703,7 +705,9 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L
 			"the worker's heartbeat is stale, so it was demoted for being gone rather than for having no route")
 
 		// The agent worker, in the same cluster, under the same grace, on the
-		// same health monitor, is untouched. It holds no tunnel either.
+		// same health monitor, is untouched. Its tunnel is blocked by the same
+		// balancer, so this is a node whose departure really has outlived the
+		// grace and which must still not be demoted for it.
 		Consistently(func() string { return atSurvivor.statusOf(c.AgentWorkerName(0)) }, "20s", "2s").
 			Should(Equal("healthy"),
 				atSurvivor.explain("an agent worker was demoted by a rule about tunnels, and agent workers never hold one"))

tests/e2e/distributed/cluster_tunnel_test.go

@@ -383,6 +383,38 @@ var _ = Describe("Worker tunnel end to end", Label("Distributed"), Label("Cluste
 			"a worker with no advertised address must still serve inference over its tunnel")
 	})
 
+	// Scenario 1b. The AGENT worker, which used to be gated out of tunnels at
+	// the credential mint site and now dials one like any other node.
+	//
+	// A wrong implementation is silent in both directions. If the mint gate is
+	// still closed, the agent worker registers, heartbeats and works exactly as
+	// before, with its tunnel dial refused 401 forever in a log nothing reads;
+	// no other spec in this suite would notice. If the absence rules were
+	// widened to follow, the agent worker would instead start being demoted for
+	// a tunnel its real work does not travel on.
+	It("holds a tunnel for an AGENT worker, and still does not judge it by one", func() {
+		c, dsn := startClusterOnFreshDB(1, 0, withAgentWorkers(1))
+		client := controlSession(c)
+
+		probe := newRosterProbe(c, client, 0)
+		Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll).
+			Should(ContainElement(c.AgentWorkerName(0)), probe.describe)
+		nodeID := probe.idOf(c.AgentWorkerName(0))
+		Expect(nodeID).ToNot(BeEmpty())
+
+		// The claim: a live replica holds this agent worker's tunnel. Read
+		// through the production Owner query, which joins against live
+		// instances, so a row left by a dead replica is not an owner.
+		owners := newTunnelOwners(openClusterDB(dsn))
+		Eventually(func() int { return owners.ownerIndexOf(c, 1, nodeID) }, tunnelOwnershipTimeout, tunnelOwnershipPoll).
+			Should(Equal(0), owners.describe)
+
+		// And it is still an agent worker: holding a tunnel changed nothing
+		// about how this deployment decides whether it is present.
+		Consistently(func() string { return probe.statusOf(c.AgentWorkerName(0)) }, "10s", "2s").
+			Should(Equal("healthy"), probe.describe)
+	})
+
 	// Scenario 2. With N replicas behind round robin this is (N-1)/N of
 	// production traffic. A wrong implementation answers it by dialling the
 	// worker from the replica that took the request, which works on one host

tests/e2e/distributed/node_lifecycle_test.go

@@ -177,7 +177,9 @@ var _ = Describe("Node Backend Lifecycle over the worker control plane", Label("
 		})
 
 		// The one node subject left, and it is addressed only to AGENT workers:
-		// they hold no tunnel and subscribe to it to drop cached MCP sessions.
+		// they subscribe to it to drop cached MCP sessions. They serve the same
+		// verb on their tunnel as well; this subject survives because the
+		// PUBLISHER has not moved onto that route yet.
 		It("should keep the agent worker's backend.stop subject", func() {
 			Expect(messaging.SubjectNodeBackendStop("node-abc")).To(Equal("nodes.node-abc.backend.stop"))
 		})