feat(distributed): make MCP execution and discovery a selection

mcp.tools.execute and mcp.discovery were the only NATS subjects that
combined a queue group with a reply, and no carrier in this design
provides both. They never needed one: a queue group is a way of choosing
a subscriber, and choosing is a query.

The frontend now lists the approved, non-draining agent nodes, asks the
node_connections table in one joined statement which of those tunnels a
live replica holds, prefers one this replica holds so the call skips the
relay hop, and issues an ordinary control RPC on the path task 4 already
mounted. A peer-held tunnel is reached through the relay. That is a
choice a broker's hidden balancing could not make.

The selection reads presence and nothing else. It is filtered only on
node type and on the two statuses an operator controls, never on a health
verdict written on another clock, because refusing a worker that is
connected and answering is the same defect as picking one that is gone.
An empty fleet answers ErrNoAgentWorker, which is deliberately neither
ErrWorkerUnroutable nor anything cluster.IsWorkerAnswer accepts: nothing
was asked of any worker, so no reap guard may act on it.

A reply carrying an Error is the worker's own answer and is returned
unchanged; it is never offered to a second worker, which would turn "this
MCP server rejected your arguments" into "the fleet is broken" and could
run a tool twice. A call that never reached a worker is retried against a
different pick, at most three times, and whatever error is finally
returned is returned unwrapped so its identity survives the loop.

MCP prompts and resources now answer 501 in distributed mode instead of
an empty 200. They are served only from sessions the frontend holds, and
in distributed mode it holds none. That gap predates the removal of the
bus and is not closed by it; this only stops it being silent.

Agent workers keep every other subject, including nodes.<id>.backend.stop.
Their minted JWT loses the two MCP subjects and keeps a non-empty allow
list, because NATS reads an empty one as no restriction at all.

Assisted-by: Claude Opus 5 [claude-code]
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>

core/cli/agent_worker.go

@@ -178,11 +178,14 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 
 	// The tunnel, and the loopback control plane behind it.
 	//
-	// ADDED to this worker rather than swapping anything out: every verb below
-	// still arrives on NATS, and will until the tasks that move them land. What
-	// this buys today is that the frontend can reach an agent worker by RPC at
-	// all, on the same carrier and with the same failure vocabulary a backend
-	// worker already uses, without the agent worker opening an inbound port.
+	// This is now the ONLY way MCP tool execution and discovery reach this
+	// worker: their queue-group subjects are gone, because a queue group was
+	// only ever a way of SELECTING a worker and the frontend makes that
+	// selection itself (nodes.AgentSelector). The remaining verbs below still
+	// arrive on NATS and will until the tasks that move them land.
+	//
+	// The worker opens no inbound port for any of it: it dials out and the
+	// control plane rides the tunnel it holds.
 	//
 	// The credential is read through credMgr rather than captured from res,
 	// because every re-registration the manager performs ROTATES it and a
@@ -191,9 +194,9 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 	//
 	// It is started AFTER registration, which is what supplies both the node
 	// identity the dial names and the credential it presents, and BEFORE the
-	// NATS subscriptions, so that a frontend that reaches this worker over the
-	// tunnel finds its verbs mounted rather than a 404 it would read as a
-	// version skew.
+	// remaining NATS subscriptions, so that a frontend that reaches this worker
+	// over the tunnel finds its verbs mounted rather than a 404 it would read
+	// as a version skew.
 	agentCtl, err := agentworker.Start(shutdownCtx, agentworker.Options{
 		FrontendURL:  cmd.RegisterTo,
 		NodeID:       nodeID,
@@ -244,19 +247,6 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error {
 		return fmt.Errorf("starting dispatcher: %w", err)
 	}
 
-	// Subscribe to MCP tool execution requests (load-balanced across workers).
-	// The frontend routes model-level MCP tool calls here via NATS request-reply.
-	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPToolExecute, messaging.QueueAgentWorkers,
-		replyOverNATS(messaging.SubjectMCPToolExecute, serveMCPToolRequest)); err != nil {
-		return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPToolExecute, err)
-	}
-
-	// Subscribe to MCP discovery requests (load-balanced across workers).
-	if _, err := natsClient.QueueSubscribeReply(messaging.SubjectMCPDiscovery, messaging.QueueAgentWorkers,
-		replyOverNATS(messaging.SubjectMCPDiscovery, serveMCPDiscoveryRequest)); err != nil {
-		return fmt.Errorf("subscribing to %s: %w", messaging.SubjectMCPDiscovery, err)
-	}
-
 	// Subscribe to MCP CI job execution (load-balanced across agent workers).
 	// In distributed mode, MCP CI jobs are routed here because the frontend
 	// cannot create MCP sessions (e.g., stdio servers using docker).
@@ -456,23 +446,6 @@ func encodeMCPReply(resp any) (json.RawMessage, error) {
 	return out, nil
 }
 
-// replyOverNATS adapts one of the serve* functions to a NATS request-reply
-// subscription, so the bus carries exactly the bytes the tunnel does.
-func replyOverNATS(subject string, serve func(context.Context, json.RawMessage) (json.RawMessage, error)) func([]byte, func([]byte)) {
-	return func(data []byte, reply func([]byte)) {
-		out, err := serve(context.Background(), data)
-		if err != nil {
-			// Nothing is sent. A requester on the bus reads that as a timeout,
-			// which is the closest the carrier has to "this worker did not
-			// answer"; inventing a reply body here would put a failure to serve
-			// into the bucket reserved for the worker's own verdict.
-			xlog.Error("Agent worker could not serve a bus request", "subject", subject, "error", err)
-			return
-		}
-		reply(out)
-	}
-}
-
 // handleMCPCIJob processes an MCP CI job on the agent worker.
 // The agent worker can create MCP sessions (has docker) and call the LocalAI API for inference.
 func handleMCPCIJob(shutdownCtx context.Context, data []byte, apiURL, apiToken string, natsClient messaging.MessagingClient, jobTimeout time.Duration) {

core/cli/agent_worker_test.go

@@ -11,20 +11,16 @@ import (
 	"github.com/mudler/LocalAI/core/services/messaging"
 )
 
-// The agent worker answers its MCP verbs on two carriers at once: the NATS
-// subject it has always answered, and the control route on the tunnel it now
-// holds. These specs pin the two properties that stops those drifting apart.
+// The agent worker answers its MCP verbs on ONE carrier now: the control route
+// on the tunnel it holds. The queue-group subjects these used to arrive on are
+// gone, because a queue group was only ever a way of SELECTING a worker, and
+// the frontend now makes that selection itself (nodes.AgentSelector).
 //
-// The first is that there is ONE implementation. A frontend that reaches a
-// worker over the bus and one that reaches the same worker over its tunnel must
-// get the same bytes, because during this migration both are live and which one
-// is used is not a decision anybody makes deliberately.
-//
-// The second is the split between an ANSWER and a FAILURE TO SERVE. A tool that
-// ran and failed is the worker's own verdict and travels inside the reply; a
-// verb this worker could not serve at all is a Go error, which becomes a
-// non-2xx over the tunnel and silence on the bus, and which nothing may read as
-// evidence about anything.
+// What these specs pin is the split between an ANSWER and a FAILURE TO SERVE. A
+// tool that ran and failed is the worker's own verdict and travels inside the
+// reply, on a 200; a verb this worker could not serve at all is a Go error,
+// which becomes a non-2xx, and which nothing may read as evidence about
+// anything.
 var _ = Describe("The agent worker's MCP verbs", func() {
 	It("answers a tool request it could not decode, rather than failing to serve it", func() {
 		// The decode happened on this worker and its outcome is something the
@@ -48,36 +44,6 @@ var _ = Describe("The agent worker's MCP verbs", func() {
 		Expect(resp.Error).To(ContainSubstring("unmarshal error"))
 	})
 
-	It("puts on the bus exactly the bytes the tunnel route returns", func() {
-		// The one property that keeps the two carriers honest. A second
-		// implementation for the bus is how a deployment ends up behaving
-		// differently depending on which one a frontend happened to pick.
-		request := json.RawMessage(`{"tool_name":`)
-		overTunnel, err := serveMCPToolRequest(context.Background(), request)
-		Expect(err).ToNot(HaveOccurred())
-
-		sent := make(chan []byte, 1)
-		replyOverNATS("mcp.tools.execute", serveMCPToolRequest)(request, func(b []byte) { sent <- b })
-
-		var overBus []byte
-		Eventually(sent).Should(Receive(&overBus))
-		Expect(string(overBus)).To(Equal(string(overTunnel)))
-	})
-
-	It("sends nothing on the bus when the verb could not be served", func() {
-		// A requester reads the silence as a timeout, which is the closest the
-		// bus has to "this worker did not answer". Inventing a reply body would
-		// put a failure to serve into the bucket reserved for the worker's own
-		// verdict, which is the collapse this whole phase exists to prevent.
-		sent := make(chan []byte, 1)
-		failing := func(context.Context, json.RawMessage) (json.RawMessage, error) {
-			return nil, context.DeadlineExceeded
-		}
-
-		replyOverNATS("mcp.tools.execute", failing)(json.RawMessage(`{}`), func(b []byte) { sent <- b })
-
-		Expect(sent).ToNot(Receive())
-	})
 })
 
 var _ = Describe("The agent worker's backend stop", func() {