From 730d259976761e5f2b9d038d36aa7ea29234dd32 Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Sat, 5 Sep 2026 03:27:17 +0000 Subject: [PATCH] chore(distributed): take the nats-io modules out of the build Distributed mode has not dialled a message broker since the control plane moved onto the workers' own outward tunnels and every fan-out family moved onto PostgreSQL LISTEN/NOTIFY. What was left was the dependency itself, and the code that existed only to feed it. Dropped from go.mod: nats-io/jwt/v2, nats-io/nats.go, nats-io/nkeys, nats-io/nuid and testcontainers-go/modules/nats, along with the fourteen indirect requires that only the NATS testcontainer pulled in. go.sum carries no nats line either, so the removal is not the partial kind where the require goes and the checksum stays. Deleted with them: pkg/natsauth in full, the broker client's remaining options and TLS files, the per-node JWT minting on both the register and the approve path, and the natsauth.Config parameter threaded through the node routes. The credential manager is renamed and stripped rather than deleted, because it still holds the tunnel token that every re-registration rotates. The bus flags stay accepted and ignored, and are now hidden, on every command that had them, so an existing unit file, compose file or Helm values file still starts on the day of the upgrade. What is not kept is the validation that REQUIRED one: a distributed frontend started with no bus URL is no longer fatal. The TLS paths lose type:"existingfile" deliberately, so a certificate deleted along with the broker cannot fail a startup. One operator-visible behaviour change: --nats-require-auth no longer makes an agent worker wait through admin approval. Ask for that wait with --distributed-require-auth, which already implied it. It is documented in the migration section and pinned from both sides. A deployment now needs PostgreSQL and the frontends' own HTTP listener, and nothing else. coverage-baseline.txt moves from 54.2 to 62.0. Assisted-by: Claude Opus 5 [claude-code] Signed-off-by: Ettore Di Giacinto --- .agents/building-and-testing.md | 14 +- .agents/localai-assistant-mcp.md | 2 +- CONTRIBUTING.md | 6 +- Makefile | 14 +- core/application/cache_fanout_wiring.go | 29 +- core/application/cache_fanout_wiring_test.go | 10 +- core/application/distributed.go | 37 +-- core/application/distributed_test.go | 9 +- core/application/fanout_wiring.go | 13 +- core/application/startup.go | 6 +- core/cli/agent_worker.go | 76 +++-- core/cli/agent_worker_test.go | 73 +++++ core/cli/cli.go | 2 +- core/cli/run.go | 72 ++--- core/cli/run_bus_flags_test.go | 172 +++++++++++ core/cli/worker/worker_vllm.go | 2 +- core/cli/workerregistry/client.go | 34 +-- core/cli/workerregistry/client_test.go | 41 ++- core/cli/workerregistry/credentials.go | 188 ++++-------- core/cli/workerregistry/credentials_test.go | 150 ++++------ core/config/distributed_config.go | 139 +-------- core/config/distributed_config_test.go | 61 ++-- core/http/app.go | 11 +- core/http/auth/helpers_test.go | 2 - core/http/endpoints/cluster/connect.go | 15 +- core/http/endpoints/localai/nodes.go | 32 +- core/http/endpoints/localai/nodes_test.go | 77 +++-- .../nodes-per-node-backend-actions.spec.js | 2 +- core/http/routes/nodes.go | 9 +- core/http/routes/openresponses.go | 8 +- core/http/routes/openresponses_wiring_test.go | 9 +- core/services/finetune/service.go | 5 +- core/services/messaging/client.go | 282 ------------------ core/services/messaging/interfaces.go | 19 +- core/services/messaging/interfaces_test.go | 68 ++--- core/services/messaging/nats_absent_test.go | 67 +++++ core/services/messaging/options.go | 34 --- core/services/messaging/tls.go | 68 ----- core/services/messaging/tls_test.go | 25 -- core/services/pgbus/bus.go | 7 +- core/services/quantization/service.go | 5 +- coverage-baseline.txt | 2 +- docs/content/features/distributed-mode.md | 20 +- docs/content/reference/cli-reference.md | 20 +- go.mod | 21 +- go.sum | 47 +-- pkg/natsauth/config.go | 66 ---- pkg/natsauth/decode.go | 16 - pkg/natsauth/mint.go | 59 ---- pkg/natsauth/mint_test.go | 72 ----- pkg/natsauth/permissions.go | 67 ----- pkg/natsauth/permissions_coverage_test.go | 164 ---------- scripts/build/healthcheck_test.sh | 5 +- tests/e2e/distributed/cluster/cluster.go | 80 +++-- tests/e2e/distributed/cluster/cluster_test.go | 12 +- tests/e2e/distributed/cluster/failure.go | 2 +- .../e2e/distributed/cluster_baseline_test.go | 13 +- tests/e2e/distributed/cluster_control_test.go | 11 +- .../distributed/finetune_distributed_test.go | 8 +- tests/e2e/distributed/foundation_test.go | 32 +- .../distributed/gallery_distributed_test.go | 4 +- tests/e2e/distributed/job_dispatch_test.go | 4 +- tests/e2e/distributed/model_routing_test.go | 4 +- .../distributed/skills_distributed_test.go | 6 +- tests/e2e/distributed/sse_routes_test.go | 4 +- tests/e2e/distributed/testhelpers_test.go | 8 - website/layouts/index.html | 2 +- 67 files changed, 965 insertions(+), 1679 deletions(-) create mode 100644 core/cli/run_bus_flags_test.go delete mode 100644 core/services/messaging/client.go create mode 100644 core/services/messaging/nats_absent_test.go delete mode 100644 core/services/messaging/options.go delete mode 100644 core/services/messaging/tls.go delete mode 100644 core/services/messaging/tls_test.go delete mode 100644 pkg/natsauth/config.go delete mode 100644 pkg/natsauth/decode.go delete mode 100644 pkg/natsauth/mint.go delete mode 100644 pkg/natsauth/mint_test.go delete mode 100644 pkg/natsauth/permissions.go delete mode 100644 pkg/natsauth/permissions_coverage_test.go diff --git a/.agents/building-and-testing.md b/.agents/building-and-testing.md index 3f614d039..ae1c9a4f6 100644 --- a/.agents/building-and-testing.md +++ b/.agents/building-and-testing.md @@ -48,17 +48,17 @@ Rules (both gates): ## Distributed-mode test suites -Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL, NATS), split by a Ginkgo label: +Two suites cover distributed mode (frontend replicas, worker nodes, PostgreSQL), split by a Ginkgo label. Neither starts a message broker: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and the `nats-io` modules are not in the build. -- `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. ~240 specs, ~75s. -- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 6 specs, about 8m30s measured over three consecutive runs (509.1s / 509.8s / 512.3s, so 8m29s to 8m32s). +- `make test-e2e-distributed` runs `Distributed && !VLLMMultinode && !Cluster` over `./tests/e2e/distributed` recursively. Services are wired directly into the test binary. 231 specs in about 2m20s (measured 133.3s for the main suite plus the 8 argument-validation specs in the subpackage). +- `make test-e2e-cluster` runs `Cluster` and spawns real `local-ai` child processes through the `tests/e2e/distributed/cluster` helper package. It **builds `./local-ai` itself** (the `e2e-binary` target), so you do not run `make build` first and cannot forget to. 21 specs, 802.5s of specs in a 13m26s target run (one measurement; the suite grew from the 6 specs and roughly 509s recorded earlier, so budget on the newer number). Both jobs live in `.github/workflows/tests-e2e-distributed.yml`, with `timeout-minutes: 45` each. They trigger on pull requests *and* on every push to `master`; the `paths-ignore` filter (see [.agents/ci-caching.md](ci-caching.md)) sits on the pull-request trigger only, so a master push always runs both. They are advisory only because `master` carries no branch protection, which is a repository setting and not a YAML key: `continue-on-error: true` would flip the run's *conclusion* to success and hide the failure, so it is not used. -- **Containers are suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a NATS (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. (213 rather than the ~240 above: the larger number is everything the label filter selects, the smaller one is just the specs that call `SetupInfra`.) Containers now start once in `BeforeSuite` and each spec gets its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. -- **Consequence for new specs:** the NATS bus is now *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. -- **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container pair, keeping NATS subjects isolated per process. A single shared NATS across parallel processes would let specs on different processes see each other's messages on the same subject. -- **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary, no PostgreSQL and no NATS, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). +- **The container is suite-scoped, not spec-scoped.** `SetupInfra` used to start a PostgreSQL (~10s) and a broker (~3.5s) per spec. Across the 213 specs behind it that was roughly **48 minutes of pure container startup per run**, which is why this suite was never in CI. (213 rather than the ~240 above: the larger number is everything the label filter selects, the smaller one is just the specs that call `SetupInfra`.) The broker container is gone entirely, and the PostgreSQL one now starts once in `BeforeSuite` with each spec getting its own database via `CREATE DATABASE` (~67ms), which is what the `dbName` argument was always describing. Adding a spec needs no change: call `SetupInfra("some-name")` as before, the name is a prefix and a counter keeps it unique. +- **Consequence for new specs:** a fan-out carrier is *shared* within a Ginkgo process, so a wildcard subscriber can observe another spec's traffic. Filter assertions on an identifier your spec owns (a node ID, a job ID) instead of counting everything on `jobs.*.progress`, and verify the spec with `--randomize-all`. +- **`BeforeSuite`, not `SynchronizedBeforeSuite`.** Under `ginkgo -p` each process then gets its own container, keeping each process's carrier isolated. A single shared database across parallel processes would let specs on different processes see each other's messages on the same subject. +- **The label split.** The 8 argument-validation specs under `tests/e2e/distributed/cluster/` carry `Label("Distributed")` only, on purpose: they need no binary and no PostgreSQL, so they belong in the fast job. That is why `test-e2e-distributed` keeps `-r` (it must reach the subpackage) and `test-e2e-cluster` deliberately does **not** (the subpackage is out of its scope). - **The binary is built, and a stale one is refused.** `test-e2e-cluster` used to take `./local-ai` as given and only check that a file by that name existed. An edit that was never rebuilt therefore ran the entire suite against an older build while reporting on the working tree, which surfaced once as a spec failing on caller line numbers that matched no line in any source file. Two things close it. The target depends on `e2e-binary`, which is `go build -o local-ai ./cmd/local-ai` and not `make build` (that one also builds the React UI, which this suite never touches and CI stubs). And `localAIBinary` in the harness **fails**, on a developer box as well as under CI, when the binary is older than the newest non-test Go source in the tree, which covers a run started with `LOCALAI_E2E_BINARY` or by invoking ginkgo directly. `_test.go` files are excluded from that scan on purpose: they compile into the ginkgo suite and never into `local-ai`, so counting them would fire on every edit to the spec being run. Set `LOCALAI_E2E_BINARY` to opt out of the build; there is no opt-out of the staleness check, because a rebuild is the remedy. - **`--fail-on-empty` is load-bearing on both targets.** Ginkgo exits 0 when a label filter selects nothing, so without it a refactor that renames or drops `Label("Cluster")` leaves the target reporting "Test Suite Passed" having started no cluster at all. `LOCALAI_E2E_REQUIRE_BINARIES` does not cover this case: it only fires inside a spec that is actually running. - **The binary gate.** `localAIBinary()` and `mockBackendBinary()` **fail** rather than skip when `CI` is set, or when `LOCALAI_E2E_REQUIRE_BINARIES` is truthy; `LOCALAI_E2E_REQUIRE_BINARIES=0` (also `off`, `no`, `n`, `disabled`, and anything `strconv.ParseBool` reads as false) forces skipping even under CI. **Any value that parses as neither reads as ON**, not off: setting the variable to something meaningless means someone meant to turn the gate on, and reading it as false would quietly restore the silent skip the flag exists to remove. The whole polarity is deliberate, because in CI a skipped cluster spec is indistinguishable from a passing one: Ginkgo exits 0 on skips. Locally a missing binary still just skips, since `CI` is unset in an ordinary shell. diff --git a/.agents/localai-assistant-mcp.md b/.agents/localai-assistant-mcp.md index bd391164c..57d51fbd6 100644 --- a/.agents/localai-assistant-mcp.md +++ b/.agents/localai-assistant-mcp.md @@ -94,4 +94,4 @@ The user chose KISS. Every mutating tool has a safety rule (`prompts/10_safety.m ## Distributed mode -The in-memory MCP server runs only on the head node (where the chat handler runs). `inproc.Client` wraps services that are already distributed-aware (`GalleryService` coordinates with workers; `ListNodes` reads the NATS-populated registry). No NATS routing of MCP tools — the admin surface lives on the head, period. +The in-memory MCP server runs only on the head node (where the chat handler runs). `inproc.Client` wraps services that are already distributed-aware (`GalleryService` coordinates with workers; `ListNodes` reads the PostgreSQL-backed node registry). MCP tools are not routed anywhere: the admin surface lives on the head, period. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f48c7c4a3..1d469152a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -267,16 +267,16 @@ make test-e2e ### Running distributed-mode tests -Distributed mode (several frontend replicas, worker nodes, PostgreSQL and NATS) has two suites. Both bring up their PostgreSQL and NATS with testcontainers, so Docker has to be available: +Distributed mode (several frontend replicas, worker nodes and PostgreSQL) has two suites. Both bring up their PostgreSQL with testcontainers, so Docker has to be available. There is no message broker to bring up: a distributed deployment needs PostgreSQL and the frontends' own HTTP listener, and nothing else. ```bash make test-e2e-distributed # in-process: services wired directly into the test binary make test-e2e-cluster # process-level: real local-ai child processes ``` -`make test-e2e-distributed` is the fast one (around 240 specs in roughly 75 seconds). It starts one PostgreSQL and one NATS for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. +`make test-e2e-distributed` is the fast one (231 specs in roughly 2m20s). It starts one PostgreSQL for the whole run and gives each spec its own database. It runs each spec exactly once, with no retry: `DISTRIBUTED_TEST_FLAKES` defaults to 1 and feeds ginkgo's `--flake-attempts`, which counts *total attempts*, not retries. That is deliberately below the repo-wide `TEST_FLAKES=5`, because this suite exists to catch nondeterministic cluster behaviour and a retry hides exactly the failure it is meant to catch. Raise it locally when bisecting something unrelated. -`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 8m30s (measured 509.1s / 509.8s / 512.3s over three consecutive runs): three of its six specs wait out real staleness and health-check windows. It needs a built binary and the mock backend: +`make test-e2e-cluster` runs `local-ai` as real child processes, one per frontend replica and one per worker, so a spec can kill a replica and assert what the survivors do. Budget about 13m30s (one measurement: 802.5s of specs in a 13m26s target run, which includes building the binary): several of its 21 specs wait out real staleness and health-check windows. An earlier record of 6 specs and roughly 509s is superseded. It needs a built binary and the mock backend: ```bash make build build-mock-backend diff --git a/Makefile b/Makefile index 48bde7931..3fccce3f8 100644 --- a/Makefile +++ b/Makefile @@ -347,8 +347,10 @@ run-e2e-aio: protogen-go # you are bisecting something unrelated. DISTRIBUTED_TEST_FLAKES?=1 -# Distributed architecture e2e (PostgreSQL + NATS via testcontainers). -# Includes NatsJWT specs (JWT-enabled NATS). Requires Docker. +# Distributed architecture e2e (PostgreSQL via testcontainers). Requires Docker. +# There is no broker container and no NatsJWT label any more: a distributed +# deployment needs PostgreSQL and the frontends' own HTTP listener, and this +# suite stands up exactly that. # VLLMMultinode is excluded here; use test-e2e-vllm-multinode for that. # Cluster is excluded too and runs in test-e2e-cluster below, which needs a # built binary. The argument-validation specs under tests/e2e/distributed/cluster @@ -364,7 +366,7 @@ DISTRIBUTED_TEST_FLAKES?=1 # in this file that does exec a prebuilt binary is test-e2e-cluster below, and # the reason it now BUILDS it is written there. test-e2e-distributed: protogen-go - @echo 'Running distributed e2e tests (label Distributed, incl. NatsJWT)' + @echo 'Running distributed e2e tests (label Distributed)' $(GOCMD) run github.com/onsi/ginkgo/v2/ginkgo --label-filter='Distributed && !VLLMMultinode && !Cluster' --fail-on-empty --flake-attempts $(DISTRIBUTED_TEST_FLAKES) --timeout=40m -v -r ./tests/e2e/distributed # The local-ai binary the cluster e2e execs as its frontend and worker @@ -390,7 +392,7 @@ e2e-binary: protogen-go fi # Cluster e2e: runs local-ai as real child processes (frontend replicas + -# workers) against PostgreSQL and NATS, and kills them to assert failover. +# workers) against PostgreSQL, and kills them to assert failover. # It BUILDS that binary rather than checking that a file by that name exists, # and that is a correctness fix rather than a convenience. This target used to # take ./local-ai as given, so an edit to core/ that was never rebuilt left the @@ -407,8 +409,8 @@ e2e-binary: protogen-go # the harness makes every other path loud. # # The argument-validation specs in tests/e2e/distributed/cluster deliberately -# stay in test-e2e-distributed above: they need no binary, no PostgreSQL and no -# NATS, so no -r here and that package is simply out of scope. +# stay in test-e2e-distributed above: they need no binary and no PostgreSQL, so +# no -r here and that package is simply out of scope. # # --fail-on-empty is load-bearing, not tidiness. Ginkgo exits 0 when a label # filter selects nothing, so without it a refactor that renames or drops diff --git a/core/application/cache_fanout_wiring.go b/core/application/cache_fanout_wiring.go index a351a8efc..bbe4d592a 100644 --- a/core/application/cache_fanout_wiring.go +++ b/core/application/cache_fanout_wiring.go @@ -18,22 +18,21 @@ import ( // The four process-lifetime caches, each wired onto the broadcast carrier by // one function here, and none of them by a call site naming a carrier. // -// Two are methods on DistributedServices and take NO carrier at all, because -// their call sites hold the whole struct and could otherwise reach for the -// cancel carrier. -// The other two run inside initDistributed before that struct exists, so they -// take the CONCRETE *pgbus.Bus rather than messaging.Broadcaster. +// Two are methods on DistributedServices and take NO carrier at all. The other +// two run inside initDistributed before that struct exists, so they take the +// CONCRETE *pgbus.Bus rather than messaging.Broadcaster. // -// Concrete on purpose. *messaging.Client satisfies messaging.Broadcaster just -// as well as the carrier does, so an interface parameter at these sites lets a -// caller hand over the NATS client that is also in scope: it compiles, it -// starts, it publishes, it is delivered, onto a carrier the deployment is being -// taken off, and nothing fails until NATS goes away. With the concrete type -// that mistake is a build error rather than a deployment that looks healthy. +// Concrete on purpose, and the purpose has outlived what prompted it. It was +// written against a SECOND carrier that was in scope at these call sites and +// satisfied the same interface, so an interface parameter let a caller hand +// over the wrong one: it compiled, started, published and was delivered, to +// nobody the deployment would still be listening on. That carrier went with the +// message broker and there is one left. The narrowing stays because a second +// carrier is exactly what it guards against, and widening these parameters back +// would give the guard away silently on the day another one arrives. // // The adopters themselves still take the interface, so their own specs drive -// them with an in-memory double. The narrowing is only here, where the wrong -// carrier is in scope. +// them with an in-memory double. The narrowing is only here. // wireGalleryBroadcasts puts the gallery service's progress, cancel and // cache-invalidation traffic on the carrier, and opens the wildcard @@ -62,8 +61,8 @@ func (ds *DistributedServices) wireGallery(gs *galleryop.GalleryService) error { // // Exported, unlike its siblings, because the OpCache is built in the HTTP layer // rather than in initDistributed. It takes neither a carrier nor a store: both -// come off this struct, so the HTTP layer cannot pass the NATS client that -// hangs off it beside them. +// come off this struct, so the HTTP layer names no carrier and cannot pass one +// that is merely in scope beside them. // // A hydrate failure is the OpCache's own business and is logged there; a // subscribe failure is returned, because a cache that hydrated and did not diff --git a/core/application/cache_fanout_wiring_test.go b/core/application/cache_fanout_wiring_test.go index 0cc078230..45cf857da 100644 --- a/core/application/cache_fanout_wiring_test.go +++ b/core/application/cache_fanout_wiring_test.go @@ -25,10 +25,12 @@ import ( // // Every case here wires the cache on busA and drives it from busB. A cache // talking to itself would pass with the wiring pointed at any carrier at all, -// which is the defect these exist to catch: the NATS client is in scope at -// three of the four call sites and satisfies the same interface, so a site left -// holding it publishes successfully and is delivered, to nobody the deployment -// will still be listening on. +// which is the defect these exist to catch: a site holding a carrier other than +// the deployment's publishes successfully and is delivered, to nobody the +// deployment will still be listening on. The second carrier that made that +// concrete went with the message broker, and these stay because they are what +// would catch it on the day a third arrives: busB IS the other carrier, so the +// defect is exercised rather than argued about. var _ = Describe("wiring the process-lifetime caches onto the broadcast carrier", func() { var ( ctx context.Context diff --git a/core/application/distributed.go b/core/application/distributed.go index 37dbbb3ac..6ad0726f6 100644 --- a/core/application/distributed.go +++ b/core/application/distributed.go @@ -31,21 +31,21 @@ import ( // DistributedServices holds all services initialized for distributed mode. type DistributedServices struct { - Store storage.ObjectStore - Registry *nodes.NodeRegistry - Router *nodes.SmartRouter - Health *nodes.HealthMonitor - Reconciler *nodes.ReplicaReconciler - JobStore *jobs.JobStore - Dispatcher *jobs.Dispatcher - AgentStore *agents.AgentStore - AgentBridge *agents.EventBridge - DistStores *distributed.Stores - FileMgr *storage.FileManager - FileStager nodes.FileStager - ModelAdapter *nodes.ModelRouterAdapter - Unloader *nodes.RemoteUnloaderAdapter - ModelCleanup *nodes.ModelCleanupService + Store storage.ObjectStore + Registry *nodes.NodeRegistry + Router *nodes.SmartRouter + Health *nodes.HealthMonitor + Reconciler *nodes.ReplicaReconciler + JobStore *jobs.JobStore + Dispatcher *jobs.Dispatcher + AgentStore *agents.AgentStore + AgentBridge *agents.EventBridge + DistStores *distributed.Stores + FileMgr *storage.FileManager + FileStager nodes.FileStager + ModelAdapter *nodes.ModelRouterAdapter + Unloader *nodes.RemoteUnloaderAdapter + ModelCleanup *nodes.ModelCleanupService // Bus is the deployment's fan-out carrier, riding the auth database's // PostgreSQL rather than a message broker. Every cross-replica family the @@ -524,8 +524,11 @@ func initDistributed(cfg *config.ApplicationConfig, authDB *gorm.DB, configLoade } idx := prefixcache.NewIndex(prefixCfg) // S4. One call puts this replica's observations and its peers' on the - // same carrier, and it takes the concrete carrier so the NATS client - // still in scope here cannot be handed to it by accident. + // same carrier, and it takes the CONCRETE carrier so that no other + // thing satisfying messaging.Broadcaster can be handed to it by + // accident. There is no second carrier in this scope to hand over any + // more; the type stays narrow so there is still none on the day one is + // added. See cache_fanout_wiring.go for the whole argument. prefixSync, err := wirePrefixCacheBroadcasts(bus, prefixCfg, idx) if err != nil { return nil, err diff --git a/core/application/distributed_test.go b/core/application/distributed_test.go index 731b86cf2..bedd6dfb8 100644 --- a/core/application/distributed_test.go +++ b/core/application/distributed_test.go @@ -119,11 +119,12 @@ var _ = Describe("shutting the distributed services down", func() { // It was five field reads before this: the fine-tune service, the quantization // service, the agent-task setter on two startup paths, the per-user services // manager and the Open Responses store. Every one of them takes a -// messaging.Broadcaster, which *messaging.Client satisfies too, so a site left -// holding the struct's NATS field compiled, started, published and was +// messaging.Broadcaster, which the broker client on the same struct satisfied +// too, so a site left holding that field compiled, started, published and was // delivered onto a carrier only agent workers read, and nothing failed until -// NATS did. Collapsing the choice into one function is what makes it a fact -// these specs can hold. +// the broker went away. That field and its type are now gone; collapsing the +// choice into one function is what keeps it a fact these specs can hold, rather +// than a property that lasted only as long as there was one carrier. var _ = Describe("handing the broadcast carrier to its adopters", func() { It("returns the carrier the deployment opened", func() { db, dsn := testutil.SetupTestDBWithDSN() diff --git a/core/application/fanout_wiring.go b/core/application/fanout_wiring.go index 15ef31ca9..defcecf98 100644 --- a/core/application/fanout_wiring.go +++ b/core/application/fanout_wiring.go @@ -20,12 +20,13 @@ import ( // parameter type is the reason. jobs.NewDispatcher and agents.NewEventBridge // both take a messaging.Broadcaster, which they must: neither may know which // carrier a deployment runs, and their specs publish through a double. But that -// also means *messaging.Client satisfies them, so wiring either of them to the -// NATS client instead of the carrier COMPILES, passes every unit spec in both -// packages, and presents only as an SSE stream that stays empty while the work -// it is watching runs to completion on the other side of a carrier nobody is -// subscribed to. Naming *pgbus.Bus here is what makes that a build failure -// rather than a silent one, and it is why this exists as a function instead of +// also means ANY type satisfying the interface can be wired to them, so while a +// second carrier existed, pointing either of them at it COMPILED, passed every +// unit spec in both packages, and presented only as an SSE stream that stayed +// empty while the work it was watching ran to completion on the other side of a +// carrier nobody subscribed to. The second carrier went with the message +// broker. Naming *pgbus.Bus here is what would make that a build failure rather +// than a silent one again, and it is why this exists as a function instead of // as two lines and a comment asking the reader to be careful. // // The observable persister is STARTED here for the reason startJobDispatchLoop diff --git a/core/application/startup.go b/core/application/startup.go index b9d371336..44962d841 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -402,8 +402,10 @@ func New(opts ...config.AppOption) (*Application, error) { } } // S2. One call sets the carrier and opens the wildcard - // subscriptions, and it names no carrier, so the NATS client on - // distSvc cannot be passed here by accident. + // subscriptions, and it names no carrier at all, so no carrier that + // happens to hang off distSvc can be passed here by accident. See + // cache_fanout_wiring.go for why that shape is kept now that the + // broker's client is no longer one of them. if err := distSvc.wireGallery(application.galleryService); err != nil { xlog.Warn("Gallery service subscribe failed", "error", err) } diff --git a/core/cli/agent_worker.go b/core/cli/agent_worker.go index 6df16fc21..7f7a6caf7 100644 --- a/core/cli/agent_worker.go +++ b/core/cli/agent_worker.go @@ -67,26 +67,54 @@ type AgentWorkerCMD struct { APIURL string `env:"LOCALAI_API_URL" help:"LocalAI API URL for inference (auto-derived from RegisterTo if not set)" group:"api"` APIToken string `env:"LOCALAI_API_TOKEN" help:"API token for LocalAI inference (auto-provisioned during registration if not set)" group:"api"` - NatsJWT string `env:"LOCALAI_NATS_JWT" help:"NATS user JWT override (defaults to nats_jwt from registration)" group:"distributed"` - NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"NATS user seed override (defaults to nats_user_seed from registration)" group:"distributed"` - NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Fallback NATS service JWT when registration does not mint agent JWT" group:"distributed"` - NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Fallback NATS service seed paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` - NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Require NATS JWT+seed to connect" group:"distributed"` + // The broker credential and TLS flags, accepted and ignored, hidden, on the + // same terms as NatsURL above. There is no connection left to present a + // credential on. + NatsJWT string `env:"LOCALAI_NATS_JWT" help:"Ignored. An agent worker opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsUserSeed string `env:"LOCALAI_NATS_USER_SEED" help:"Ignored. Paired with LOCALAI_NATS_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Ignored. An agent worker opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Ignored. Paired with LOCALAI_NATS_SERVICE_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Ignored. Use --distributed-require-auth to make this worker wait through admin approval." group:"distributed" hidden:""` // DistributedRequireAuth is the umbrella switch; for the agent worker (which - // has no file-transfer server) it implies NATS auth is required. - DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch implying --nats-require-auth (agent workers have no file-transfer server)" group:"distributed"` - NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" type:"existingfile" help:"PEM file for NATS server CA (private PKI)" group:"distributed"` - NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" type:"existingfile" help:"Client certificate for NATS mTLS" group:"distributed"` - NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" type:"existingfile" help:"Client private key for NATS mTLS" group:"distributed"` + // has no file-transfer server) it makes registration WAIT THROUGH ADMIN + // APPROVAL rather than starting against a pending node. + // + // It used to imply --nats-require-auth as well, and the wait was a side + // effect of that: the worker was waiting for a broker credential to be + // minted. There is no credential and no broker, so the wait is now what the + // switch is FOR, and it is described that way rather than by what it used + // to imply. + DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Wait through admin approval at registration instead of starting against a node an admin has not approved" group:"distributed"` + // type:"existingfile" is deliberately NOT kept: validating a path this + // process never opens would fail a worker at startup over a certificate for + // a broker the operator has already shut down. + NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" help:"Ignored. No bus connection is opened, so no server certificate is verified." group:"distributed" hidden:""` + NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" help:"Ignored. No bus connection is opened, so no client certificate is presented." group:"distributed" hidden:""` + NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" help:"Ignored. Paired with LOCALAI_NATS_TLS_CERT, which is itself ignored." group:"distributed" hidden:""` // Timeouts MCPCIJobTimeout string `env:"LOCALAI_MCP_CI_JOB_TIMEOUT" default:"10m" help:"Timeout for MCP CI job execution" group:"distributed"` } -// natsAuthRequired reports whether NATS JWT credentials must be present — the -// granular flag or the umbrella (LOCALAI_DISTRIBUTED_REQUIRE_AUTH). -func (cmd *AgentWorkerCMD) natsAuthRequired() bool { - return cmd.NatsRequireAuth || cmd.DistributedRequireAuth +// waitThroughApproval reports whether registration should block until an admin +// approves this node, instead of returning a pending response and starting. +// +// A method rather than the field read it wraps, because the answer CHANGED and +// the change is the one thing in this command an operator can be surprised by. +// It used to be --distributed-require-auth OR --nats-require-auth, narrowed +// further by whether the operator had supplied a broker JWT by hand. Every term +// but the first was about a credential that no longer exists, so the gate is +// now the first term alone. +// +// An operator who set ONLY --nats-require-auth therefore loses the wait and +// gets the historical default: register, start, and let the tunnel dialer be +// refused with 403 until an admin approves. That is a visible change, it is +// documented in docs/content/features/distributed-mode.md, and it is a seam so +// that it is also pinned: inlined at the call site it sat inside a Run that +// dials a frontend, where no spec could reach it and swapping the two flags +// back would have stayed green. +func (cmd *AgentWorkerCMD) waitThroughApproval() bool { + return cmd.DistributedRequireAuth } func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { @@ -121,17 +149,15 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { // Register, and obtain this node's identity and its tunnel credential. // - // The manager is still the NATS credential manager and still gated on the - // NATS auth flags, and that is deliberate rather than left over: what the - // gate decides is whether registration WAITS THROUGH ADMIN APPROVAL instead - // of returning a pending response, and that behaviour is unchanged by this - // worker no longer dialling a bus. What it no longer does is dial one: the - // only value read off it below is TunnelToken. - credMgr := workerregistry.NewNATSCredentialManager( + // The manager holds one thing now: the tunnel token, which every + // re-registration rotates. Its gate decides whether registration WAITS + // THROUGH ADMIN APPROVAL instead of returning a pending response; which + // flag decides that, and what changed about it, is on waitThroughApproval. + credMgr := workerregistry.NewCredentialManager( func(ctx context.Context) (*workerregistry.RegisterResponse, error) { return regClient.RegisterFull(ctx, registrationBody) }, - cmd.natsAuthRequired() && cmd.NatsJWT == "" && cmd.NatsServiceJWT == "", + cmd.waitThroughApproval(), ) res, err := credMgr.Acquire(shutdownCtx) if err != nil { @@ -208,8 +234,8 @@ func (cmd *AgentWorkerCMD) Run(ctx *cliContext.Context) error { xlog.Info("Agent worker ready, serving agent execution and MCP CI runs on its tunnel", "node", nodeID) // Wait for an OS signal. There is no internal fatal condition left to wait - // on: the one that existed was a NATS credential this worker could no - // longer renew, and it renews none. + // on: the one that existed was a broker credential this worker could no + // longer renew, and there is no credential and no broker. sigCh := make(chan os.Signal, 1) signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) <-sigCh @@ -385,7 +411,7 @@ func handleMCPCIJob(ctx context.Context, data []byte, apiURL, apiToken string, p task := evt.Task if job == nil || task == nil { xlog.Error("MCP CI job missing enriched data", "jobID", evt.JobID) - return mcpCIAnswer(pub, evt.JobID, "failed", "", "job or task data missing from NATS event") + return mcpCIAnswer(pub, evt.JobID, "failed", "", "job or task data missing from the job event") } modelCfg := evt.ModelConfig diff --git a/core/cli/agent_worker_test.go b/core/cli/agent_worker_test.go index 1e537999a..39a52d47c 100644 --- a/core/cli/agent_worker_test.go +++ b/core/cli/agent_worker_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "os" + "path/filepath" "strings" "time" @@ -229,4 +230,76 @@ var _ = Describe("The agent worker's bus requirement", func() { // edit every unit file in the same change. Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed()) }) + + It("still accepts the broker credentials that came with it", func() { + // The whole set, because an operator's unit file carries the whole set: + // a command line that parses --nats-url and then dies on --nats-jwt has + // bought the fleet nothing. + Expect(parse("--register-to", "http://frontend:8080", + "--nats-url", "nats://bus:4222", + "--nats-jwt", "eyJ0", + "--nats-user-seed", "SUUSER", + "--nats-service-jwt", "eyJ0", + "--nats-service-seed", "SUSERVICE", + "--nats-require-auth")).To(Succeed()) + }) + + It("does not stat the TLS material it no longer presents", func() { + // These paths were validated as existing files while they were dialled + // with. Keeping that on an ignored flag would fail a worker at startup + // over a certificate for a broker the operator has already deleted, + // which is the exact upgrade the acceptance exists to survive. + missing := filepath.Join(GinkgoT().TempDir(), "a-broker-ca-that-was-deleted.pem") + Expect(parse("--register-to", "http://frontend:8080", + "--nats-tlsca", missing, + "--nats-tls-cert", missing, + "--nats-tls-key", missing)).To(Succeed()) + }) + + It("keeps every accepted bus flag hidden from --help", func() { + var cli struct { + AgentWorker AgentWorkerCMD `cmd:""` + } + parser, err := kong.New(&cli) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if strings.HasPrefix(flag.Name, "nats-") && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) +}) + +// Which flag makes an agent worker wait through admin approval. +// +// The docs promise this specifically, and it is the one behavioural change in +// the broker removal that an operator can be surprised by, so it is asserted +// both ways round. The positive half alone would stay green if the gate were +// widened back to OR --nats-require-auth; the negative half is what says the +// change actually happened. +var _ = Describe("The agent worker's approval gate", func() { + It("waits when --distributed-require-auth is set", func() { + cmd := &AgentWorkerCMD{DistributedRequireAuth: true} + Expect(cmd.waitThroughApproval()).To(BeTrue()) + }) + + It("does not wait for the broker flag that used to imply it", func() { + // --nats-require-auth named a bus this worker does not dial. An + // operator who set only that one now gets the historical default: + // register, start, and be refused at every tunnel dial with 403 until + // an admin approves. Documented in the migration section of + // docs/content/features/distributed-mode.md. + cmd := &AgentWorkerCMD{NatsRequireAuth: true} + Expect(cmd.waitThroughApproval()).To(BeFalse(), + "an ignored flag is gating a real behaviour again") + }) + + It("does not wait when neither is set", func() { + Expect((&AgentWorkerCMD{}).waitThroughApproval()).To(BeFalse()) + }) }) diff --git a/core/cli/cli.go b/core/cli/cli.go index 77bf128cc..f5d107fe3 100644 --- a/core/cli/cli.go +++ b/core/cli/cli.go @@ -18,7 +18,7 @@ var CLI struct { Transcript TranscriptCMD `cmd:"" help:"Convert audio to text"` P2PWorker worker.Worker `cmd:"" name:"p2p-worker" help:"Run workers to distribute workload via p2p (llama.cpp-only)"` Worker WorkerCMD `cmd:"" help:"Start a worker for distributed mode (generic, backend-agnostic)"` - AgentWorker AgentWorkerCMD `cmd:"" name:"agent-worker" help:"Start an agent worker for distributed mode (executes agent chats via NATS)"` + AgentWorker AgentWorkerCMD `cmd:"" name:"agent-worker" help:"Start an agent worker for distributed mode (executes agent chats over the tunnel it dials out to a frontend)"` Util UtilCMD `cmd:"" help:"Utility commands"` Agent AgentCMD `cmd:"" help:"Run agents standalone without the full LocalAI server"` MCPServer MCPServerCMD `cmd:"" name:"mcp-server" help:"Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP)"` diff --git a/core/cli/run.go b/core/cli/run.go index 1c773f5c1..2eac5d89b 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -164,7 +164,7 @@ type RunCMD struct { // Distributed / Horizontal Scaling Distributed bool `env:"LOCALAI_DISTRIBUTED" default:"false" help:"Enable distributed mode (requires PostgreSQL; no message bus)" group:"distributed"` InstanceID string `env:"LOCALAI_INSTANCE_ID" help:"Unique instance ID for distributed mode (auto-generated UUID if empty)" group:"distributed"` - NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. No component of a distributed deployment connects to a message bus; state and fan-out ride PostgreSQL and workers are reached over their own tunnels. Accepted so an existing command line still starts." group:"distributed"` + NatsURL string `env:"LOCALAI_NATS_URL" help:"Ignored. No component of a distributed deployment connects to a message bus; state and fan-out ride PostgreSQL and workers are reached over their own tunnels. Accepted so an existing command line still starts." group:"distributed" hidden:""` DistributedAdvertiseAddr string `env:"LOCALAI_DISTRIBUTED_ADVERTISE_ADDR" help:"host:port other frontend replicas dial to reach this one (peer link). Empty = derived from the local address that routes to PostgreSQL, which only works when the database is on another host." group:"distributed"` StorageURL string `env:"LOCALAI_STORAGE_URL" help:"S3-compatible storage endpoint URL (e.g., http://minio:9000)" group:"distributed"` StorageBucket string `env:"LOCALAI_STORAGE_BUCKET" default:"localai" help:"S3 bucket name for object storage" group:"distributed"` @@ -173,28 +173,39 @@ type RunCMD struct { StorageSecretKey string `env:"LOCALAI_STORAGE_SECRET_KEY" help:"S3 secret access key" group:"distributed"` RegistrationToken string `env:"LOCALAI_REGISTRATION_TOKEN" help:"Token that backend nodes must provide to register (empty = no auth required)" group:"distributed"` RegistrationRequireAuth bool `env:"LOCALAI_REGISTRATION_REQUIRE_AUTH" default:"false" help:"Fail startup when distributed mode is enabled but LOCALAI_REGISTRATION_TOKEN is empty (node endpoints and worker file-transfer server would otherwise be unauthenticated)" group:"distributed"` - DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch: require BOTH NATS JWT credentials and a registration token when distributed mode is enabled (implies --nats-require-auth and --registration-require-auth)" group:"distributed"` + DistributedRequireAuth bool `env:"LOCALAI_DISTRIBUTED_REQUIRE_AUTH" default:"false" help:"Umbrella switch: require a registration token when distributed mode is enabled (implies --registration-require-auth)" group:"distributed"` AutoApproveNodes bool `env:"LOCALAI_AUTO_APPROVE_NODES" default:"false" help:"Auto-approve new worker nodes (skip admin approval)" group:"distributed"` DistributedSharedModels bool `env:"LOCALAI_DISTRIBUTED_SHARED_MODELS" default:"false" help:"Assert that every node mounts the SAME models directory at the SAME path (shared volume). When true, the router skips staging model files to workers and loads them directly from the shared path, avoiding re-downloads." group:"distributed"` DistributedPrefixCache bool `env:"LOCALAI_DISTRIBUTED_PREFIX_CACHE" default:"true" help:"Enable prefix-cache-aware routing in distributed mode (default true). When false, routing falls back to round-robin." group:"distributed"` DistributedDiskHeadroomCheck bool `env:"LOCALAI_DISTRIBUTED_DISK_HEADROOM_CHECK" default:"true" help:"Reject worker nodes that lack free space to store the model, at scheduling time rather than partway through staging (default true). Free space is measured on the filesystem backing each worker's models directory, and compared against the model's own size plus a small margin. When false, node selection ignores free disk (pre-#11054 behaviour); the check still runs and warns when it would have rejected every node. Can also be toggled at runtime via the distributed_disk_headroom_check setting." group:"distributed"` DistributedPrefixCacheTTL string `env:"LOCALAI_DISTRIBUTED_PREFIX_CACHE_TTL" help:"Idle-timeout for prefix-cache index entries; also drives the background eviction cadence (every TTL/2). Default 5m." group:"distributed"` - BackendInstallTimeout string `env:"LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT" help:"NATS round-trip timeout for backend.install requests sent to worker nodes (default 15m). Increase for slow links pulling multi-GB images." group:"distributed"` - BackendUpgradeTimeout string `env:"LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT" help:"NATS round-trip timeout for backend.upgrade requests (default 15m)." group:"distributed"` + BackendInstallTimeout string `env:"LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT" help:"Timeout for a backend.install request the frontend sends a worker over its tunnel (default 15m). Increase for slow links pulling multi-GB images." group:"distributed"` + BackendUpgradeTimeout string `env:"LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT" help:"Timeout for a backend.upgrade request the frontend sends a worker over its tunnel (default 15m)." group:"distributed"` ModelLoadTimeout string `env:"LOCALAI_NATS_MODEL_LOAD_TIMEOUT" help:"Fixed gRPC deadline for the remote LoadModel call sent to a worker node once its backend is installed and model files are staged. Unset (the default), the deadline is derived from the checkpoint size instead: 5m plus 20s per GiB, capped at 6h, so multi-tens-of-GB diffusion/video checkpoints get the minutes they need without a fixed cliff. Set this only to pin a specific budget; the value is used verbatim, including when it is shorter than the derived one." group:"distributed"` ModelLoadWait string `env:"LOCALAI_MODEL_LOAD_WAIT" help:"How long an inference request waits for a model that is still cold-loading onto a worker before it is answered with 503, a Retry-After header and live staging progress (default 60s). The request is served the moment the model becomes ready, so a model already most of the way staged needs no client retry. Set to 0 to wait as long as the load takes — only safe when no ingress or load balancer with an idle timeout sits in front." group:"distributed"` WorkerReconnectGrace string `env:"LOCALAI_WORKER_RECONNECT_GRACE" help:"How long a worker whose tunnel was lost is treated as reconnecting rather than gone (default 90s, clear of two of the worker's own ceiling backoffs plus the dial between them). Only after this window may the scheduler stop placing work on that worker and clean up its rows, so a value below the worker's backoff condemns workers that are re-homing normally; raise it to make a rolling frontend restart safer, lower it to reap a genuinely dead worker sooner. Measured on the database clock, so every replica agrees." group:"distributed"` - NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"NATS account signing seed (SU...) used to mint per-node worker JWTs at registration" group:"distributed"` - NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"NATS user JWT for the frontend (and agent workers) to publish control-plane messages" group:"distributed"` - NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"NATS user signing seed (SU...) paired with LOCALAI_NATS_SERVICE_JWT" group:"distributed"` - NatsWorkerJWTTTL string `env:"LOCALAI_NATS_WORKER_JWT_TTL" help:"Lifetime of minted per-node NATS JWTs (e.g. 24h, default 24h)" group:"distributed"` - NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Require NATS JWT credentials (service JWT + account seed) when distributed mode is enabled" group:"distributed"` - NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" type:"existingfile" help:"PEM file for NATS server CA (private PKI); use with tls:// in --nats-url" group:"distributed"` - NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" type:"existingfile" help:"Client certificate for NATS mTLS" group:"distributed"` - NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" type:"existingfile" help:"Client private key for NATS mTLS" group:"distributed"` - ExposeNodeHeader bool `env:"LOCALAI_EXPOSE_NODE_HEADER" default:"false" help:"Set the X-LocalAI-Node response header on inference responses (OpenAI chat/completions/embeddings, Anthropic /v1/messages, Ollama /api/chat,/api/generate,/api/embed) with the ID of the worker that served the request. Disabled by default: the node ID reveals internal topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency the header may reflect a recent routing decision rather than this exact request's." group:"distributed"` - ModelScheduling string `env:"LOCALAI_MODEL_SCHEDULING" help:"Declarative per-model scheduling config applied at startup (inline JSON list of {model_name,node_selector,min_replicas,max_replicas,replicas:\"all\"}). Authoritative: overwrites matching models on every boot. Distributed mode only." group:"distributed"` - ModelSchedulingConfig string `env:"LOCALAI_MODEL_SCHEDULING_CONFIG" help:"Path to a YAML file with the same per-model scheduling list as LOCALAI_MODEL_SCHEDULING. Distributed mode only." group:"distributed"` + // The broker credential and TLS flags, accepted and ignored, hidden, on the + // same terms as --nats-url above and for the same reason: kong fails on an + // unknown flag, so deleting them turns every existing unit file, compose + // file and Helm values file into a startup parse error at upgrade, in + // exchange for nothing. There is no bus connection to present a credential + // on and no minting left to do. + // + // type:"existingfile" is deliberately NOT kept on the TLS paths. Validating + // a path this process never opens would fail a deployment at startup over a + // certificate for a broker the operator has already shut down, which is + // exactly the upgrade the acceptance exists to survive. + NatsAccountSeed string `env:"LOCALAI_NATS_ACCOUNT_SEED" help:"Ignored. The frontend mints no per-node broker credential; nodes are authenticated by their registration and tunnel tokens." group:"distributed" hidden:""` + NatsServiceJWT string `env:"LOCALAI_NATS_SERVICE_JWT" help:"Ignored. The frontend opens no bus connection to present a credential on." group:"distributed" hidden:""` + NatsServiceSeed string `env:"LOCALAI_NATS_SERVICE_SEED" help:"Ignored. Paired with LOCALAI_NATS_SERVICE_JWT, which is itself ignored." group:"distributed" hidden:""` + NatsWorkerJWTTTL string `env:"LOCALAI_NATS_WORKER_JWT_TTL" help:"Ignored. No per-node broker credential is minted, so none has a lifetime." group:"distributed" hidden:""` + NatsRequireAuth bool `env:"LOCALAI_NATS_REQUIRE_AUTH" default:"false" help:"Ignored on the frontend. Use --registration-require-auth, or the --distributed-require-auth umbrella, to fail closed on a missing registration token." group:"distributed" hidden:""` + NatsTLSCA string `env:"LOCALAI_NATS_TLS_CA" help:"Ignored. No bus connection is opened, so no server certificate is verified." group:"distributed" hidden:""` + NatsTLSCert string `env:"LOCALAI_NATS_TLS_CERT" help:"Ignored. No bus connection is opened, so no client certificate is presented." group:"distributed" hidden:""` + NatsTLSKey string `env:"LOCALAI_NATS_TLS_KEY" help:"Ignored. Paired with LOCALAI_NATS_TLS_CERT, which is itself ignored." group:"distributed" hidden:""` + ExposeNodeHeader bool `env:"LOCALAI_EXPOSE_NODE_HEADER" default:"false" help:"Set the X-LocalAI-Node response header on inference responses (OpenAI chat/completions/embeddings, Anthropic /v1/messages, Ollama /api/chat,/api/generate,/api/embed) with the ID of the worker that served the request. Disabled by default: the node ID reveals internal topology and should not be exposed on a public endpoint. Best-effort: under heavy concurrency the header may reflect a recent routing decision rather than this exact request's." group:"distributed"` + ModelScheduling string `env:"LOCALAI_MODEL_SCHEDULING" help:"Declarative per-model scheduling config applied at startup (inline JSON list of {model_name,node_selector,min_replicas,max_replicas,replicas:\"all\"}). Authoritative: overwrites matching models on every boot. Distributed mode only." group:"distributed"` + ModelSchedulingConfig string `env:"LOCALAI_MODEL_SCHEDULING_CONFIG" help:"Path to a YAML file with the same per-model scheduling list as LOCALAI_MODEL_SCHEDULING. Distributed mode only." group:"distributed"` Version bool @@ -356,9 +367,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { if r.DistributedAdvertiseAddr != "" { opts = append(opts, config.WithDistributedAdvertiseAddr(r.DistributedAdvertiseAddr)) } - if r.NatsURL != "" { - opts = append(opts, config.WithNatsURL(r.NatsURL)) - } if r.StorageURL != "" { opts = append(opts, config.WithStorageURL(r.StorageURL)) } @@ -421,34 +429,6 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { if r.DistributedSharedModels { opts = append(opts, config.EnableDistributedSharedModels) } - if r.NatsAccountSeed != "" { - opts = append(opts, config.WithNatsAccountSeed(r.NatsAccountSeed)) - } - if r.NatsServiceJWT != "" { - opts = append(opts, config.WithNatsServiceJWT(r.NatsServiceJWT)) - } - if r.NatsServiceSeed != "" { - opts = append(opts, config.WithNatsServiceSeed(r.NatsServiceSeed)) - } - if r.NatsWorkerJWTTTL != "" { - d, err := time.ParseDuration(r.NatsWorkerJWTTTL) - if err != nil { - return fmt.Errorf("invalid LOCALAI_NATS_WORKER_JWT_TTL %q: %w", r.NatsWorkerJWTTTL, err) - } - opts = append(opts, config.WithNatsWorkerJWTTTL(d)) - } - if r.NatsRequireAuth { - opts = append(opts, config.EnableNatsRequireAuth) - } - if r.NatsTLSCA != "" { - opts = append(opts, config.WithNatsTLSCA(r.NatsTLSCA)) - } - if r.NatsTLSCert != "" { - opts = append(opts, config.WithNatsTLSCert(r.NatsTLSCert)) - } - if r.NatsTLSKey != "" { - opts = append(opts, config.WithNatsTLSKey(r.NatsTLSKey)) - } if r.AutoApproveNodes { opts = append(opts, config.EnableAutoApproveNodes) } diff --git a/core/cli/run_bus_flags_test.go b/core/cli/run_bus_flags_test.go new file mode 100644 index 000000000..c0aefa323 --- /dev/null +++ b/core/cli/run_bus_flags_test.go @@ -0,0 +1,172 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + + "github.com/alecthomas/kong" + + "github.com/mudler/LocalAI/core/config" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// The frontend's broker flags are ACCEPTED and IGNORED, and the two halves are +// separate Its on purpose. +// +// Accepted, because kong fails on an unknown flag: deleting --nats-url would +// turn every existing unit file, compose file and Helm chart into a startup +// parse error on the day of the upgrade, in exchange for nothing, since the +// flag has nothing left to do. Ignored, because a flag that parses and is then +// quietly honoured is the failure this spec exists to exclude, and a single +// "it starts" assertion cannot tell the two apart. +// runFlagVars supplies the kong variables cmd/local-ai/main.go supplies, so a +// RunCMD can be parsed here at all: its path defaults interpolate ${basepath}. +func runFlagVars() kong.Vars { + return kong.Vars{ + "basepath": GinkgoT().TempDir(), + "generatedcontentpath": DefaultGeneratedContentPath(), + "uploadpath": DefaultUploadPath(), + "galleries": config.DefaultGalleriesJSON, + "backends": config.DefaultBackendGalleriesJSON, + "version": "test", + } +} + +var _ = Describe("The frontend's broker flags", func() { + busFlags := []string{ + "--nats-url", "nats://bus:4222", + "--nats-account-seed", "SUACCOUNT", + "--nats-service-jwt", "eyJ0", + "--nats-service-seed", "SUSERVICE", + "--nats-worker-jwtttl", "24h", + "--nats-require-auth", + } + + parse := func(args ...string) (*RunCMD, error) { + // kong resolves env: tags from the process environment, so a + // LOCALAI_NATS_URL inherited from a developer's shell would let the + // first spec pass for the wrong reason. + for _, name := range []string{"LOCALAI_NATS_URL", "LOCALAI_NATS_ACCOUNT_SEED", "LOCALAI_NATS_REQUIRE_AUTH"} { + if prior, had := os.LookupEnv(name); had { + Expect(os.Unsetenv(name)).To(Succeed()) + DeferCleanup(func() { _ = os.Setenv(name, prior) }) + } + } + var cli struct { + Run RunCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + _, err = parser.Parse(append([]string{"run"}, args...)) + return &cli.Run, err + } + + It("starts with no bus named at all", func() { + _, err := parse("--distributed") + Expect(err).To(Succeed(), + "a distributed frontend dials no message bus and must not demand the URL of one") + }) + + It("still accepts a command line that names one", func() { + _, err := parse(append([]string{"--distributed"}, busFlags...)...) + Expect(err).To(Succeed()) + }) + + It("does not stat the TLS material it no longer presents", func() { + // The paths were validated as existing files while they were dialled + // with. Keeping that validation on an ignored flag would fail a + // deployment at startup over a certificate for a broker that is gone, + // which is the exact upgrade the acceptance is meant to survive. + missing := filepath.Join(GinkgoT().TempDir(), "a-broker-ca-that-was-deleted.pem") + _, err := parse("--distributed", + "--nats-tlsca", missing, + "--nats-tls-cert", missing, + "--nats-tls-key", missing) + Expect(err).To(Succeed()) + }) + + It("keeps every accepted bus flag hidden from --help", func() { + // Accepted for the upgrade, not offered to a new operator: a flag that + // does nothing must not appear in the list of things to configure. + var cli struct { + Run RunCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if len(flag.Name) >= 5 && flag.Name[:5] == "nats-" && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) +}) + +// The serve-backend worker's half of the same promise, which nothing pinned. +// +// `local-ai worker` kept ONE broker flag and dropped the rest a release earlier, +// when it stopped connecting to a broker at all. That asymmetry is documented +// in docs/content/reference/cli-reference.md, and a documented promise with no +// spec is how the wrong half gets deleted: --nats-url is the one an operator's +// worker unit file actually carries, and it is the one whose removal would turn +// an upgrade into a parse error on every worker in the fleet at once. +// +// The negative It is here for the same reason the frontend's two halves are +// separate. Without it, "accepted" could be satisfied by quietly re-adding the +// credential flags, and the docs would be describing a surface nobody checked. +var _ = Describe("The serve-backend worker's broker flags", func() { + parse := func(args ...string) error { + for _, name := range []string{"LOCALAI_NATS_URL", "LOCALAI_ADDRESS"} { + if prior, had := os.LookupEnv(name); had { + Expect(os.Unsetenv(name)).To(Succeed()) + DeferCleanup(func() { _ = os.Setenv(name, prior) }) + } + } + var cli struct { + Worker WorkerCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + _, err = parser.Parse(append([]string{"worker"}, args...)) + return err + } + + It("still accepts the bus URL an existing worker unit file carries", func() { + Expect(parse("--register-to", "http://frontend:8080", "--nats-url", "nats://bus:4222")).To(Succeed(), + "a serve-backend worker dials no message bus, and an operator whose unit file still names one must still get a worker that starts") + }) + + It("keeps it hidden from --help", func() { + var cli struct { + Worker WorkerCMD `cmd:""` + } + parser, err := kong.New(&cli, runFlagVars()) + Expect(err).ToNot(HaveOccurred()) + var visible []string + for _, node := range parser.Model.Children { + for _, flag := range node.Flags { + if strings.HasPrefix(flag.Name, "nats-") && !flag.Hidden { + visible = append(visible, flag.Name) + } + } + } + Expect(visible).To(BeEmpty(), + "%v are still offered in --help while doing nothing", visible) + }) + + It("took no broker CREDENTIAL flag back", func() { + // The credential and TLS flags left this command in phase 3 and must + // stay gone: re-adding one would put a broker credential back on the + // surface of a process that opens no broker connection, and the docs + // say per-command which flags survive. + Expect(parse("--register-to", "http://frontend:8080", "--nats-jwt", "eyJ0")).To(HaveOccurred()) + Expect(parse("--register-to", "http://frontend:8080", "--nats-service-jwt", "eyJ0")).To(HaveOccurred()) + Expect(parse("--register-to", "http://frontend:8080", "--nats-tlsca", "/dev/null")).To(HaveOccurred()) + }) +}) diff --git a/core/cli/worker/worker_vllm.go b/core/cli/worker/worker_vllm.go index 8596546b5..1471d780f 100644 --- a/core/cli/worker/worker_vllm.go +++ b/core/cli/worker/worker_vllm.go @@ -96,7 +96,7 @@ func (r *VLLMDistributed) Run(ctx *cliContext.Context) error { FrontendURL: r.RegisterTo, RegistrationToken: r.RegistrationToken, } - nodeID, _, _, _, regErr := regClient.RegisterWithRetry(context.Background(), r.registrationBody(), 10) + nodeID, _, regErr := regClient.RegisterWithRetry(context.Background(), r.registrationBody(), 10) if regErr != nil { return fmt.Errorf("registering with frontend: %w", regErr) } diff --git a/core/cli/workerregistry/client.go b/core/cli/workerregistry/client.go index fb00fb3f1..1073b46ae 100644 --- a/core/cli/workerregistry/client.go +++ b/core/cli/workerregistry/client.go @@ -68,15 +68,19 @@ type RegisterResponse struct { // hash, so this is the ONLY time the plaintext exists anywhere but in this // worker's memory: a worker that discards it cannot get it back without // registering again. - TunnelToken string `json:"tunnel_token,omitempty"` - NatsJWT string `json:"nats_jwt,omitempty"` - NatsUserSeed string `json:"nats_user_seed,omitempty"` + TunnelToken string `json:"tunnel_token,omitempty"` + // There are no nats_jwt / nats_user_seed fields. A frontend that predates + // this release still sends them and this decodes fine: encoding/json + // ignores a key with no field, so an old frontend talking to a new worker + // is a no-op rather than a decode failure. } // RegisterFull sends a single registration request and returns the full -// response (node ID, approval status, and optional API token / NATS creds). +// response (node ID, approval status, and optional API and tunnel tokens). // Re-registration is idempotent: the frontend preserves the node row and mints -// a fresh NATS JWT each call, so this doubles as the credential-refresh call. +// a fresh TUNNEL token each call, so this doubles as the rotation call. It is +// the only credential a registration mints; the per-node broker JWT it used to +// carry went with the bus. func (c *RegistrationClient) RegisterFull(ctx context.Context, body map[string]any) (*RegisterResponse, error) { jsonBody, _ := json.Marshal(body) url := c.baseURL() + "/api/node/register" @@ -157,26 +161,20 @@ func isRegistrationRejection(status int) bool { return status >= 400 && status < 500 } -// Register sends a single registration request and returns the node ID and -// optional credentials (API token for agent workers, NATS JWT when configured). -func (c *RegistrationClient) Register(ctx context.Context, body map[string]any) (nodeID, apiToken, natsJWT, natsSeed string, err error) { - res, err := c.RegisterFull(ctx, body) - if err != nil { - return "", "", "", "", err - } - return res.ID, res.APIToken, res.NatsJWT, res.NatsUserSeed, nil -} - // RegisterWithRetry retries registration with exponential backoff. // // It drops every field of the response it does not name, the tunnel credential // among them. Callers that need one use RegisterFullWithRetry. -func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken, natsJWT, natsSeed string, err error) { +// +// The two broker-credential returns it used to carry are gone with the bus, and +// so is the Register one-shot that existed only to carry them: it had no caller +// left once nothing dialled a broker. +func (c *RegistrationClient) RegisterWithRetry(ctx context.Context, body map[string]any, maxRetries int) (nodeID, apiToken string, err error) { res, err := c.RegisterFullWithRetry(ctx, body, maxRetries) if err != nil { - return "", "", "", "", err + return "", "", err } - return res.ID, res.APIToken, res.NatsJWT, res.NatsUserSeed, nil + return res.ID, res.APIToken, nil } // RegisterFullWithRetry retries registration with exponential backoff and diff --git a/core/cli/workerregistry/client_test.go b/core/cli/workerregistry/client_test.go index 5870d2524..f666e5f2b 100644 --- a/core/cli/workerregistry/client_test.go +++ b/core/cli/workerregistry/client_test.go @@ -5,6 +5,7 @@ import ( "errors" "net/http" "net/http/httptest" + "reflect" "sync/atomic" "time" @@ -128,7 +129,7 @@ var _ = Describe("Registration client refusals", func() { // The default worker path goes through Acquire, not the ladder above, // and its bound is 100 attempts rather than 10. A refusal there is the // same verdict and has to end the same way. - mgr := NewNATSCredentialManager(func(ctx context.Context) (*RegisterResponse, error) { + mgr := NewCredentialManager(func(ctx context.Context) (*RegisterResponse, error) { return client.RegisterFull(ctx, map[string]any{"name": "w1"}) }, true) _, err := mgr.Acquire(context.Background()) @@ -136,3 +137,41 @@ var _ = Describe("Registration client refusals", func() { Expect(attempts.Load()).To(Equal(int32(1))) }) }) + +// The other direction of the same upgrade: a worker of this release registering +// against a frontend that still mints a per-node broker credential. +// +// The fields are gone from RegisterResponse, so the only question is what +// happens to the keys still on the wire. encoding/json ignores a key with no +// field, and that is asserted rather than assumed, because a decoder switched +// to DisallowUnknownFields would turn every registration against an older +// frontend into a hard failure with no other symptom. +var _ = Describe("Registering against a frontend that still mints broker credentials", func() { + It("decodes the response and drops the keys it no longer has fields for", func() { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"node-7","status":"healthy","api_token":"key-7","tunnel_token":"tunnel-7","nats_jwt":"eyJ0","nats_user_seed":"SUUSER"}`)) + })) + DeferCleanup(server.Close) + + client := &RegistrationClient{FrontendURL: server.URL, HTTPTimeout: 2 * time.Second} + res, err := client.RegisterFull(context.Background(), map[string]any{"name": "w1"}) + Expect(err).ToNot(HaveOccurred()) + + // The fields it DOES have, so the assertion below is about the two + // unknown keys and not about a decode that produced nothing. + Expect(res.ID).To(Equal("node-7")) + Expect(res.APIToken).To(Equal("key-7")) + Expect(res.TunnelToken).To(Equal("tunnel-7")) + + // And nowhere for a broker credential to land: asserted on the struct's + // own type, because a value assertion would need a field to read and + // would stop compiling exactly when the field came back. + t := reflect.TypeOf(*res) + for _, gone := range []string{"NatsJWT", "NatsUserSeed"} { + _, found := t.FieldByName(gone) + Expect(found).To(BeFalse(), + "RegisterResponse.%s is back: a worker that stores a broker credential is a worker something expects to dial a broker", gone) + } + }) +}) diff --git a/core/cli/workerregistry/credentials.go b/core/cli/workerregistry/credentials.go index b023b9916..2a45673bc 100644 --- a/core/cli/workerregistry/credentials.go +++ b/core/cli/workerregistry/credentials.go @@ -7,7 +7,6 @@ import ( "sync" "time" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) @@ -16,40 +15,44 @@ import ( // package (and its gorm/DB dependencies). const statusPending = "pending" -// defaultMaxAttempts bounds how many times Acquire registers (and how many -// consecutive times RefreshLoop may fail) before giving up. It is high enough -// to ride out a slow admin approval or a transient frontend outage, but finite -// so an unauthorized/unapprovable worker exits and surfaces the problem (via a -// non-zero exit and the resulting restart) rather than waiting forever. +// defaultMaxAttempts bounds how many times Acquire registers before giving up. +// It is high enough to ride out a slow admin approval or a transient frontend +// outage, but finite so an unauthorized/unapprovable worker exits and surfaces +// the problem (via a non-zero exit and the resulting restart) rather than +// waiting forever. const defaultMaxAttempts = 100 // RegisterFunc performs one idempotent registration round-trip. type RegisterFunc func(ctx context.Context) (*RegisterResponse, error) -// NATSCredentialManager acquires NATS credentials at startup — waiting through -// admin approval when required — and refreshes them before the minted JWT -// expires, by re-registering (which mints a fresh JWT). The live NATS -// connection adopts a refreshed JWT on its next reconnect via Provider. Safe -// for concurrent use. +// CredentialManager acquires a node's own credentials at startup, waiting +// through admin approval when that is required, and holds the tunnel token the +// most recent registration minted. Safe for concurrent use. // -// It addresses two failure modes: a worker that needs credentials but registers -// while still pending approval (it would otherwise give up and never connect), -// and a long-running worker whose 24h JWT expires with no way to renew it. -type NATSCredentialManager struct { - register RegisterFunc - requireCreds bool // block until credentials are present (frontend minting in use) +// Renamed from NATSCredentialManager and stripped rather than deleted. The JWT +// half went with the message bus: nothing mints a broker credential and nothing +// opens a connection to present one on. The tunnel token did not go with it, +// and it is the reason a manager is still worth having: it is ROTATED by a +// registration rather than expiring on a clock, and the frontend keeps only its +// hash, so the dialer has to read the current value at dial time instead of +// being handed one at startup. +type CredentialManager struct { + register RegisterFunc + // requireApproval blocks Acquire until the node is out of pending. + // + // Narrower than the requireCreds it replaces: there is no credential left + // to wait for being MINTED, only an admin decision to wait through. A + // worker that proceeds while pending registers and heartbeats fine, and is + // then refused at every tunnel dial, so an operator who wants the wait + // rather than the refusal loop asks for it here. + requireApproval bool - // Tunables; defaults set by NewNATSCredentialManager, overridable in tests. + // Tunables; defaults set by NewCredentialManager, overridable in tests. initialBackoff time.Duration maxBackoff time.Duration - maxAttempts int // bound on Acquire attempts / consecutive refresh failures (<=0 = unlimited) - refreshLead float64 // refresh once this fraction of the JWT lifetime has elapsed - refreshRetry time.Duration - expiryOf func(jwt string) (time.Time, bool) + maxAttempts int // bound on Acquire attempts (<=0 = unlimited) mu sync.RWMutex - jwt string - seed string nodeID string // tunnelToken is the node's own tunnel credential from the most recent // registration. It is kept here because every re-registration this manager @@ -58,46 +61,28 @@ type NATSCredentialManager struct { tunnelToken string } -// NewNATSCredentialManager builds a manager over register. When requireCreds is -// true, Acquire blocks until the node is approved and credentials are minted. -func NewNATSCredentialManager(register RegisterFunc, requireCreds bool) *NATSCredentialManager { - return &NATSCredentialManager{ - register: register, - requireCreds: requireCreds, - initialBackoff: 2 * time.Second, - maxBackoff: 30 * time.Second, - maxAttempts: defaultMaxAttempts, - refreshLead: 0.75, - refreshRetry: 30 * time.Second, - expiryOf: jwtExpiry, +// NewCredentialManager builds a manager over register. When requireApproval is +// true, Acquire blocks through admin approval instead of returning a pending +// response. +func NewCredentialManager(register RegisterFunc, requireApproval bool) *CredentialManager { + return &CredentialManager{ + register: register, + requireApproval: requireApproval, + initialBackoff: 2 * time.Second, + maxBackoff: 30 * time.Second, + maxAttempts: defaultMaxAttempts, } } -// jwtExpiry decodes the expiry of a minted user JWT. ok is false when the token -// is empty/undecodable or carries no expiry (e.g. a non-expiring service JWT). -func jwtExpiry(token string) (time.Time, bool) { - if token == "" { - return time.Time{}, false - } - uc, err := natsauth.DecodeUserClaims(token) - if err != nil || uc.Expires == 0 { - return time.Time{}, false - } - return time.Unix(uc.Expires, 0), true -} - -func (m *NATSCredentialManager) store(res *RegisterResponse) { +func (m *CredentialManager) store(res *RegisterResponse) { m.mu.Lock() defer m.mu.Unlock() m.nodeID = res.ID - if res.NatsJWT != "" && res.NatsUserSeed != "" { - m.jwt, m.seed = res.NatsJWT, res.NatsUserSeed - } - // Guarded the same way the NATS pair is: a response that carries no tunnel - // token (a frontend that predates them, or one whose minting failed) must - // not wipe a working credential this worker already holds. Overwriting with - // "" would lock the tunnel out until the next registration that did carry - // one, which is the opposite of what an empty field means. + // A response that carries no tunnel token (a frontend that predates them, + // or one whose minting failed) must not wipe a working credential this + // worker already holds. Overwriting with "" would lock the tunnel out until + // the next registration that did carry one, which is the opposite of what + // an empty field means. if res.TunnelToken != "" { m.tunnelToken = res.TunnelToken } @@ -105,43 +90,23 @@ func (m *NATSCredentialManager) store(res *RegisterResponse) { // TunnelToken returns the node's current tunnel credential, empty until one has // been issued. It is the callback the tunnel client reads on every dial. -func (m *NATSCredentialManager) TunnelToken() string { +func (m *CredentialManager) TunnelToken() string { m.mu.RLock() defer m.mu.RUnlock() return m.tunnelToken } -// Current returns the latest NATS credentials (both empty until acquired). -func (m *NATSCredentialManager) Current() (jwt, seed string) { - m.mu.RLock() - defer m.mu.RUnlock() - return m.jwt, m.seed -} - // NodeID returns the node ID from the most recent registration. -func (m *NATSCredentialManager) NodeID() string { +func (m *CredentialManager) NodeID() string { m.mu.RLock() defer m.mu.RUnlock() return m.nodeID } -// Provider returns a callback compatible with messaging.WithUserJWTProvider, -// supplying the current credentials on each (re)connect. -func (m *NATSCredentialManager) Provider() func() (string, string) { - return m.Current -} - -// HasCredentials reports whether complete NATS credentials have been obtained. -func (m *NATSCredentialManager) HasCredentials() bool { - jwt, seed := m.Current() - return jwt != "" && seed != "" -} - -// Acquire registers and, when requireCreds is set, keeps re-registering with -// exponential backoff until the node is approved (status != pending) and -// credentials are minted. Without requireCreds it returns the first successful -// response (the historical one-shot behavior, preserved for anonymous NATS). -func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { +// Acquire registers and, when requireApproval is set, keeps re-registering with +// exponential backoff until the node is approved (status != pending). Without +// requireApproval it returns the first successful response. +func (m *CredentialManager) Acquire(ctx context.Context) (*RegisterResponse, error) { backoff := m.initialBackoff var lastReason error for attempt := 1; m.maxAttempts <= 0 || attempt <= m.maxAttempts; attempt++ { @@ -155,15 +120,12 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, case err != nil: lastReason = err xlog.Warn("Registration failed, retrying", "attempt", attempt, "next_retry", backoff, "error", err) - case !m.requireCreds: + case !m.requireApproval: m.store(res) return res, nil case res.Status == statusPending: lastReason = fmt.Errorf("node %s still pending admin approval", res.ID) xlog.Info("Node pending admin approval; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) - case res.NatsJWT == "" || res.NatsUserSeed == "": - lastReason = fmt.Errorf("node %s approved but NATS credentials not minted", res.ID) - xlog.Info("Node approved but NATS credentials not yet minted; waiting", "node", res.ID, "attempt", attempt, "next_retry", backoff) default: m.store(res) return res, nil @@ -175,53 +137,5 @@ func (m *NATSCredentialManager) Acquire(ctx context.Context) (*RegisterResponse, } backoff = min(backoff*2, m.maxBackoff) } - return nil, fmt.Errorf("giving up acquiring NATS credentials after %d attempts: %w", m.maxAttempts, lastReason) -} - -// RefreshLoop re-registers to mint a fresh JWT before the current one expires, -// updating the credentials returned by Current/Provider so the NATS connection -// adopts them on its next reconnect. It returns nil when ctx is cancelled or -// when the current credential has no expiry (nothing to refresh), and a non-nil -// error after maxAttempts consecutive refresh failures — letting the caller -// exit the worker so it restarts and re-acquires (or surfaces the outage) -// rather than silently drifting toward an expired, unrenewable JWT. -func (m *NATSCredentialManager) RefreshLoop(ctx context.Context) error { - failures := 0 - for { - jwt, _ := m.Current() - exp, ok := m.expiryOf(jwt) - if !ok { - xlog.Debug("NATS credential has no expiry; refresh loop exiting") - return nil - } - wait := max(time.Duration(float64(time.Until(exp))*m.refreshLead), 0) - select { - case <-ctx.Done(): - return nil - case <-time.After(wait): - } - - res, err := m.register(ctx) - if err == nil && res.NatsJWT != "" && res.NatsUserSeed != "" { - m.store(res) - failures = 0 - xlog.Info("Refreshed NATS credentials", "node", res.ID) - continue - } - failures++ - if err != nil { - xlog.Warn("NATS credential refresh failed; will retry", "attempt", failures, "error", err) - } else { - xlog.Warn("NATS credential refresh returned no credentials; will retry", "attempt", failures) - } - if m.maxAttempts > 0 && failures >= m.maxAttempts { - return fmt.Errorf("NATS credential refresh failed %d times in a row", failures) - } - // Back off before retrying so a persistent failure near expiry does not spin. - select { - case <-ctx.Done(): - return nil - case <-time.After(m.refreshRetry): - } - } + return nil, fmt.Errorf("giving up registering after %d attempts: %w", m.maxAttempts, lastReason) } diff --git a/core/cli/workerregistry/credentials_test.go b/core/cli/workerregistry/credentials_test.go index 02ed413c4..cbc372d9d 100644 --- a/core/cli/workerregistry/credentials_test.go +++ b/core/cli/workerregistry/credentials_test.go @@ -6,9 +6,6 @@ import ( "testing" "time" - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/nkeys" - . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" ) @@ -50,20 +47,20 @@ func (f *fakeRegister) count() int { return f.calls } -var _ = Describe("NATSCredentialManager", func() { - approved := func(jwt, seed string) *RegisterResponse { - return &RegisterResponse{ID: "node-1", Status: "healthy", NatsJWT: jwt, NatsUserSeed: seed} +var _ = Describe("CredentialManager", func() { + approved := func(tunnelToken string) *RegisterResponse { + return &RegisterResponse{ID: "node-1", Status: "healthy", TunnelToken: tunnelToken} } pending := &RegisterResponse{ID: "node-1", Status: "pending"} - Describe("Acquire (#4 — wait through admin approval)", func() { - It("keeps re-registering until the node is approved and credentials are minted", func() { + Describe("Acquire (wait through admin approval)", func() { + It("keeps re-registering until the node is approved", func() { f := &fakeRegister{steps: []step{ - {res: pending}, // not approved yet - {res: approved("", "")}, // approved but JWT not minted yet - {res: approved("jwt-1", "seed-1")}, // finally minted + {res: pending}, // not approved yet + {res: pending}, // still not approved + {res: approved("tunnel-1")}, // approved, and handed its tunnel credential }} - m := NewNATSCredentialManager(f.fn(), true /* requireCreds */) + m := NewCredentialManager(f.fn(), true /* requireApproval */) m.initialBackoff = time.Millisecond m.maxBackoff = time.Millisecond @@ -71,28 +68,24 @@ var _ = Describe("NATSCredentialManager", func() { Expect(err).ToNot(HaveOccurred()) Expect(res.ID).To(Equal("node-1")) Expect(f.count()).To(Equal(3)) - - jwt, seed := m.Current() - Expect(jwt).To(Equal("jwt-1")) - Expect(seed).To(Equal("seed-1")) - Expect(m.HasCredentials()).To(BeTrue()) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) Expect(m.NodeID()).To(Equal("node-1")) }) - It("returns immediately on the first success when credentials are not required (anonymous NATS)", func() { + It("returns immediately on the first success when approval is not required", func() { f := &fakeRegister{steps: []step{{res: pending}}} - m := NewNATSCredentialManager(f.fn(), false /* requireCreds */) + m := NewCredentialManager(f.fn(), false /* requireApproval */) res, err := m.Acquire(context.Background()) Expect(err).ToNot(HaveOccurred()) Expect(res.Status).To(Equal("pending")) Expect(f.count()).To(Equal(1)) - Expect(m.HasCredentials()).To(BeFalse()) + Expect(m.TunnelToken()).To(BeEmpty()) }) It("aborts when the context is cancelled while waiting for approval", func() { f := &fakeRegister{steps: []step{{res: pending}}} - m := NewNATSCredentialManager(f.fn(), true) + m := NewCredentialManager(f.fn(), true) m.initialBackoff = 10 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) @@ -103,7 +96,7 @@ var _ = Describe("NATSCredentialManager", func() { It("gives up after a bounded number of attempts so the worker exits and alerts", func() { f := &fakeRegister{steps: []step{{res: pending}}} // never approved - m := NewNATSCredentialManager(f.fn(), true) + m := NewCredentialManager(f.fn(), true) m.initialBackoff = time.Millisecond m.maxBackoff = time.Millisecond m.maxAttempts = 5 @@ -116,83 +109,48 @@ var _ = Describe("NATSCredentialManager", func() { }) }) - Describe("RefreshLoop (#5 — renew before the JWT expires)", func() { - It("re-registers before expiry and updates the credentials served to new connections", func() { - f := &fakeRegister{steps: []step{{res: approved("jwt-2", "seed-2")}}} - m := NewNATSCredentialManager(f.fn(), true) - m.refreshLead = 0.5 - m.refreshRetry = time.Millisecond - // jwt-1 expires soon; jwt-2 is long-lived so the loop then idles. - m.expiryOf = func(jwt string) (time.Time, bool) { - switch jwt { - case "jwt-1": - return time.Now().Add(40 * time.Millisecond), true - case "jwt-2": - return time.Now().Add(time.Hour), true - default: - return time.Time{}, false - } - } - m.store(approved("jwt-1", "seed-1")) + // The behaviour that survived the rename, and which nothing pinned while + // the manager was about a JWT. + // + // The frontend mints a FRESH tunnel token on every registration and keeps + // only its hash, so the previous one stops working the moment a new one is + // issued. A manager that handed out the first value it saw would lock the + // worker out of its own tunnel after any re-registration, and the symptom + // would be a 401 on a dial rather than anything at registration time. + Describe("TunnelToken (rotation)", func() { + It("serves the token from the most recent registration, not the first", func() { + m := NewCredentialManager(nil, false) + m.store(approved("tunnel-1")) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - go func() { _ = m.RefreshLoop(ctx) }() - - Eventually(func() string { - jwt, _ := m.Current() - return jwt - }, "2s", "10ms").Should(Equal("jwt-2")) + m.store(approved("tunnel-2")) + Expect(m.TunnelToken()).To(Equal("tunnel-2"), + "the frontend keeps only the newest token's hash, so serving the first one locks this worker out of its own tunnel") }) - It("returns an error after the bounded number of consecutive failures so the caller can exit", func() { - f := &fakeRegister{steps: []step{{err: context.DeadlineExceeded}}} // refresh always fails - m := NewNATSCredentialManager(f.fn(), true) - m.refreshLead = 0.5 - m.refreshRetry = time.Millisecond - m.maxAttempts = 3 - m.expiryOf = func(string) (time.Time, bool) { return time.Now().Add(time.Millisecond), true } - m.store(approved("jwt-1", "seed-1")) - - errCh := make(chan error, 1) - go func() { errCh <- m.RefreshLoop(context.Background()) }() - Eventually(errCh, "2s").Should(Receive(MatchError(ContainSubstring("3 times in a row")))) - }) - - It("exits promptly when the current credential has no expiry (nothing to refresh)", func() { - f := &fakeRegister{steps: []step{{res: approved("x", "y")}}} - m := NewNATSCredentialManager(f.fn(), true) - m.expiryOf = func(string) (time.Time, bool) { return time.Time{}, false } - m.store(approved("static", "seed")) - - done := make(chan struct{}) - go func() { _ = m.RefreshLoop(context.Background()); close(done) }() - Eventually(done, "1s").Should(BeClosed()) - Expect(f.count()).To(Equal(0)) // never tried to re-register - }) - }) - - Describe("jwtExpiry default", func() { - It("decodes the expiry of a real minted worker JWT", func() { - akp, err := nkeys.CreateAccount() - Expect(err).ToNot(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).ToNot(HaveOccurred()) - - cfg := natsauth.Config{AccountSeed: string(seed), WorkerJWTTTL: time.Hour} - token, _, err := cfg.MintWorkerJWT("node-1", "backend") - Expect(err).ToNot(HaveOccurred()) - - exp, ok := jwtExpiry(token) - Expect(ok).To(BeTrue()) - Expect(exp).To(BeTemporally("~", time.Now().Add(time.Hour), 2*time.Minute)) - }) - - It("reports no expiry for an empty or undecodable token", func() { - _, ok := jwtExpiry("") - Expect(ok).To(BeFalse()) - _, ok = jwtExpiry("not-a-jwt") - Expect(ok).To(BeFalse()) + It("keeps a working token when a registration carries none", func() { + // A frontend that predates tunnel tokens, or one whose minting + // failed, sends the field empty. Empty means "nothing new", not + // "revoked": overwriting would lock the tunnel out until the next + // registration that did carry one. + m := NewCredentialManager(nil, false) + m.store(approved("tunnel-1")) + m.store(approved("")) + Expect(m.TunnelToken()).To(Equal("tunnel-1")) }) }) }) + +// Deleted with the bus: the "RefreshLoop" Describe and the "jwtExpiry default" +// Describe. +// +// RefreshLoop pinned that a worker re-registers before its minted broker JWT +// expires and serves the new credential to the next connection. Every noun in +// that sentence is gone: there is no JWT, no expiry to read, and no connection +// to serve it to. It is retired rather than moved. The one credential a +// registration still mints, the tunnel token, is rotated by a REGISTRATION +// rather than by a clock, and the property that matters about it is that the +// dialer reads the current value, which the rotation Describe above pins. +// +// jwtExpiry pinned that a real minted worker JWT's expiry decoded, which was +// the only thing in this package that needed a broker library at all. diff --git a/core/config/distributed_config.go b/core/config/distributed_config.go index 65334b86b..1236d77b8 100644 --- a/core/config/distributed_config.go +++ b/core/config/distributed_config.go @@ -5,13 +5,12 @@ import ( "fmt" "time" - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/xlog" ) // DistributedConfig holds configuration for horizontal scaling mode. -// When Enabled is true, PostgreSQL and NATS are required. +// When Enabled is true, PostgreSQL is required. Nothing else is: fan-out rides +// PostgreSQL and every worker is reached over the tunnel it dials out. type DistributedConfig struct { Enabled bool // --distributed / LOCALAI_DISTRIBUTED InstanceID string // --instance-id / LOCALAI_INSTANCE_ID (auto-generated UUID if empty) @@ -22,22 +21,19 @@ type DistributedConfig struct { // answer is only usable when the database is remote, so a deployment with // a local or sidecar database has to set this. AdvertiseAddr string // LOCALAI_DISTRIBUTED_ADVERTISE_ADDR - // NatsURL is accepted and ignored. No component of a distributed - // deployment dials a message bus any more. - NatsURL string // --nats-url / LOCALAI_NATS_URL StorageURL string // --storage-url / LOCALAI_STORAGE_URL (S3 endpoint) RegistrationToken string // --registration-token / LOCALAI_REGISTRATION_TOKEN (required token for node registration) // RegistrationRequireAuth fails startup when distributed mode is enabled but // RegistrationToken is empty. The default (false) keeps the historical // fail-open behavior with a loud warning; production should set it so the // node-register endpoints and the worker file-transfer server cannot run - // unauthenticated. Mirrors NatsRequireAuth for the NATS bus. + // unauthenticated. RegistrationRequireAuth bool // LOCALAI_REGISTRATION_REQUIRE_AUTH // RequireAuth is the umbrella switch (LOCALAI_DISTRIBUTED_REQUIRE_AUTH) for - // distributed-mode auth: when true it implies BOTH NatsRequireAuth and - // RegistrationRequireAuth, so a single knob locks down the bus and the - // registration/file-transfer layer together. The granular flags remain - // available to enforce just one layer. + // distributed-mode auth. It implies RegistrationRequireAuth, and that is + // now ALL it implies: it used to imply a NatsRequireAuth as well, and there + // is no message bus left for that half to lock down. The granular flag + // remains available. RequireAuth bool // LOCALAI_DISTRIBUTED_REQUIRE_AUTH AutoApproveNodes bool // --auto-approve-nodes / LOCALAI_AUTO_APPROVE_NODES (skip admin approval for new workers) // SharedModels asserts that every node (frontend and workers) mounts the @@ -49,16 +45,6 @@ type DistributedConfig struct { // false preserves the historical per-node staging behavior. SharedModels bool // --distributed-shared-models / LOCALAI_DISTRIBUTED_SHARED_MODELS - // NATS JWT auth (optional; see pkg/natsauth and docs/features/distributed-mode.md) - NatsAccountSeed string // LOCALAI_NATS_ACCOUNT_SEED — account signing seed to mint per-node worker JWTs - NatsServiceJWT string // LOCALAI_NATS_SERVICE_JWT — user JWT for frontends / agent workers - NatsServiceSeed string // LOCALAI_NATS_SERVICE_SEED — signing seed paired with service JWT - NatsWorkerJWTTTL time.Duration // LOCALAI_NATS_WORKER_JWT_TTL — minted worker JWT lifetime (default 24h) - NatsRequireAuth bool // LOCALAI_NATS_REQUIRE_AUTH — fail startup if NATS credentials are missing - NatsTLSCA string // LOCALAI_NATS_TLS_CA — PEM file for private CA (server verify) - NatsTLSCert string // LOCALAI_NATS_TLS_CERT — client cert for NATS mTLS - NatsTLSKey string // LOCALAI_NATS_TLS_KEY — client key paired with NatsTLSCert - // S3 configuration (used when StorageURL is set) StorageBucket string // --storage-bucket / LOCALAI_STORAGE_BUCKET StorageRegion string // --storage-region / LOCALAI_STORAGE_REGION @@ -94,8 +80,8 @@ type DistributedConfig struct { // database clock, so every replica agrees on when the window ends. WorkerReconnectGrace time.Duration // LOCALAI_WORKER_RECONNECT_GRACE - BackendInstallTimeout time.Duration // NATS round-trip timeout for backend.install (default 15m) - BackendUpgradeTimeout time.Duration // NATS round-trip timeout for backend.upgrade (default 15m) + BackendInstallTimeout time.Duration // control round-trip timeout for backend.install (default 15m) + BackendUpgradeTimeout time.Duration // control round-trip timeout for backend.upgrade (default 15m) // ModelLoadTimeout is the gRPC deadline for the remote LoadModel call the // router issues once a worker has the backend installed and the model files // staged. It therefore covers only the backend's own checkpoint load and @@ -178,13 +164,6 @@ func (c DistributedConfig) Validate() error { } xlog.Warn("distributed mode running without registration token — node endpoints and the worker file-transfer server are unprotected; set LOCALAI_REGISTRATION_TOKEN, or LOCALAI_DISTRIBUTED_REQUIRE_AUTH=true to fail closed") } - if err := c.NatsAuthConfig().Validate(); err != nil { - return err - } - if err := c.NatsTLSFiles().Validate(); err != nil { - return err - } - c.NatsAuthConfig().WarnIfInsecure(true) // Check for negative durations for name, d := range map[string]time.Duration{ FlagMCPToolTimeout: c.MCPToolTimeout, @@ -226,54 +205,20 @@ func WithDistributedAdvertiseAddr(addr string) AppOption { } } -func WithNatsURL(url string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsURL = url - } -} - func WithRegistrationToken(token string) AppOption { return func(o *ApplicationConfig) { o.Distributed.RegistrationToken = token } } -func WithNatsAccountSeed(seed string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsAccountSeed = seed - } -} - -func WithNatsServiceJWT(jwt string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsServiceJWT = jwt - } -} - -func WithNatsServiceSeed(seed string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsServiceSeed = seed - } -} - -func WithNatsWorkerJWTTTL(d time.Duration) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsWorkerJWTTTL = d - } -} - -var EnableNatsRequireAuth = func(o *ApplicationConfig) { - o.Distributed.NatsRequireAuth = true -} - // EnableRegistrationRequireAuth makes an empty registration token a hard error // in distributed mode (see DistributedConfig.RegistrationRequireAuth). var EnableRegistrationRequireAuth = func(o *ApplicationConfig) { o.Distributed.RegistrationRequireAuth = true } -// EnableDistributedRequireAuth is the umbrella switch implying both -// NatsRequireAuth and RegistrationRequireAuth (see DistributedConfig.RequireAuth). +// EnableDistributedRequireAuth is the umbrella switch implying +// RegistrationRequireAuth (see DistributedConfig.RequireAuth). var EnableDistributedRequireAuth = func(o *ApplicationConfig) { o.Distributed.RequireAuth = true } @@ -284,30 +229,6 @@ func (c DistributedConfig) RegistrationAuthRequired() bool { return c.RegistrationRequireAuth || c.RequireAuth } -// NatsAuthRequired reports whether NATS JWT credentials must be present — the -// granular flag or the umbrella. -func (c DistributedConfig) NatsAuthRequired() bool { - return c.NatsRequireAuth || c.RequireAuth -} - -func WithNatsTLSCA(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSCA = path - } -} - -func WithNatsTLSCert(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSCert = path - } -} - -func WithNatsTLSKey(path string) AppOption { - return func(o *ApplicationConfig) { - o.Distributed.NatsTLSKey = path - } -} - func WithStorageURL(url string) AppOption { return func(o *ApplicationConfig) { o.Distributed.StorageURL = url @@ -502,44 +423,6 @@ const ModelLoadWaitUnbounded = -1 * time.Second // DefaultMaxUploadSize is the default maximum upload body size (50 GB). const DefaultMaxUploadSize int64 = 50 << 30 -// NatsTLSFiles returns NATS TLS/mTLS PEM paths for the messaging client. -func (c DistributedConfig) NatsTLSFiles() messaging.TLSFiles { - return messaging.TLSFiles{ - CA: c.NatsTLSCA, - Cert: c.NatsTLSCert, - Key: c.NatsTLSKey, - } -} - -// NatsMessagingOptions builds messaging client options (JWT + TLS) for distributed components. -// Pass explicit userJWT/userSeed when set (e.g. worker overrides); empty uses service JWT from config. -func (c DistributedConfig) NatsMessagingOptions(userJWT, userSeed string) []messaging.Option { - var opts []messaging.Option - jwt, seed := userJWT, userSeed - if jwt == "" && seed == "" { - auth := c.NatsAuthConfig() - jwt, seed = auth.ServiceUserJWT, auth.ServiceUserSeed - } - if jwt != "" && seed != "" { - opts = append(opts, messaging.WithUserJWT(jwt, seed)) - } - if tls := c.NatsTLSFiles(); tls.Enabled() { - opts = append(opts, messaging.WithTLS(tls)) - } - return opts -} - -// NatsAuthConfig builds pkg/natsauth settings from distributed configuration. -func (c DistributedConfig) NatsAuthConfig() natsauth.Config { - return natsauth.Config{ - AccountSeed: c.NatsAccountSeed, - ServiceUserJWT: c.NatsServiceJWT, - ServiceUserSeed: c.NatsServiceSeed, - WorkerJWTTTL: c.NatsWorkerJWTTTL, - RequireAuth: c.NatsAuthRequired(), - } -} - // ReconnectGraceOrDefault returns the configured worker reconnect grace or the // default. // diff --git a/core/config/distributed_config_test.go b/core/config/distributed_config_test.go index deab290fa..14049e557 100644 --- a/core/config/distributed_config_test.go +++ b/core/config/distributed_config_test.go @@ -1,6 +1,8 @@ package config_test import ( + "reflect" + "strings" "time" . "github.com/onsi/ginkgo/v2" @@ -73,7 +75,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative BackendInstallTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", BackendInstallTimeout: -1 * time.Second, } err := c.Validate() @@ -85,7 +86,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative BackendUpgradeTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", BackendUpgradeTimeout: -1 * time.Second, } err := c.Validate() @@ -96,7 +96,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("rejects a negative ModelLoadTimeout with the flag name in the error", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", ModelLoadTimeout: -1 * time.Second, } err := c.Validate() @@ -108,7 +107,6 @@ var _ = Describe("DistributedConfig.Validate negative-duration errors", func() { It("accepts all-zero durations as valid (defaults apply)", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", } Expect(c.Validate()).To(Succeed()) }) @@ -118,7 +116,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("rejects an empty registration token when RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationRequireAuth: true, } err := c.Validate() @@ -130,7 +127,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("accepts a set registration token when RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationToken: "s3cret", RegistrationRequireAuth: true, } @@ -140,7 +136,6 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("warns but succeeds with an empty token when RequireAuth is unset", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", } Expect(c.Validate()).To(Succeed()) }) @@ -148,12 +143,7 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { It("rejects an empty token when the umbrella RequireAuth is set", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RequireAuth: true, - // Provide NATS creds so only the registration-token gap remains. - NatsServiceJWT: "jwt", - NatsServiceSeed: "seed", - NatsAccountSeed: "acct", } err := c.Validate() Expect(err).To(HaveOccurred()) @@ -161,19 +151,25 @@ var _ = Describe("DistributedConfig.Validate registration auth", func() { Expect(err.Error()).To(ContainSubstring("LOCALAI_REGISTRATION_TOKEN")) }) - It("the umbrella implies NATS auth is required", func() { + // The umbrella used to imply two things, and now implies one. + // + // The It that stood here pinned "LOCALAI_DISTRIBUTED_REQUIRE_AUTH makes a + // missing broker service JWT fatal". That is retired, not moved: there is + // no broker connection to demand a credential for, so a startup that failed + // on a missing one would fail on the absence of something nothing uses. The + // half of the umbrella that survives is the registration layer, and the It + // above it pins exactly that: an empty registration token under the + // umbrella is still fatal, and still says which knob to set. + It("implies the registration layer and nothing else", func() { c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", - RegistrationToken: "tok", // registration layer satisfied - RequireAuth: true, // umbrella → NATS creds now required + RegistrationToken: "tok", + RequireAuth: true, } - Expect(c.NatsAuthRequired()).To(BeTrue()) Expect(c.RegistrationAuthRequired()).To(BeTrue()) - // Missing NATS service JWT/seed must now be fatal. - err := c.Validate() - Expect(err).To(HaveOccurred()) - Expect(err.Error()).To(ContainSubstring("LOCALAI_NATS_REQUIRE_AUTH")) + // And with the registration layer satisfied there is nothing else left + // for the umbrella to demand, so startup succeeds. + Expect(c.Validate()).To(Succeed()) }) }) @@ -210,7 +206,6 @@ var _ = Describe("DistributedConfig worker reconnect grace", func() { // a caller may reap and evict on. c := config.DistributedConfig{ Enabled: true, - NatsURL: "nats://localhost:4222", RegistrationToken: "tok", WorkerReconnectGrace: -1 * time.Second, } @@ -225,3 +220,25 @@ var _ = Describe("DistributedConfig worker reconnect grace", func() { Expect(o.Distributed.ReconnectGraceOrDefault()).To(Equal(90 * time.Second)) }) }) + +// The frontend's distributed configuration has nowhere to put a broker URL, and +// that absence is what makes the accepted-and-ignored CLI flags ignored. +// +// A help string saying "ignored" is a promise; a missing field is the mechanism. +// Asserted by reflection rather than by reading a value, because a value +// assertion needs a field to read and would therefore stop compiling exactly +// when the property it guards is restored, which is the failure mode this +// replaces: a spec that vanishes with the regression it was meant to catch. +var _ = Describe("the distributed configuration's broker surface", func() { + It("carries no NATS credential, TLS or URL field", func() { + t := reflect.TypeOf(config.DistributedConfig{}) + var carried []string + for i := range t.NumField() { + if strings.HasPrefix(t.Field(i).Name, "Nats") { + carried = append(carried, t.Field(i).Name) + } + } + Expect(carried).To(BeEmpty(), + "DistributedConfig grew %v back: a value the frontend can store is a value something can dial, and no component of a distributed deployment dials a message bus", carried) + }) +}) diff --git a/core/http/app.go b/core/http/app.go index 530d149ef..19e185bd7 100644 --- a/core/http/app.go +++ b/core/http/app.go @@ -509,8 +509,10 @@ func API(application *application.Application) (*echo.Echo, error) { // "operation visible" and "operation gone" between replicas. // // S1. The carrier choice lives in core/application with the other three - // caches, and this call names no carrier at all, so the NATS client - // hanging off the same struct cannot be handed over here by accident. + // caches, and this call names no carrier at all, so nothing hanging off + // the same struct can be handed over here by accident. See + // core/application/cache_fanout_wiring.go for why that shape is kept + // now that the broker's client is no longer one of those things. if d := application.Distributed(); d != nil { if err := d.WireOpCache(application.ApplicationConfig().Context, opcache); err != nil { xlog.Warn("OpCache distributed subscribe failed; running standalone", "error", err) @@ -587,9 +589,8 @@ func API(application *application.Application) (*echo.Echo, error) { } } } - natsCfg := distCfg.NatsAuthConfig() - routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, natsCfg) - routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, natsCfg, workerHTTPDialFor) + routes.RegisterNodeSelfServiceRoutes(e, registry, distCfg.RegistrationToken, distCfg.AutoApproveNodes, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret) + routes.RegisterNodeAdminRoutes(e, registry, remoteUnloader, application.GalleryService(), opcache, application.ApplicationConfig(), adminMiddleware, application.AuthDB(), application.ApplicationConfig().Auth.APIKeyHMACSecret, application.ApplicationConfig().Distributed.RegistrationToken, workerHTTPDialFor) // Replica-to-replica peer link. Registered only in distributed mode: in // single-node mode there are no peers, and the route authenticates with the diff --git a/core/http/auth/helpers_test.go b/core/http/auth/helpers_test.go index 1e31ac27f..047a4e188 100644 --- a/core/http/auth/helpers_test.go +++ b/core/http/auth/helpers_test.go @@ -12,7 +12,6 @@ import ( "github.com/mudler/LocalAI/core/http/auth" "github.com/mudler/LocalAI/core/http/routes" "github.com/mudler/LocalAI/core/services/nodes" - "github.com/mudler/LocalAI/pkg/natsauth" . "github.com/onsi/gomega" "gorm.io/gorm" ) @@ -114,7 +113,6 @@ func newNodeSelfServiceTestApp(db *gorm.DB, appConfig *config.ApplicationConfig, false, nil, "", - natsauth.Config{}, ) return e } diff --git a/core/http/endpoints/cluster/connect.go b/core/http/endpoints/cluster/connect.go index 8d1757046..d660c9c21 100644 --- a/core/http/endpoints/cluster/connect.go +++ b/core/http/endpoints/cluster/connect.go @@ -128,13 +128,14 @@ func ConnectHandler(registry *nodes.NodeRegistry, tunnels *clustersvc.TunnelRegi // 401 would send an operator looking at tokens. // // Only StatusPending is refused. The rest of /api/node/ self-service - // gates on nothing at all, but the two places that hand a node something - // DURABLE both refuse a pending one: the agent worker's API key - // (provisionAgentWorkerKey, guarded at its call site in - // core/http/endpoints/localai/nodes.go) and its NATS credential - // (attachNatsJWT in the same file). Cited by NAME, not by line: the - // previous version of this comment cited line numbers into a file this - // same commit was editing, and both were stale before it landed. + // gates on nothing at all, but the one place that hands a node + // something DURABLE refuses a pending one too: the agent worker's API + // key (provisionAgentWorkerKey, guarded at its call site in + // core/http/endpoints/localai/nodes.go). Cited by NAME, not by line: + // an earlier version of this comment cited line numbers into a file the + // same commit was editing, and both were stale before it landed. The + // per-node broker credential this used to name alongside it is gone + // with the bus, and so is the function that minted it. // // A tunnel is that kind of grant, not a heartbeat: it is // a standing pipe into the worker recorded in node_connections and diff --git a/core/http/endpoints/localai/nodes.go b/core/http/endpoints/localai/nodes.go index d98572810..fc6bbc6c5 100644 --- a/core/http/endpoints/localai/nodes.go +++ b/core/http/endpoints/localai/nodes.go @@ -31,7 +31,6 @@ import ( "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/nodes/prefixcache" "github.com/mudler/LocalAI/pkg/httpclient" - "github.com/mudler/LocalAI/pkg/natsauth" "github.com/mudler/LocalAI/pkg/vrambudget" ) @@ -117,7 +116,7 @@ type RegisterNodeRequest struct { // RegisterNodeEndpoint registers a new backend node. // expectedToken is the registration token configured on the frontend (may be empty to disable auth). // autoApprove controls whether new nodes go directly to "healthy" or require admin approval. -func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) echo.HandlerFunc { +func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string) echo.HandlerFunc { return func(c echo.Context) error { var req RegisterNodeRequest if err := c.Bind(&req); err != nil { @@ -242,7 +241,6 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au } attachTunnelToken(ctx, response, registry, node) - attachNatsJWT(response, node, natsCfg) return c.JSON(http.StatusCreated, response) } @@ -250,7 +248,7 @@ func RegisterNodeEndpoint(registry *nodes.NodeRegistry, expectedToken string, au // ApproveNodeEndpoint approves a pending node, setting its status to healthy. // For agent workers, it also provisions an API key so they can call the inference API. -func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) echo.HandlerFunc { +func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecret string) echo.HandlerFunc { return func(c echo.Context) error { ctx := c.Request().Context() id := c.Param("id") @@ -280,8 +278,6 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr } } - attachNatsJWT(response, node, natsCfg) - return c.JSON(http.StatusOK, response) } } @@ -290,11 +286,11 @@ func ApproveNodeEndpoint(registry *nodes.NodeRegistry, authDB *gorm.DB, hmacSecr // hash, and puts the plaintext in the registration response. // // It is minted for EVERY node that registers, pending ones included, which is a -// deliberate divergence from the two other per-node credentials in this file: -// the agent worker's API key (provisionAgentWorkerKey) and its NATS JWT -// (attachNatsJWT) are both withheld from a node awaiting approval. Those two -// are bearer grants that WORK the moment they are issued, so issuing one to an -// unapproved node would route around the admin. A tunnel credential is not: +// deliberate divergence from the other per-node credential in this file: the +// agent worker's API key (provisionAgentWorkerKey) is withheld from a node +// awaiting approval. It is a bearer grant that WORKS the moment it is issued, +// so issuing one to an unapproved node would route around the admin. A tunnel +// credential is not: // core/http/endpoints/cluster/connect.go re-reads the node's status on every // dial and refuses a pending node with 403, so the credential is inert until an // admin approves and stays inert if approval is revoked. Withholding it would @@ -379,20 +375,6 @@ func tunnelEligible(nodeType string) bool { return nodeType == nodes.NodeTypeBackend || nodeType == nodes.NodeTypeAgent } -// attachNatsJWT adds a per-node NATS user JWT to a register/approve response when minting is enabled. -func attachNatsJWT(response map[string]any, node *nodes.BackendNode, natsCfg natsauth.Config) { - if !natsCfg.CanMintWorkers() || node == nil || node.Status == nodes.StatusPending { - return - } - jwt, seed, err := natsCfg.MintWorkerJWT(node.ID, node.NodeType) - if err != nil { - xlog.Warn("Failed to mint NATS JWT for node", "node", node.Name, "id", node.ID, "error", err) - return - } - response["nats_jwt"] = jwt - response["nats_user_seed"] = seed -} - // provisionAgentWorkerKey creates a dedicated user and API key for an agent worker node. // Returns the plaintext API key on success. func provisionAgentWorkerKey(ctx context.Context, authDB *gorm.DB, registry *nodes.NodeRegistry, node *nodes.BackendNode, hmacSecret string) (string, error) { diff --git a/core/http/endpoints/localai/nodes_test.go b/core/http/endpoints/localai/nodes_test.go index 0fbd07957..e8422c6cc 100644 --- a/core/http/endpoints/localai/nodes_test.go +++ b/core/http/endpoints/localai/nodes_test.go @@ -11,11 +11,10 @@ import ( "strings" "time" + "github.com/google/uuid" "github.com/labstack/echo/v4" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/testutil" - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/nkeys" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -73,7 +72,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -92,7 +91,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, expectedToken, autoApprove, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, expectedToken, autoApprove, nil, "") ExpectWithOffset(1, handler(c)).To(Succeed()) ExpectWithOffset(1, rec.Code).To(Equal(http.StatusCreated)) @@ -275,27 +274,61 @@ var _ = Describe("Node HTTP handlers", func() { "an ineligible node kept a usable tunnel credential, so the mint-site gate is not structural") }) - It("returns nats_jwt when account seed is configured", func() { - akp, err := nkeys.CreateAccount() - Expect(err).ToNot(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).ToNot(HaveOccurred()) - + // The register path, and the approve path is asserted separately below: + // the helper that attached a per-node broker credential had two call + // sites, and one restored call site is exactly the regression a single + // spec would miss. + // + // Asserted on the DECODED MAP with ToNot(HaveKey(...)), not on a struct. + // A struct assertion would pass vacuously: the response is assembled as a + // map[string]any, so the only way a key can be present is if something put + // it there, and the only way to see that is to look at the keys. + It("mints no broker credential into a registration response", func() { e := echo.New() - body := `{"name":"worker-nats","address":"10.0.0.2:50051"}` + body := `{"name":"worker-no-bus","address":"10.0.0.2:50051"}` req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body)) req.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) rec := httptest.NewRecorder() c := e.NewContext(req, rec) - natsCfg := natsauth.Config{AccountSeed: string(seed)} - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsCfg) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) var resp map[string]any Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) - Expect(resp["nats_jwt"]).ToNot(BeEmpty()) + // The keys it DOES carry, so the two absences below are facts about + // the response and not about a decode that produced an empty map. + Expect(resp).To(HaveKey("id")) + Expect(resp).To(HaveKey("tunnel_token")) + Expect(resp).ToNot(HaveKey("nats_jwt")) + Expect(resp).ToNot(HaveKey("nats_user_seed")) + }) + + It("mints no broker credential into an approval response either", func() { + node := &nodes.BackendNode{ + ID: uuid.New().String(), + Name: "worker-approve-no-bus", + Address: "10.0.0.3:50051", + NodeType: nodes.NodeTypeBackend, + } + Expect(registry.Register(context.Background(), node, false)).To(Succeed()) + + e := echo.New() + req := httptest.NewRequest(http.MethodPost, "/", nil) + rec := httptest.NewRecorder() + c := e.NewContext(req, rec) + c.SetParamNames("id") + c.SetParamValues(node.ID) + + Expect(ApproveNodeEndpoint(registry, nil, "")(c)).To(Succeed()) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var resp map[string]any + Expect(json.Unmarshal(rec.Body.Bytes(), &resp)).To(Succeed()) + Expect(resp).To(HaveKey("id")) + Expect(resp).ToNot(HaveKey("nats_jwt")) + Expect(resp).ToNot(HaveKey("nats_user_seed")) }) It("returns 400 when name is missing", func() { @@ -306,7 +339,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -326,7 +359,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -349,7 +382,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -372,7 +405,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -390,7 +423,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusBadRequest)) @@ -409,7 +442,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "correct-token", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "correct-token", true, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusUnauthorized)) }) @@ -422,7 +455,7 @@ var _ = Describe("Node HTTP handlers", func() { rec := httptest.NewRecorder() c := e.NewContext(req, rec) - handler := RegisterNodeEndpoint(registry, "", false, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", false, nil, "") Expect(handler(c)).To(Succeed()) Expect(rec.Code).To(Equal(http.StatusCreated)) @@ -445,7 +478,7 @@ var _ = Describe("Node HTTP handlers", func() { req1 := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(body1)) req1.Header.Set(echo.HeaderContentType, echo.MIMEApplicationJSON) rec1 := httptest.NewRecorder() - handler := RegisterNodeEndpoint(registry, "", true, nil, "", natsauth.Config{}) + handler := RegisterNodeEndpoint(registry, "", true, nil, "") Expect(handler(e.NewContext(req1, rec1))).To(Succeed()) Expect(rec1.Code).To(Equal(http.StatusCreated)) diff --git a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js index 33d871442..e790892d4 100644 --- a/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js +++ b/core/http/react-ui/e2e/nodes-per-node-backend-actions.spec.js @@ -5,7 +5,7 @@ import { test, expect } from './coverage-fixtures.js' // - a delete affordance is present and goes through ConfirmDialog // // We mock the distributed-mode API so the tests can run against the -// standalone ui-test-server without spinning up workers/NATS. +// standalone ui-test-server without spinning up workers. const NODE_ID = 'test-node-1' const NODE_NAME = 'worker-test' diff --git a/core/http/routes/nodes.go b/core/http/routes/nodes.go index a511c6b7c..e9377ab50 100644 --- a/core/http/routes/nodes.go +++ b/core/http/routes/nodes.go @@ -10,7 +10,6 @@ import ( "github.com/mudler/LocalAI/core/http/endpoints/localai" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/nodes" - "github.com/mudler/LocalAI/pkg/natsauth" "gorm.io/gorm" ) @@ -36,7 +35,7 @@ func nodeReadyMiddleware(registry *nodes.NodeRegistry) echo.MiddlewareFunc { // token but do not verify per-node identity. A compromised worker can heartbeat/drain/ // deregister other nodes. Future: issue per-node JWT at registration, validate node // identity on subsequent requests (compare :id param with token subject). -func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, registrationToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string, natsCfg natsauth.Config) { +func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, registrationToken string, autoApprove bool, authDB *gorm.DB, hmacSecret string) { if registry == nil { return } @@ -45,7 +44,7 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r tokenAuthMw := nodeTokenAuth(registrationToken) node := e.Group("/api/node", readyMw, tokenAuthMw) - node.POST("/register", localai.RegisterNodeEndpoint(registry, registrationToken, autoApprove, authDB, hmacSecret, natsCfg)) + node.POST("/register", localai.RegisterNodeEndpoint(registry, registrationToken, autoApprove, authDB, hmacSecret)) node.POST("/:id/heartbeat", localai.HeartbeatEndpoint(registry)) node.POST("/:id/drain", localai.DrainNodeEndpoint(registry)) node.POST("/:id/resume", localai.ResumeNodeEndpoint(registry)) @@ -67,7 +66,7 @@ func RegisterNodeSelfServiceRoutes(e *echo.Echo, registry *nodes.NodeRegistry, r // registered. It is nil outside distributed mode, and those two routes then // answer 502 rather than dialling, because a worker with no tunnel has nothing // for them to proxy to. -func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, natsCfg natsauth.Config, workerDialFor nodes.WorkerNetDialerFor) { +func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloader nodes.NodeCommandSender, galleryService *galleryop.GalleryService, opcache *galleryop.OpCache, appConfig *config.ApplicationConfig, adminMw echo.MiddlewareFunc, authDB *gorm.DB, hmacSecret string, registrationToken string, workerDialFor nodes.WorkerNetDialerFor) { if registry == nil { return } @@ -91,7 +90,7 @@ func RegisterNodeAdminRoutes(e *echo.Echo, registry *nodes.NodeRegistry, unloade admin.DELETE("/:id", localai.DeregisterNodeEndpoint(registry)) admin.POST("/:id/drain", localai.DrainNodeEndpoint(registry)) admin.POST("/:id/resume", localai.ResumeNodeEndpoint(registry)) - admin.POST("/:id/approve", localai.ApproveNodeEndpoint(registry, authDB, hmacSecret, natsCfg)) + admin.POST("/:id/approve", localai.ApproveNodeEndpoint(registry, authDB, hmacSecret)) // Backend management on workers admin.GET("/:id/backends", localai.ListBackendsOnNodeEndpoint(unloader, registry)) diff --git a/core/http/routes/openresponses.go b/core/http/routes/openresponses.go index 235639a08..085e022f6 100644 --- a/core/http/routes/openresponses.go +++ b/core/http/routes/openresponses.go @@ -94,9 +94,11 @@ func setOpenResponsesRequestContext(re *middleware.RequestExtractor) echo.Middle // // A named function rather than a block inside route registration, and that is // the point of it. EnableDistributed takes a messaging.Broadcaster, as it must: -// its own specs publish through a double. So handing it the NATS client instead -// of the deployment's carrier COMPILES and reddens nothing anywhere, and the -// only symptom is a cancel that answers 404 on every replica but one. Registering +// its own specs publish through a double. So handing it any carrier other than +// the deployment's COMPILES and reddens nothing anywhere, and the only symptom +// is a cancel that answers 404 on every replica but one. The broker client that +// used to be the second carrier in scope is gone; what pins the choice is the +// spec beside this file, which drives it from the OTHER carrier. Registering // routes needs a whole Application and therefore has no spec; this needs a // DistributedServices and a store, and therefore has one. func enableDistributedResponses(ctx context.Context, d *application.DistributedServices, diff --git a/core/http/routes/openresponses_wiring_test.go b/core/http/routes/openresponses_wiring_test.go index 149251139..9720b4061 100644 --- a/core/http/routes/openresponses_wiring_test.go +++ b/core/http/routes/openresponses_wiring_test.go @@ -24,10 +24,11 @@ import ( // EnableDistributed takes a messaging.Broadcaster, which it must: its own specs // publish through a double, and it cannot be made to name a concrete carrier // without dragging that dependency through the whole endpoint package. The -// consequence is that handing it the NATS client instead of the deployment's -// carrier compiles and reddens nothing, and the only symptom is a cancel that -// answers 404 on every replica but the creator. So it is pinned here, by -// watching what actually arrives on the carrier. +// consequence is that handing it any carrier other than the deployment's +// compiles and reddens nothing, and the only symptom is a cancel that answers +// 404 on every replica but the creator. So it is pinned here, by watching what +// actually arrives on the carrier: busB below IS the other carrier, which is +// why this spec keeps its force now that the broker's client is gone. var _ = Describe("wiring the Open Responses store to a carrier", func() { var ( ctx context.Context diff --git a/core/services/finetune/service.go b/core/services/finetune/service.go index 8e3978a00..65504a688 100644 --- a/core/services/finetune/service.go +++ b/core/services/finetune/service.go @@ -50,9 +50,10 @@ type FineTuneService struct { // across replicas; pass nil for both in standalone mode, where the disk Loader // hydrates the map and there is nothing to broadcast. // -// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// bus is messaging.Broadcaster and not a concrete carrier: this state.*.delta // family travels on whatever the deployment's fan-out carrier is, and in -// distributed mode that is PostgreSQL LISTEN/NOTIFY. +// distributed mode that is PostgreSQL LISTEN/NOTIFY. Which one it gets is +// decided in core/application, not here. func NewFineTuneService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, diff --git a/core/services/messaging/client.go b/core/services/messaging/client.go deleted file mode 100644 index 47a98862e..000000000 --- a/core/services/messaging/client.go +++ /dev/null @@ -1,282 +0,0 @@ -package messaging - -import ( - "encoding/json" - "errors" - "fmt" - "strings" - "sync" - "time" - - "github.com/mudler/LocalAI/pkg/sanitize" - "github.com/mudler/xlog" - "github.com/nats-io/nats.go" - "github.com/nats-io/nkeys" -) - -// subscribeConfirmTimeout bounds the server round-trip used to detect whether a -// subscription was rejected (e.g. by JWT permissions) before returning to the caller. -const subscribeConfirmTimeout = 5 * time.Second - -// Client is a NATS connection, and NO PRODUCTION PATH CONSTRUCTS ONE. -// -// The last family that needed a bus was agent..cancel, whose subscriber is -// the agent WORKER: it has no database, so it could never join the PostgreSQL -// carrier the rest of the deployment fans out on. It does not need a bus either -// now, because it holds an outward tunnel and a cancel is a control verb on it -// (workerctl.PathAgentCancel). Nothing in core/ or pkg/ calls messaging.New. -// -// What survives here is this type, its connect options and its TLS plumbing, -// still exercised by the NATS JWT permission specs. Deleting them is a -// demolition of its own, together with the JWT minting at registration and -// pkg/natsauth's permission tables. -// -// The methods that carried everything else are already gone: queue -// subscriptions became a claim on the job store, and request/reply became a -// streaming control RPC on the tunnel each worker dials. Deleting the METHODS -// rather than only the call sites is what makes putting a family back on this -// carrier a build error, instead of a line that compiles, publishes -// successfully, and is delivered onto a carrier nothing reads. -type Client struct { - conn *nats.Conn - mu sync.RWMutex - - // reconnectCbs are invoked after the underlying connection is - // re-established. nats.go transparently resubscribes existing - // subscriptions on reconnect, but it cannot know that a consumer kept - // derived in-memory state (e.g. syncstate.SyncedMap) that may have drifted - // while the link was down — these callbacks let such consumers re-hydrate. - cbMu sync.Mutex - reconnectCbs []func() -} - -// New creates a new NATS client with auto-reconnect. -func New(url string, opts ...Option) (*Client, error) { - var cfg connectConfig - for _, o := range opts { - o(&cfg) - } - - // Allocate the client up front so the reconnect handler closure can reach - // it; conn is populated after nats.Connect succeeds below. - c := &Client{} - - natsOpts := []nats.Option{ - nats.RetryOnFailedConnect(true), - nats.MaxReconnects(-1), - nats.DisconnectErrHandler(func(_ *nats.Conn, err error) { - if err != nil { - xlog.Warn("NATS disconnected", "error", err) - } - }), - nats.ReconnectHandler(func(_ *nats.Conn) { - xlog.Info("NATS reconnected") - c.runReconnectCallbacks() - }), - nats.ClosedHandler(func(_ *nats.Conn) { - xlog.Info("NATS connection closed") - }), - // Surface async errors (notably permission violations) that NATS would - // otherwise deliver silently. A subscription the server rejects for a - // JWT permission means the worker never receives those messages, so make - // it loud rather than letting the feature fail invisibly. - nats.ErrorHandler(func(_ *nats.Conn, sub *nats.Subscription, err error) { - subject := "" - if sub != nil { - subject = sub.Subject - } - if errors.Is(err, nats.ErrPermissionViolation) { - xlog.Error("NATS permission violation — check JWT pub/sub allow lists", "subject", subject, "error", err) - return - } - xlog.Warn("NATS async error", "subject", subject, "error", err) - }), - } - switch { - case cfg.jwtProvider != nil: - // Fetch creds on every (re)connect so a refresh loop can rotate the JWT - // before expiry; the server expiring the old JWT triggers a reconnect - // that transparently picks up the new one. - natsOpts = append(natsOpts, nats.UserJWT( - func() (string, error) { - jwt, _ := cfg.jwtProvider() - if jwt == "" { - return "", fmt.Errorf("no NATS user JWT available") - } - return jwt, nil - }, - func(nonce []byte) ([]byte, error) { - _, seed := cfg.jwtProvider() - kp, err := nkeys.FromSeed([]byte(seed)) - if err != nil { - return nil, fmt.Errorf("loading NATS user seed: %w", err) - } - defer kp.Wipe() - return kp.Sign(nonce) - }, - )) - case cfg.userJWT != "" && cfg.userSeed != "": - natsOpts = append(natsOpts, nats.UserJWTAndSeed(cfg.userJWT, cfg.userSeed)) - } - if cfg.tls.Enabled() { - if err := cfg.tls.Validate(); err != nil { - return nil, err - } - tlsOpts, err := cfg.tls.natsOptions() - if err != nil { - return nil, err - } - natsOpts = append(natsOpts, tlsOpts...) - } - - nc, err := nats.Connect(url, natsOpts...) - if err != nil { - return nil, fmt.Errorf("connecting to NATS at %s: %w", sanitize.URL(url), err) - } - - c.conn = nc - return c, nil -} - -// OnReconnect registers a callback invoked after the NATS connection is -// re-established. It is consumed via an optional interface type-assertion -// (interface{ OnReconnect(func()) }) rather than being added to Broadcaster, so -// the messaging abstraction stays minimal and standalone/test clients are not -// forced to implement reconnect semantics. A nil callback is ignored. -func (c *Client) OnReconnect(cb func()) { - if cb == nil { - return - } - c.cbMu.Lock() - c.reconnectCbs = append(c.reconnectCbs, cb) - c.cbMu.Unlock() -} - -// runReconnectCallbacks invokes registered reconnect callbacks. It copies the -// slice under the lock so a callback that (re)registers cannot deadlock. -func (c *Client) runReconnectCallbacks() { - c.cbMu.Lock() - cbs := append([]func(){}, c.reconnectCbs...) - c.cbMu.Unlock() - for _, cb := range cbs { - cb() - } -} - -// Publish marshals data as JSON and publishes it to the given subject. -func (c *Client) Publish(subject string, data any) error { - payload, err := json.Marshal(data) - if err != nil { - return fmt.Errorf("marshalling message for %s: %w", subject, err) - } - c.mu.RLock() - defer c.mu.RUnlock() - return c.conn.Publish(subject, payload) -} - -// Subscribe creates a subscription on the given subject. All subscribers receive every message. -func (c *Client) Subscribe(subject string, handler func([]byte)) (Subscription, error) { - return c.confirmSubscription(subject, func(conn *nats.Conn) (*nats.Subscription, error) { - return conn.Subscribe(subject, func(msg *nats.Msg) { - handler(msg.Data) - }) - }) -} - -// confirmSubscription creates a subscription via mk and forces a server -// round-trip so that a permissions violation — which NATS otherwise reports -// only asynchronously — is returned to the caller synchronously. The server -// emits the "-ERR Permissions Violation" for a rejected SUB before the PONG -// that satisfies the flush, so by the time FlushTimeout returns the violation -// is recorded as the connection's last error. Without this, a worker whose JWT -// lacks a subject gets a non-nil subscription that never receives a message, -// turning a permission misconfiguration into a silent failure. -func (c *Client) confirmSubscription(subject string, mk func(*nats.Conn) (*nats.Subscription, error)) (Subscription, error) { - c.mu.RLock() - conn := c.conn - c.mu.RUnlock() - if conn == nil { - return nil, fmt.Errorf("subscribe to %s: nil NATS connection", subject) - } - - sub, err := mk(conn) - if err != nil { - return nil, err - } - - // A failed flush here means we could not round-trip to the server (not yet - // connected, reconnecting, slow link). RetryOnFailedConnect intentionally - // buffers subscriptions across that gap, so do NOT fail — keep the - // subscription and let it replay on (re)connect; a later permission - // violation is still logged by the async error handler in New. - if err := conn.FlushTimeout(subscribeConfirmTimeout); err != nil { - xlog.Debug("Could not confirm NATS subscription (will replay on connect)", "subject", subject, "error", err) - return sub, nil - } - // Flush succeeded, so any permission violation for this SUB has already been - // recorded as the connection's last error (the server emits it before the - // PONG). LastError is per-connection; match the exact quoted subject the - // server echoes ("Subscription to \"\"") so a stale violation for - // another subject can't be mis-attributed here. - if lerr := conn.LastError(); lerr != nil && - errors.Is(lerr, nats.ErrPermissionViolation) && - strings.Contains(lerr.Error(), `Subscription to "`+subject+`"`) { - _ = sub.Unsubscribe() - return nil, fmt.Errorf("subscription to %s denied by NATS server (check JWT sub allow list): %w", subject, lerr) - } - return sub, nil -} - -// ConfirmRoundTrip forces a round trip to the server and returns whatever the -// server pushed back asynchronously, so that a refusal becomes an error a -// caller holds rather than a line in a log. -// -// It replaces a Conn() accessor that handed out the raw *nats.Conn. That -// accessor was the hole in this type's method set: every half deleted above is -// still one call away on a *nats.Conn, so a family could be put back on this -// carrier through it without a single build error, which is the whole thing the -// deletions are for. -// -// What it does is the publish-side twin of confirmSubscription. NATS reports a -// permission violation asynchronously and does NOT close the connection, so a -// denied publish is indistinguishable from an accepted one until something -// round-trips and reads the connection's last error. A flush that fails is -// returned as-is: the caller could not reach the server at all, which is a -// different fact from the server refusing it, and neither is evidence about any -// node. -// -// No production path calls it, and this carrier no longer has production users -// at all. What needs the verdict is pkg/natsauth's permission grants, which are -// asserted against a real enforcing server and would otherwise be asserted -// against nothing, since an allow list that is EMPTY means unrestricted in NATS -// and a spec that only checks IsConnected cannot tell a granted publish from a -// denied one. -func (c *Client) ConfirmRoundTrip(timeout time.Duration) error { - c.mu.RLock() - conn := c.conn - c.mu.RUnlock() - if conn == nil { - return fmt.Errorf("confirming a round trip: nil NATS connection") - } - if err := conn.FlushTimeout(timeout); err != nil { - return fmt.Errorf("round trip to the NATS server: %w", err) - } - return conn.LastError() -} - -// IsConnected returns true if the client is currently connected to a NATS server. -func (c *Client) IsConnected() bool { - c.mu.RLock() - defer c.mu.RUnlock() - return c.conn != nil && c.conn.IsConnected() -} - -// Close drains and closes the NATS connection, waiting for in-flight messages. -func (c *Client) Close() { - c.mu.Lock() - defer c.mu.Unlock() - if c.conn != nil { - c.conn.Drain() - c.conn.FlushTimeout(5 * time.Second) - } -} diff --git a/core/services/messaging/interfaces.go b/core/services/messaging/interfaces.go index 980927c02..a70cf1b43 100644 --- a/core/services/messaging/interfaces.go +++ b/core/services/messaging/interfaces.go @@ -35,13 +35,14 @@ type Broadcaster interface { Subscribe(subject string, handler func([]byte)) (Subscription, error) } -// One of the two carriers a deployment dials, asserted in the package that owns -// the interface rather than at whichever call site is migrated next. +// Broadcaster has one implementation a deployment runs on, *pgbus.Bus, and it +// cannot be named here, because pgbus imports this package. The conformance +// assertion lives in interfaces_test.go alongside the test double's, which is +// also where the method-set pin lives: a conformance assertion stays true +// however many methods grow back, so it cannot say that the retired halves are +// gone. // -// *pgbus.Bus is the other and cannot be named here, because it imports this -// package; it is asserted from interfaces_test.go, which also pins that -// *Client's method set is this interface plus its own lifecycle and nothing -// else. That pin is the guard, not this line: a conformance assertion stays -// true however many methods grow back, so it cannot say that the retired halves -// are gone. -var _ Broadcaster = (*Client)(nil) +// The NATS client that used to be asserted on this line is deleted. There is no +// second carrier: the last family that needed one, agent..cancel, is a +// control verb on the agent worker's own tunnel, and no component of a +// distributed deployment opens a connection to a message broker. diff --git a/core/services/messaging/interfaces_test.go b/core/services/messaging/interfaces_test.go index 5a11fdbdb..c58fcc53f 100644 --- a/core/services/messaging/interfaces_test.go +++ b/core/services/messaging/interfaces_test.go @@ -12,21 +12,24 @@ import ( "github.com/mudler/LocalAI/core/services/testutil" ) -// The three things that must answer one contract, asserted where the contract +// The two things that must answer one contract, asserted where the contract // lives rather than at whichever adopter is migrated next. // -// *pgbus.Bus is the carrier a deployment runs on, *messaging.Client is the one -// carrier left that a deployment still dials (see below), and *testutil.FakeBus -// is what every consumer's spec runs against. A double that drifts out of the -// interface makes each adopter's suite fail in turn, in packages that do not -// own the interface and cannot say why. +// *pgbus.Bus is the carrier a deployment runs on and *testutil.FakeBus is what +// every consumer's spec runs against. A double that drifts out of the interface +// makes each adopter's suite fail in turn, in packages that do not own the +// interface and cannot say why. +// +// There were three. *messaging.Client, the NATS connection, is deleted: it +// carried one family, agent..cancel, and that family is a control verb on +// the agent worker's own tunnel now, so the type had no caller and the module +// had no reason to require a broker client. // // This file is package messaging_test because messaging cannot import testutil // or pgbus: both import messaging. var ( _ messaging.Broadcaster = (*testutil.FakeBus)(nil) _ messaging.Broadcaster = (*pgbus.Bus)(nil) - _ messaging.Broadcaster = (*messaging.Client)(nil) ) // exportedMethods names the exported method set of t, sorted, so a spec can @@ -64,43 +67,16 @@ var _ = Describe("the messaging surface", func() { }) }) -// The NATS client survives this commit, and it survives for exactly one family. +// The two Describes that stood here pinned the NATS client's method set: that +// it was a Broadcaster plus its own lifecycle, and that QueueSubscribe, +// QueueSubscribeReply, SubscribeReply, Request and Conn could not come back on +// it. They are retired rather than moved, because a method set is a property of +// a type and the type is gone. // -// agent..cancel is the one fan-out family that did not move to the -// PostgreSQL carrier: its only subscriber is the agent WORKER, which has no -// database and cannot join that carrier at all, so a cancel published there -// would reach no worker and be reported as sent. Both ends of that family still -// dial NATS, which is why messaging.New, the connect options and the TLS files -// are all still here. -// -// What the client may no longer do is everything else. Its queue and -// request/reply halves are deleted, so a family being put BACK on NATS is now a -// build error at the call site rather than a line that compiles, publishes -// successfully, and is delivered onto a carrier nobody subscribes to. -// -// The method set is pinned by NAME and not by a conformance assertion, because -// a conformance assertion cannot express absence: *messaging.Client satisfying -// Broadcaster stays true no matter how many methods are added back. -var _ = Describe("the NATS client's method set", func() { - It("is a Broadcaster plus its own lifecycle, and nothing more", func() { - Expect(exportedMethods(reflect.TypeOf((*messaging.Client)(nil)))). - To(Equal([]string{"Close", "ConfirmRoundTrip", "IsConnected", "OnReconnect", "Publish", "Subscribe"}), - "the NATS client grew a method back: it is the cancel family's carrier and may carry nothing else") - }) - - It("cannot be handed a queue group or a request", func() { - // Named individually so the failure message says WHICH half came back. - // As a set assertion alone, re-adding Request reads as an off-by-one. - t := reflect.TypeOf((*messaging.Client)(nil)) - // Conn is in this list and is the reason the list is checkable at all: - // while it existed, every name above it was one c.Conn().X() away, so - // the deletions would have been a naming convention rather than a - // constraint. ConfirmRoundTrip is what replaced it, and it hands back - // an error rather than the connection. - for _, retired := range []string{"QueueSubscribe", "QueueSubscribeReply", "SubscribeReply", "Request", "Conn"} { - _, found := t.MethodByName(retired) - Expect(found).To(BeFalse(), - "%s is back on *messaging.Client; that half of the carrier was retired and its call sites moved to the worker's tunnel or to the claim queue", retired) - } - }) -}) +// What they were really guarding is the carrier, not the type, and that guard +// survives twice over. The retired halves cannot come back on the interface, +// which the "is fan-out and nothing else" spec above still asserts by count. +// And the connection itself cannot come back at all without a broker client in +// the module, which nats_absent_test.go asserts against go.mod and go.sum: a +// deleted method is one file away from being written again, while a deleted +// require has to be re-added on purpose and shows up in a diff. diff --git a/core/services/messaging/nats_absent_test.go b/core/services/messaging/nats_absent_test.go new file mode 100644 index 000000000..9566710d0 --- /dev/null +++ b/core/services/messaging/nats_absent_test.go @@ -0,0 +1,67 @@ +package messaging_test + +import ( + "os" + "path/filepath" + "runtime" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// moduleFile reads a file from the repository root, located from this source +// file rather than from the working directory, which Ginkgo does not promise. +func moduleFile(name string) string { + _, thisFile, _, ok := runtime.Caller(0) + Expect(ok).To(BeTrue(), "could not locate this source file, so the assertion below would read nothing") + dir := filepath.Dir(thisFile) + for { + candidate := filepath.Join(dir, "go.mod") + if _, err := os.Stat(candidate); err == nil { + body, err := os.ReadFile(filepath.Join(dir, name)) + Expect(err).ToNot(HaveOccurred()) + return string(body) + } + parent := filepath.Dir(dir) + Expect(parent).ToNot(Equal(dir), "walked past the filesystem root without finding go.mod") + dir = parent + } +} + +// The point of the whole removal, asserted as a property of the module rather +// than as a property of any one package. +// +// Every other guard in this repository is a guard on a call site: a deleted +// method, a deleted parameter, a deleted field. None of them can say that the +// DEPENDENCY is gone, because a dependency survives on a single blank import in +// a single test file in a package nobody reads, and `go mod tidy` succeeding +// proves nothing about that: tidy keeps a require exactly when something still +// reaches it. +// +// This is the assertion that reddens when it comes back. It reads the module +// files themselves, so it is blind to build tags, to GOOS, and to whether a +// package can be loaded at all, which is what the source-level greps cannot +// promise and the dependency walks cannot promise either. +var _ = Describe("the module's dependency on a message broker", func() { + It("requires no nats-io module", func() { + Expect(moduleFile("go.mod")).ToNot(ContainSubstring("github.com/nats-io/"), + "a nats-io module is required again: nothing in this deployment dials a message bus, so a require is a family being invited back onto a carrier nothing reads") + }) + + It("requires no NATS testcontainer", func() { + // Named separately because it comes back by a different route: a spec + // standing a broker up for a suite, rather than production code + // dialling one. It is also the reference a dependency walk WITHOUT + // -test reports as absent while the suite is still starting a + // nats:2-alpine container. + Expect(moduleFile("go.mod")).ToNot(ContainSubstring("testcontainers-go/modules/nats"), + "a suite is standing a broker up again") + }) + + It("carries no nats-io checksum", func() { + // go.sum outlives go.mod by a release when a require is dropped by hand + // rather than by tidy, so it is asserted on its own: a checksum left + // behind is the evidence that the removal was partial. + Expect(moduleFile("go.sum")).ToNot(ContainSubstring("github.com/nats-io/")) + }) +}) diff --git a/core/services/messaging/options.go b/core/services/messaging/options.go deleted file mode 100644 index b4c67b16c..000000000 --- a/core/services/messaging/options.go +++ /dev/null @@ -1,34 +0,0 @@ -package messaging - -// Option configures NATS client connection behavior. -type Option func(*connectConfig) - -// CredentialProvider returns the NATS user JWT and signing seed to use for the -// next (re)connect. It is consulted on every connection attempt, so a refresh -// loop can rotate credentials before they expire and the connection picks them -// up automatically when the server expires the old JWT and triggers a reconnect. -type CredentialProvider func() (jwt, seed string) - -type connectConfig struct { - userJWT string - userSeed string - jwtProvider CredentialProvider - tls TLSFiles -} - -// WithUserJWT connects using a static NATS user JWT and signing seed (UserJWTAndSeed). -func WithUserJWT(jwt, seed string) Option { - return func(c *connectConfig) { - c.userJWT = jwt - c.userSeed = seed - } -} - -// WithUserJWTProvider connects using credentials fetched from provider on each -// (re)connect, enabling JWT rotation without dropping the client. Takes -// precedence over WithUserJWT when both are set. -func WithUserJWTProvider(provider CredentialProvider) Option { - return func(c *connectConfig) { - c.jwtProvider = provider - } -} diff --git a/core/services/messaging/tls.go b/core/services/messaging/tls.go deleted file mode 100644 index b594845cc..000000000 --- a/core/services/messaging/tls.go +++ /dev/null @@ -1,68 +0,0 @@ -package messaging - -import ( - "fmt" - "os" - - "github.com/nats-io/nats.go" -) - -// TLSFiles holds PEM paths for NATS TLS / mTLS. Cert and key must be set together. -// Use tls:// in LOCALAI_NATS_URL; CA and client cert paths are optional extras. -type TLSFiles struct { - CA string // LOCALAI_NATS_TLS_CA — private CA for server verification - Cert string // LOCALAI_NATS_TLS_CERT — client certificate (mTLS) - Key string // LOCALAI_NATS_TLS_KEY — client private key -} - -// Enabled reports whether any TLS file path is configured. -func (f TLSFiles) Enabled() bool { - return f.CA != "" || f.Cert != "" || f.Key != "" -} - -// Validate checks path pairing and that files exist. -func (f TLSFiles) Validate() error { - if f.Cert != "" && f.Key == "" { - return fmt.Errorf("LOCALAI_NATS_TLS_KEY is required when LOCALAI_NATS_TLS_CERT is set") - } - if f.Key != "" && f.Cert == "" { - return fmt.Errorf("LOCALAI_NATS_TLS_CERT is required when LOCALAI_NATS_TLS_KEY is set") - } - for _, path := range []struct { - name, path string - }{ - {"LOCALAI_NATS_TLS_CA", f.CA}, - {"LOCALAI_NATS_TLS_CERT", f.Cert}, - {"LOCALAI_NATS_TLS_KEY", f.Key}, - } { - if path.path == "" { - continue - } - if _, err := os.Stat(path.path); err != nil { - return fmt.Errorf("%s: %w", path.name, err) - } - } - return nil -} - -// natsOptions builds nats-go TLS options. Call Validate first. -func (f TLSFiles) natsOptions() ([]nats.Option, error) { - if !f.Enabled() { - return nil, nil - } - opts := []nats.Option{nats.Secure()} - if f.CA != "" { - opts = append(opts, nats.RootCAs(f.CA)) - } - if f.Cert != "" { - opts = append(opts, nats.ClientCert(f.Cert, f.Key)) - } - return opts, nil -} - -// WithTLS configures CA and/or client certificate paths for the NATS connection. -func WithTLS(files TLSFiles) Option { - return func(c *connectConfig) { - c.tls = files - } -} diff --git a/core/services/messaging/tls_test.go b/core/services/messaging/tls_test.go deleted file mode 100644 index 9fdb3e816..000000000 --- a/core/services/messaging/tls_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package messaging_test - -import ( - "os" - "path/filepath" - - "github.com/mudler/LocalAI/core/services/messaging" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -var _ = Describe("TLSFiles", func() { - It("requires cert and key together", func() { - Expect((messaging.TLSFiles{Cert: "/tmp/c.pem"}).Validate()).To(HaveOccurred()) - Expect((messaging.TLSFiles{Key: "/tmp/k.pem"}).Validate()).To(HaveOccurred()) - }) - - It("validates files exist", func() { - dir := GinkgoT().TempDir() - ca := filepath.Join(dir, "ca.pem") - Expect(os.WriteFile(ca, []byte("x"), 0600)).To(Succeed()) - Expect((messaging.TLSFiles{CA: ca}).Validate()).To(Succeed()) - }) -}) diff --git a/core/services/pgbus/bus.go b/core/services/pgbus/bus.go index 1e0ff6dda..fbe7ebbcc 100644 --- a/core/services/pgbus/bus.go +++ b/core/services/pgbus/bus.go @@ -592,9 +592,10 @@ type subscription struct { // message, a job result rather than a progress tick, can type-assert to this // and go read the row. // -// It is not on messaging.Subscription: the NATS client's subscription cannot -// answer it, and widening that interface would make every existing consumer -// claim a guarantee it does not have. +// It is not on messaging.Subscription: a subscription only this carrier can +// answer for does not belong on the interface every carrier implements, and +// widening that interface would make every existing consumer claim a guarantee +// it does not have. type DropCounter interface { Dropped() uint64 } diff --git a/core/services/quantization/service.go b/core/services/quantization/service.go index c51bd0021..521a1f713 100644 --- a/core/services/quantization/service.go +++ b/core/services/quantization/service.go @@ -50,9 +50,10 @@ type QuantizationService struct { // consistent across replicas; pass nil for both in standalone mode, where the // disk Loader hydrates the map and there is nothing to broadcast. // -// bus is messaging.Broadcaster and not the NATS client: this state.*.delta +// bus is messaging.Broadcaster and not a concrete carrier: this state.*.delta // family travels on whatever the deployment's fan-out carrier is, and in -// distributed mode that is PostgreSQL LISTEN/NOTIFY. +// distributed mode that is PostgreSQL LISTEN/NOTIFY. Which one it gets is +// decided in core/application, not here. func NewQuantizationService( appConfig *config.ApplicationConfig, modelLoader *model.ModelLoader, diff --git a/coverage-baseline.txt b/coverage-baseline.txt index ac1025b80..2fb78f04e 100644 --- a/coverage-baseline.txt +++ b/coverage-baseline.txt @@ -1 +1 @@ -54.2 +62.0 diff --git a/docs/content/features/distributed-mode.md b/docs/content/features/distributed-mode.md index 9721d3419..d57ed90bf 100644 --- a/docs/content/features/distributed-mode.md +++ b/docs/content/features/distributed-mode.md @@ -601,18 +601,26 @@ A frontend replica that dies mid-load does not wedge the model: the job row carr Earlier releases of distributed mode required a NATS cluster alongside PostgreSQL. **They no longer do. Shut the broker down.** Nothing in LocalAI opens a connection to one: the frontend's cross-replica fan-out is on PostgreSQL, queued work is a claim on a PostgreSQL table, a serve-backend worker takes every verb on its own tunnel, and an agent worker does too, including the cancel that was the last family on a bus. +A distributed deployment needs **PostgreSQL and the frontends' own HTTP listener, and nothing else.** Workers dial out to that listener and hold the tunnel open, so no worker needs an inbound port either. There is no broker client left in LocalAI at all: as of this release the `nats-io` modules are not in the build, so the binary cannot open a broker connection even if something asked it to. + There is no migration step and no cutover window. Stop the broker, delete its service from your compose file, chart or unit files, and delete the credentials you generated for it. A deployment that keeps running one is paying for infrastructure that carries nothing. -**Your existing command lines still start.** Every `LOCALAI_NATS_*` setting below is parsed and then ignored, so an unedited command line, unit file or Helm values file needs no change on the day you upgrade. Remove them at your convenience. +**Your existing command lines still start.** Every `LOCALAI_NATS_*` setting below is parsed and then ignored, so an unedited command line, unit file or Helm values file needs no change on the day you upgrade. They are hidden from `--help`, because there is nothing left to configure with them. **They are scheduled for removal in the release after next**; remove them from your own files at your convenience before then. | Flag | Env Var | Status | |------|---------|--------| | `--nats-url` | `LOCALAI_NATS_URL` | Accepted and ignored on the frontend, `local-ai worker` and `local-ai agent-worker`. The value is never dialled, so it may point at a broker that is already gone. | -| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | The frontend still mints a per-node user JWT at registration (`nats_jwt` in the register response). Nothing consumes it. | -| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted, unused: the frontend opens no bus connection to present them on. | -| `--nats-worker-jwt-ttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Lifetime of the minted-but-unused worker JWTs. | -| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | On an agent worker this still makes registration **wait through admin approval** rather than starting against a pending node. It gates no connection. | -| `--nats-tls-ca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted, unused. | +| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | Accepted and ignored. The frontend mints no per-node broker credential: a register or approve response carries no `nats_jwt` and no `nats_user_seed`, and a worker that reads those keys finds nothing. Nodes are authenticated by their registration token and their tunnel token. | +| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | Accepted and ignored: the frontend opens no bus connection to present them on. | +| `--nats-worker-jwtttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | Accepted and ignored. No per-node broker credential is minted, so none has a lifetime. | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | Accepted and ignored. It used to make an agent worker wait through admin approval; use `--distributed-require-auth` for that (see below). | +| `--nats-tlsca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_*` | Accepted and ignored. The paths are no longer checked for existence either, so a certificate deleted with the broker does not fail startup. | + +{{% notice warning %}} +**One behaviour changed, on the agent worker.** `--nats-require-auth` used to make `local-ai agent-worker` wait through admin approval at registration instead of starting against a pending node. That wait is now asked for with `--distributed-require-auth` / `LOCALAI_DISTRIBUTED_REQUIRE_AUTH`, which already implied it. An agent worker started with only `--nats-require-auth` no longer waits: it registers, starts, and its tunnel dials are refused with 403 until an admin approves it, which is the historical default behaviour. If you relied on the wait, set `--distributed-require-auth`. + +On the **frontend**, `--distributed-require-auth` now implies only `--registration-require-auth`. It used to also require broker credentials, and there are none to require. +{{% /notice %}} {{% notice warning %}} `LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT`, `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT` and `LOCALAI_NATS_MODEL_LOAD_TIMEOUT` are **not** in the table above and must **not** be removed. Despite their names they were never broker settings: each one is a control-RPC budget the frontend applies to a worker, and each is still read and still enforced. They are documented with the other frontend flags in [Frontend Configuration](#frontend-configuration). The names are kept because renaming them would break every existing deployment for cosmetics. diff --git a/docs/content/reference/cli-reference.md b/docs/content/reference/cli-reference.md index 15b456bd5..de83d64bc 100644 --- a/docs/content/reference/cli-reference.md +++ b/docs/content/reference/cli-reference.md @@ -211,7 +211,7 @@ LocalAI supports several subcommands beyond `run`: - `local-ai mcp-server` - Run the LocalAI admin tool surface as a stdio MCP server (controls a remote LocalAI instance over HTTP) - `local-ai worker` - Start a worker for distributed mode (generic, backend-agnostic; needs only an outbound route to the frontend, no message bus) - `local-ai p2p-worker` - Run workers to distribute workload via p2p (llama.cpp-only) -- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution, MCP CI runs and cancellation as control verbs on the tunnel it holds; it connects to no message bus, so `--nats-url` is accepted and ignored) +- `local-ai agent-worker` - Start an agent worker for distributed mode (serves agent execution, MCP CI runs and cancellation as control verbs on the tunnel it holds; it connects to no message bus) - `local-ai util` - Utility commands - `local-ai explorer` - Run P2P explorer - `local-ai federated` - Run LocalAI in federated mode @@ -219,6 +219,24 @@ LocalAI supports several subcommands beyond `run`: Use `local-ai --help` for more information on each command. +### Flags accepted and ignored + +Distributed mode no longer uses a message broker, and the `nats-io` client is no longer part of the build. The flags that configured one are still **accepted and ignored**, so an existing command line, unit file or Helm values file starts unchanged on the day you upgrade. They are hidden from `--help`, and they are **scheduled for removal in the release after next**. + +Which command accepts which is listed per row, because the sets differ. `local-ai worker` kept only `--nats-url`: its credential and TLS flags were removed a release earlier, when a serve-backend worker stopped connecting to a broker at all. Passing one of the others to `local-ai worker` is a parse error, not an ignored flag. + +| Flag | Env Var | Accepted by | +|------|---------|-------------| +| `--nats-url` | `LOCALAI_NATS_URL` | `run`, `worker`, `agent-worker` | +| `--nats-account-seed` | `LOCALAI_NATS_ACCOUNT_SEED` | `run` | +| `--nats-worker-jwtttl` | `LOCALAI_NATS_WORKER_JWT_TTL` | `run` | +| `--nats-service-jwt` / `--nats-service-seed` | `LOCALAI_NATS_SERVICE_JWT` / `LOCALAI_NATS_SERVICE_SEED` | `run`, `agent-worker` | +| `--nats-require-auth` | `LOCALAI_NATS_REQUIRE_AUTH` | `run`, `agent-worker` | +| `--nats-tlsca` / `--nats-tls-cert` / `--nats-tls-key` | `LOCALAI_NATS_TLS_CA` / `LOCALAI_NATS_TLS_CERT` / `LOCALAI_NATS_TLS_KEY` | `run`, `agent-worker` | +| `--nats-jwt` / `--nats-user-seed` | `LOCALAI_NATS_JWT` / `LOCALAI_NATS_USER_SEED` | `agent-worker` | + +`LOCALAI_NATS_BACKEND_INSTALL_TIMEOUT`, `LOCALAI_NATS_BACKEND_UPGRADE_TIMEOUT` and `LOCALAI_NATS_MODEL_LOAD_TIMEOUT` are **not** in that table and are still read and enforced. Despite their names they were never broker settings: each is a budget the frontend applies to a control request it sends a worker over that worker's tunnel. See [Migrating off the message broker]({{%relref "features/distributed-mode#migrating-off-the-message-broker" %}}). + ## Examples ### Basic Usage diff --git a/go.mod b/go.mod index b3ccf8dc9..a324369bd 100644 --- a/go.mod +++ b/go.mod @@ -44,9 +44,6 @@ require ( github.com/mudler/memory v0.0.0-20260406210934-424c1ecf2cf8 github.com/mudler/nib v0.6.0 github.com/mudler/xlog v0.0.6 - github.com/nats-io/jwt/v2 v2.7.4 - github.com/nats-io/nats.go v1.52.0 - github.com/nats-io/nkeys v0.4.15 github.com/ollama/ollama v0.20.4 github.com/onsi/ginkgo/v2 v2.29.0 github.com/onsi/gomega v1.41.0 @@ -66,7 +63,6 @@ require ( github.com/swaggo/echo-swagger v1.5.2 github.com/swaggo/swag v1.16.6 github.com/testcontainers/testcontainers-go v0.42.0 - github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 github.com/timbutler/zxcvbn v1.0.4 github.com/valkey-io/valkey-go v1.0.76 @@ -86,11 +82,8 @@ require ( ) require ( - cyphar.com/go-pathrs v0.2.1 // indirect filippo.io/bigmod v0.1.1-0.20260103110540-f8a47775ebe5 // indirect filippo.io/keygen v0.0.0-20260114151900-8e2790ea4c5b // indirect - github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 // indirect - github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/atotto/clipboard v0.1.4 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.9 // indirect @@ -116,14 +109,9 @@ require ( github.com/chromedp/cdproto v0.0.0-20260321001828-e3e3800016bc // indirect github.com/chromedp/chromedp v0.15.1 // indirect github.com/chromedp/sysutil v1.1.0 // indirect - github.com/containerd/containerd/api v1.8.0 // indirect - github.com/containerd/fifo v1.1.0 // indirect - github.com/containerd/ttrpc v1.2.7 // indirect - github.com/containerd/typeurl/v2 v2.2.0 // indirect github.com/cyberphone/json-canonicalization v0.0.0-20241213102144-19d51d7fe467 // indirect github.com/digitorus/pkcs7 v0.0.0-20230818184609-3a137a874352 // indirect github.com/digitorus/timestamp v0.0.0-20231217203849-220c5c2851b7 // indirect - github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/dunglas/httpsfv v1.1.0 // indirect github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect github.com/filecoin-project/go-clock v0.1.0 // indirect @@ -161,17 +149,11 @@ require ( github.com/klippa-app/go-pdfium v1.19.2 // indirect github.com/mattn/go-localereader v0.0.1 // indirect github.com/mattn/go-sqlite3 v1.14.28 // indirect - github.com/moby/locker v1.0.1 // indirect github.com/moby/moby/api v1.54.2 // indirect github.com/moby/moby/client v0.4.1 // indirect - github.com/moby/sys/mountinfo v0.7.2 // indirect - github.com/moby/sys/signal v0.7.0 // indirect github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect github.com/muesli/cancelreader v0.2.2 // indirect - github.com/nats-io/nuid v1.0.1 // indirect github.com/oklog/ulid v1.3.1 // indirect - github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/opencontainers/selinux v1.13.1 // indirect github.com/secure-systems-lab/go-securesystemslib v0.9.1 // indirect github.com/shibumi/go-pathspec v1.3.0 // indirect github.com/sigstore/protobuf-specs v0.5.1 // indirect @@ -190,7 +172,6 @@ require ( github.com/transparency-dev/merkle v0.0.2 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect go.mongodb.org/mongo-driver v1.17.6 // indirect - google.golang.org/genproto v0.0.0-20250922171735-9219d122eba9 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect sigs.k8s.io/yaml v1.6.0 // indirect ) @@ -528,7 +509,7 @@ require ( golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f // indirect golang.org/x/mod v0.36.0 // indirect golang.org/x/sync v0.20.0 - golang.org/x/sys v0.45.0 // indirect + golang.org/x/sys v0.45.0 golang.org/x/term v0.43.0 golang.org/x/text v0.37.0 golang.org/x/tools v0.45.0 // indirect diff --git a/go.sum b/go.sum index a510df3df..779238bdb 100644 --- a/go.sum +++ b/go.sum @@ -50,8 +50,6 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cyphar.com/go-pathrs v0.2.1 h1:9nx1vOgwVvX1mNBWDu93+vaceedpbsDqo+XuBGL40b8= -cyphar.com/go-pathrs v0.2.1/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -67,8 +65,6 @@ fyne.io/systray v1.12.0 h1:CA1Kk0e2zwFlxtc02L3QFSiIbxJ/P0n582YrZHT7aTM= fyne.io/systray v1.12.0/go.mod h1:RVwqP9nYMo7h5zViCBHri2FgjXF7H2cub7MAq4NSoLs= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0 h1:59MxjQVfjXsBpLy+dbd2/ELV5ofnUkUZBvWSC85sheA= -github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d h1:zjqpY4C7H15HjRPEenkS4SAn3Jy2eRRjkjZbGR30TOg= github.com/AdamKorcz/go-fuzz-headers-1 v0.0.0-20230919221257-8b5d3ce2d11d/go.mod h1:XNqJ7hv2kY++g8XEHREpi+JqZo3+0l+CH2egBVN4yqM= github.com/Azure/azure-sdk-for-go/sdk/azcore v1.20.0 h1:JXg2dwJUmPB9JmtVmdEB16APJ7jurfbY5jnfXpJoRMc= @@ -302,31 +298,18 @@ github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb h1:EDmT6Q9Zs+SbUo github.com/codahale/rfc6979 v0.0.0-20141003034818-6a90f24967eb/go.mod h1:ZjrT6AXHbDs86ZSdt/osfBi5qfexBrKUdONk989Wnk4= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= -github.com/containerd/containerd v1.7.31 h1:jn3IMuTV4Bb1Uwb0MFPW2ASJAD3W1lh6QqqZHIZwDh4= -github.com/containerd/containerd v1.7.31/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= -github.com/containerd/containerd v1.7.32 h1:S54xuVcPxeLaYgaRABtpJ2VyVUVsy0IGf7qHBs+sbY8= -github.com/containerd/containerd v1.7.32/go.mod h1:jdwD6s/BhV4XVJGrvtziNPVA+83n66TwptVaPKprq4E= github.com/containerd/containerd v1.7.33 h1:iAkYGC/ifR/V+0eR4iXWHNGYUF0DF2PmGV5iz4Irj5M= github.com/containerd/containerd v1.7.33/go.mod h1:gSbSCVjPCdkfJCjyrzz7aRC+xFlqVbatNpfHfVCYGUM= -github.com/containerd/containerd/api v1.8.0 h1:hVTNJKR8fMc/2Tiw60ZRijntNMd1U+JVMyTRdsD2bS0= -github.com/containerd/containerd/api v1.8.0/go.mod h1:dFv4lt6S20wTu/hMcP4350RL87qPWLVa/OHOwmmdnYc= github.com/containerd/continuity v0.4.4 h1:/fNVfTJ7wIl/YPMHjf+5H32uFhl63JucB34PlCpMKII= github.com/containerd/continuity v0.4.4/go.mod h1:/lNJvtJKUQStBzpVQ1+rasXO1LAWtUQssk28EZvJ3nE= github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= -github.com/containerd/fifo v1.1.0 h1:4I2mbh5stb1u6ycIABlBw9zgtlK8viPI9QkQNRQEEmY= -github.com/containerd/fifo v1.1.0/go.mod h1:bmC4NWMbXlt2EZ0Hc7Fx7QzTFxgPID13eH0Qu+MAb2o= github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= -github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRqQ= -github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o= -github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY= -github.com/containerd/typeurl/v2 v2.2.0 h1:6NBDbQzr7I5LHgp34xAXYF5DOTQDn05X58lsPEmzLso= -github.com/containerd/typeurl/v2 v2.2.0/go.mod h1:8XOOxnyatxSWuG8OfsZXVnAF4iZfedjS/8UHSPJnX4g= github.com/coreos/go-oidc/v3 v3.18.0 h1:V9orjXynvu5wiC9SemFTWnG4F45v403aIcjWo0d41+A= github.com/coreos/go-oidc/v3 v3.18.0/go.mod h1:DYCf24+ncYi+XkIH97GY1+dqoRlbaSI26KVTCI9SrY4= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -379,8 +362,6 @@ github.com/docker/docker-credential-helpers v0.9.3 h1:gAm/VtF9wgqJMoxzT3Gj5p4AqI github.com/docker/docker-credential-helpers v0.9.3/go.mod h1:x+4Gbw9aGmChi3qTLZj8Dfn0TD20M/fuWy0E5+WDeCo= github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c h1:+pKlWGMw7gf6bQ+oDZB4KHQFypsfjYlq/C4rfL7D3g8= -github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 h1:iFaUwBSo5Svw6L7HYpRu/0lE3e0BaElwnNO1qkNQxBY= @@ -982,8 +963,6 @@ github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3N github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= -github.com/moby/locker v1.0.1 h1:fOXqR41zeveg4fFODix+1Ch4mj/gT0NE1XJbp/epuBg= -github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/moby/api v1.54.2 h1:wiat9QAhnDQjA7wk1kh/TqHz2I1uUA7M7t9SAl/JNXg= github.com/moby/moby/api v1.54.2/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= github.com/moby/moby/client v0.4.1 h1:DMQgisVoMkmMs7fp3ROSdiBnoAu8+vo3GggFl06M/wY= @@ -994,8 +973,6 @@ github.com/moby/sys/mountinfo v0.7.2 h1:1shs6aH5s4o5H2zQLn796ADW1wMrIwHsyJ2v9Kou github.com/moby/sys/mountinfo v0.7.2/go.mod h1:1YOa8w8Ih7uW0wALDUgT1dTTSBrZ+HiBLGws92L2RU4= github.com/moby/sys/sequential v0.6.0 h1:qrx7XFUd/5DxtqcoH1h438hF5TmOvzC/lspjy7zgvCU= github.com/moby/sys/sequential v0.6.0/go.mod h1:uyv8EUTrca5PnDsdMGXhZe6CCe8U/UiTWd+lL+7b/Ko= -github.com/moby/sys/signal v0.7.0 h1:25RW3d5TnQEoKvRbEKUGay6DCQ46IxAVTT9CUMgmsSI= -github.com/moby/sys/signal v0.7.0/go.mod h1:GQ6ObYZfqacOwTtlXvcmh9A26dVRul/hbOZn88Kg8Tg= github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= @@ -1025,8 +1002,6 @@ github.com/mudler/edgevpn v0.34.0 h1:qDrD/rCPFY/FdURbXudIZWihVKY4VOX3nMn3CcbeQEU github.com/mudler/edgevpn v0.34.0/go.mod h1:yki7uMi5LR9gSMrw8PdPieuxsrk8BLV2Ui7VBEmbbIA= github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc h1:RxwneJl1VgvikiX28EkpdAyL4yQVnJMrbquKospjHyA= github.com/mudler/go-piper v0.0.0-20241023091659-2494246fd9fc/go.mod h1:O7SwdSWMilAWhBZMK9N9Y/oBDyMMzshE3ju8Xkexwig= -github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc h1:NEFmd7+JoImN5dZI81/vcBRjtMg+GfEa7nEjQ029hd0= -github.com/mudler/go-processmanager v0.1.2-0.20260720195933-3d64f5c974fc/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6 h1:/nFm1Ttf8g1BnWtEth986JR34pCh9rzae5A2vKBZosc= github.com/mudler/go-processmanager v0.1.2-0.20260823202314-dfa0ed852db6/go.mod h1:h6kmHUZeafr+k5hRYpGLMzJFH4hItHffgpRo2QIkP+o= github.com/mudler/localrecall v0.6.3 h1:uXOrP9JmetzxgVKzSrawviyBHZfAcvPBBIrvVUdZjDA= @@ -1075,14 +1050,6 @@ github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/natefinch/atomic v1.0.1 h1:ZPYKxkqQOx3KZ+RsbnP/YsgvxWQPGxjC0oBt2AhwV0A= github.com/natefinch/atomic v1.0.1/go.mod h1:N/D/ELrljoqDyT3rZrsUmtsuzvHkeB/wWjHV22AZRbM= -github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= -github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= -github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= -github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= -github.com/nats-io/nkeys v0.4.15 h1:JACV5jRVO9V856KOapQ7x+EY8Jo3qw1vJt/9Jpwzkk4= -github.com/nats-io/nkeys v0.4.15/go.mod h1:CpMchTXC9fxA5zrMo4KpySxNjiDVvr8ANOSZdiNfUrs= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/neelance/astrewrite v0.0.0-20160511093645-99348263ae86/go.mod h1:kHJEU3ofeGjhHklVoIGuVj85JJwZ6kWPaJwCIxgnFmo= github.com/neelance/sourcemap v0.0.0-20200213170602-2833bce08e4c/go.mod h1:Qr6/a/Q4r9LP1IltGz7tA7iOK1WonHEYhu1HRBA7ZiM= github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ= @@ -1111,10 +1078,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= -github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk= -github.com/opencontainers/runtime-spec v1.2.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.13.1 h1:A8nNeceYngH9Ow++M+VVEwJVpdFmrlxsN22F+ISDCJE= -github.com/opencontainers/selinux v1.13.1/go.mod h1:S10WXZ/osk2kWOYKy1x2f/eXF5ZHJoUs8UU/2caNRbg= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde h1:x0TT0RDC7UhAVbbWWBzr41ElhJx5tXPWkIHA2HWPRuw= @@ -1263,13 +1226,10 @@ github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/i github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= github.com/shirou/gopsutil/v4 v4.26.3 h1:2ESdQt90yU3oXF/CdOlRCJxrP+Am1aBYubTMTfxJ1qc= github.com/shirou/gopsutil/v4 v4.26.3/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= -github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= -github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= github.com/shoenig/go-m1cpu v0.2.2 h1:4nc55oVv7nygGnfI9bhLCLzUEs4794y0Bkqx4q2zy7Y= github.com/shoenig/go-m1cpu v0.2.2/go.mod h1:KkDOw6m3ZJQAPHbrzkZki4hnx+pDRR1Lo+ldA56wD5w= -github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= -github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/shurcooL/go v0.0.0-20200502201357-93f07166e636/go.mod h1:TDJrrUr11Vxrven61rcy3hJMUqaf/CLWYhHNPmT14Lk= @@ -1369,8 +1329,6 @@ github.com/temoto/robotstxt v1.1.2 h1:W2pOjSJ6SWvldyEuiFXNxz3xZ8aiWX5LbfDiOFd7Fx github.com/temoto/robotstxt v1.1.2/go.mod h1:+1AmkuG3IYkh1kv0d2qEB9Le88ehNO0zwOr3ujewlOo= github.com/testcontainers/testcontainers-go v0.42.0 h1:He3IhTzTZOygSXLJPMX7n44XtK+qhjat1nI9cneBbUY= github.com/testcontainers/testcontainers-go v0.42.0/go.mod h1:vZjdY1YmUA1qEForxOIOazfsrdyORJAbhi0bp8plN30= -github.com/testcontainers/testcontainers-go/modules/nats v0.42.0 h1:WQR0+1r4GkM5QgOBoLxlP41empovt5PxtaiDpC0G7ow= -github.com/testcontainers/testcontainers-go/modules/nats v0.42.0/go.mod h1:ZAI9iisjDNJmcRcycQFKSLpiBN9u2g1v9AJRq1afriE= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0 h1:GCbb1ndrF7OTDiIvxXyItaDab4qkzTFJ48LKFdM7EIo= github.com/testcontainers/testcontainers-go/modules/postgres v0.42.0/go.mod h1:IRPBaI8jXdrNfD0e4Zm7Fbcgaz5shKxOQv4axiL09xs= github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= @@ -1649,8 +1607,6 @@ golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= -golang.org/x/net v0.54.0 h1:2zJIZAxAHV/OHCDTCOHAYehQzLfSXuf/5SoL/Dv6w/w= -golang.org/x/net v0.54.0/go.mod h1:Sj4oj8jK6XmHpBZU/zWHw3BV3abl4Kvi+Ut7cQcY+cQ= golang.org/x/net v0.55.0 h1:bcvxaJn3e1U6InsFWt1JUq1aSjnRxLzT2rtD2KfkDF8= golang.org/x/net v0.55.0/go.mod h1:L5U2KuzuOe1lY7Z+aWVIKK6qEeJXnXV9yzGA+WCHJww= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1739,7 +1695,6 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/pkg/natsauth/config.go b/pkg/natsauth/config.go deleted file mode 100644 index e9de53470..000000000 --- a/pkg/natsauth/config.go +++ /dev/null @@ -1,66 +0,0 @@ -package natsauth - -import ( - "fmt" - "time" - - "github.com/mudler/xlog" -) - -// DefaultWorkerJWTTTL is how long a worker may use a minted NATS user JWT before re-registering. -const DefaultWorkerJWTTTL = 24 * time.Hour - -// Config holds NATS JWT authentication settings for distributed mode. -type Config struct { - // AccountSeed is the NATS account signing seed (SU...). Used to mint per-node worker JWTs. - AccountSeed string - // ServiceUserJWT is a pre-generated user JWT for frontends and agent workers (broad publish). - ServiceUserJWT string - // ServiceUserSeed is the signing seed (SU...) paired with ServiceUserJWT. - ServiceUserSeed string - // WorkerJWTTTL sets expiry on minted worker JWTs. Zero uses DefaultWorkerJWTTTL. - WorkerJWTTTL time.Duration - // RequireAuth rejects anonymous NATS when true (both ServiceUserJWT and AccountSeed expected). - RequireAuth bool -} - -// Enabled reports whether any NATS credential material is configured. -func (c Config) Enabled() bool { - return c.AccountSeed != "" || c.ServiceUserJWT != "" -} - -// CanMintWorkers reports whether per-node JWTs can be issued at registration. -func (c Config) CanMintWorkers() bool { - return c.AccountSeed != "" -} - -// WorkerTTL returns the configured worker JWT lifetime. -func (c Config) WorkerTTL() time.Duration { - if c.WorkerJWTTTL > 0 { - return c.WorkerJWTTTL - } - return DefaultWorkerJWTTTL -} - -// Validate checks consistency when distributed NATS auth is required. -func (c Config) Validate() error { - if !c.RequireAuth { - return nil - } - if c.ServiceUserJWT == "" || c.ServiceUserSeed == "" { - return fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH requires LOCALAI_NATS_SERVICE_JWT and LOCALAI_NATS_SERVICE_SEED") - } - if c.AccountSeed == "" { - return fmt.Errorf("LOCALAI_NATS_REQUIRE_AUTH is set but LOCALAI_NATS_ACCOUNT_SEED is empty") - } - return nil -} - -// WarnIfInsecure logs when distributed NATS is reachable without credentials. -func (c Config) WarnIfInsecure(distributed bool) { - if !distributed || c.Enabled() { - return - } - xlog.Warn("NATS is used without JWT credentials — any client on the bus can publish backend.install. " + - "Set LOCALAI_NATS_ACCOUNT_SEED + LOCALAI_NATS_SERVICE_JWT (see docs/features/distributed-mode.md).") -} diff --git a/pkg/natsauth/decode.go b/pkg/natsauth/decode.go deleted file mode 100644 index 1ae156f44..000000000 --- a/pkg/natsauth/decode.go +++ /dev/null @@ -1,16 +0,0 @@ -package natsauth - -import ( - "fmt" - - "github.com/nats-io/jwt/v2" -) - -// DecodeUserClaims decodes a minted worker JWT for tests and diagnostics. -func DecodeUserClaims(token string) (*jwt.UserClaims, error) { - uc, err := jwt.DecodeUserClaims(token) - if err != nil { - return nil, fmt.Errorf("natsauth: decode user JWT: %w", err) - } - return uc, nil -} diff --git a/pkg/natsauth/mint.go b/pkg/natsauth/mint.go deleted file mode 100644 index 387e8d701..000000000 --- a/pkg/natsauth/mint.go +++ /dev/null @@ -1,59 +0,0 @@ -package natsauth - -import ( - "fmt" - "time" - - "github.com/nats-io/jwt/v2" - "github.com/nats-io/nkeys" -) - -// MintWorkerJWT creates a signed NATS user JWT and user seed scoped to nodeID and nodeType. -// The seed is returned once at registration so the worker can sign NATS connections. -func (c Config) MintWorkerJWT(nodeID, nodeType string) (userJWT, userSeed string, err error) { - if c.AccountSeed == "" { - return "", "", fmt.Errorf("natsauth: account seed not configured") - } - if nodeID == "" { - return "", "", fmt.Errorf("natsauth: node ID is required") - } - - accountKP, err := nkeys.FromSeed([]byte(c.AccountSeed)) - if err != nil { - return "", "", fmt.Errorf("natsauth: invalid account seed: %w", err) - } - - userKP, err := nkeys.CreateUser() - if err != nil { - return "", "", fmt.Errorf("natsauth: create user key: %w", err) - } - seedBytes, err := userKP.Seed() - if err != nil { - return "", "", fmt.Errorf("natsauth: user seed: %w", err) - } - - accountPub, err := accountKP.PublicKey() - if err != nil { - return "", "", fmt.Errorf("natsauth: account public key: %w", err) - } - userPub, err := userKP.PublicKey() - if err != nil { - return "", "", fmt.Errorf("natsauth: user public key: %w", err) - } - - pubAllow, subAllow := WorkerPermissions(nodeID, nodeType) - - uc := jwt.NewUserClaims(userPub) - uc.Name = fmt.Sprintf("localai-%s-%s", nodeType, workerSubjectToken(nodeID)) - uc.IssuerAccount = accountPub - uc.Expires = time.Now().Add(c.WorkerTTL()).Unix() - - uc.Permissions.Pub.Allow = pubAllow - uc.Permissions.Sub.Allow = subAllow - - token, err := uc.Encode(accountKP) - if err != nil { - return "", "", fmt.Errorf("natsauth: encode user JWT: %w", err) - } - return token, string(seedBytes), nil -} diff --git a/pkg/natsauth/mint_test.go b/pkg/natsauth/mint_test.go deleted file mode 100644 index f96861357..000000000 --- a/pkg/natsauth/mint_test.go +++ /dev/null @@ -1,72 +0,0 @@ -package natsauth_test - -import ( - "testing" - "time" - - "github.com/mudler/LocalAI/pkg/natsauth" - "github.com/nats-io/jwt/v2" - "github.com/nats-io/nkeys" - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -func TestNatsAuth(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "NatsAuth") -} - -var _ = Describe("MintWorkerJWT", func() { - var accountSeed string - - BeforeEach(func() { - akp, err := nkeys.CreateAccount() - Expect(err).NotTo(HaveOccurred()) - seed, err := akp.Seed() - Expect(err).NotTo(HaveOccurred()) - accountSeed = string(seed) - }) - - It("mints a JWT with backend worker permissions", func() { - cfg := natsauth.Config{AccountSeed: accountSeed, WorkerJWTTTL: time.Hour} - token, seed, err := cfg.MintWorkerJWT("550e8400-e29b-41d4-a716-446655440000", "backend") - Expect(err).NotTo(HaveOccurred()) - Expect(token).NotTo(BeEmpty()) - Expect(seed).NotTo(BeEmpty()) - - uc, err := jwt.DecodeUserClaims(token) - Expect(err).NotTo(HaveOccurred()) - // A backend worker opens no bus connection at all, so its node subtree - // went with it. The JWT is still minted at registration and simply - // unused; asserting BOTH lists are exactly the inbox is what keeps it - // from silently becoming an unrestricted credential, since NATS reads - // an empty allow list as no restriction. - Expect(uc.Permissions.Sub.Allow).To(ConsistOf("_INBOX.>")) - // The install-progress subject is gone with the carrier: progress is a - // line in the install response now, so a minted worker JWT must not - // still be granted a publish right for it. File staging went the same - // way, so a minted worker JWT publishes nowhere but its own inbox. - Expect(uc.Permissions.Pub.Allow).To(ConsistOf("_INBOX.>")) - for _, subj := range uc.Permissions.Pub.Allow { - Expect(subj).NotTo(ContainSubstring("backend.install")) - } - }) - - It("mints agent permissions without backend install subscribe", func() { - cfg := natsauth.Config{AccountSeed: accountSeed} - token, _, err := cfg.MintWorkerJWT("node-1", "agent") - Expect(err).NotTo(HaveOccurred()) - - uc, err := jwt.DecodeUserClaims(token) - Expect(err).NotTo(HaveOccurred()) - Expect(uc.Permissions.Sub.Allow).To(ContainElement("agent.*.cancel")) - for _, subj := range uc.Permissions.Sub.Allow { - Expect(subj).NotTo(ContainSubstring("backend.install")) - } - }) - - It("rejects mint without account seed", func() { - _, _, err := (natsauth.Config{}).MintWorkerJWT("id", "backend") - Expect(err).To(HaveOccurred()) - }) -}) diff --git a/pkg/natsauth/permissions.go b/pkg/natsauth/permissions.go deleted file mode 100644 index c64bdcd88..000000000 --- a/pkg/natsauth/permissions.go +++ /dev/null @@ -1,67 +0,0 @@ -package natsauth - -import "strings" - -// workerSubjectToken mirrors messaging.sanitizeSubjectToken without importing unexported logic. -func workerSubjectToken(nodeID string) string { - r := strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-") - return r.Replace(nodeID) -} - -// WorkerPermissions returns NATS pub/sub allow lists for a registered node. -// -// It serves AGENT nodes. They are the only workers left that connect to the -// bus: an agent worker subscribes to the queue subjects listed below, while a -// backend worker connects to no bus at all, because every verb a frontend gives -// it is an HTTP route on its own server reached through its outbound tunnel -// (core/services/workerctl). -// -// The non-agent branch is therefore a grant of nothing, and it has to be -// spelled that way rather than deleted. NATS reads an EMPTY allow list as no -// restriction, so a function that returned nil here would upgrade every JWT the -// frontend still mints for a backend node from "its own inbox" to "the entire -// account". The inbox is self-scoped and reaches no cluster subject. -// -// nodeID no longer narrows anything: no allow list below is per-node, because -// the only per-node subject an agent worker ever subscribed to was its -// backend.stop, which is a control RPC on its tunnel now. It stays in the -// signature so a future per-node grant has somewhere to come from, and because -// mint.go names the JWT user after it. -func WorkerPermissions(nodeID, nodeType string) (pubAllow, subAllow []string) { - switch nodeType { - case "agent": - // Keep this list in sync with the subscriptions in core/cli/agent_worker.go. - // - // MCP tool execution, discovery, agent execution, MCP CI runs and the - // per-node backend.stop are all absent: every one of them is a control - // RPC on the tunnel the worker holds, addressed by the frontend rather - // than by a subject. The last two left when the queue groups became - // claim rows on the job store. - // - // Removing them narrowed this list; it must never be narrowed to - // nothing, because NATS reads an EMPTY allow list as no restriction at - // all, which would widen an agent JWT to the whole account. - subAllow = []string{ - "agent.*.cancel", - "gallery.*.cancel", - "gallery.*.progress", - "jobs.*.cancel", - "jobs.*.progress", - "jobs.*.result", - "staging.*.progress", - "_INBOX.>", - } - pubAllow = []string{ - "agent.>", - "jobs.>", - "_INBOX.>", - } - default: - // Backend worker: nothing, held open at its own inbox for the reason in - // the doc comment. The node subtree it used to subscribe on went with - // the connection itself, which this worker no longer opens. - subAllow = []string{"_INBOX.>"} - pubAllow = []string{"_INBOX.>"} - } - return pubAllow, subAllow -} diff --git a/pkg/natsauth/permissions_coverage_test.go b/pkg/natsauth/permissions_coverage_test.go deleted file mode 100644 index e5b9db21a..000000000 --- a/pkg/natsauth/permissions_coverage_test.go +++ /dev/null @@ -1,164 +0,0 @@ -package natsauth_test - -import ( - "strings" - - "github.com/mudler/LocalAI/core/services/messaging" - "github.com/mudler/LocalAI/pkg/natsauth" - - . "github.com/onsi/ginkgo/v2" - . "github.com/onsi/gomega" -) - -// subjectMatches implements NATS subject-token matching: "*" matches exactly one -// token and ">" matches one or more trailing tokens. It lets these tests assert -// that a permission allow-list (which uses wildcards) actually covers a concrete -// subject a component publishes/subscribes — the same check the NATS server makes. -func subjectMatches(pattern, subject string) bool { - p := strings.Split(pattern, ".") - s := strings.Split(subject, ".") - for i, tok := range p { - if tok == ">" { - return i < len(s) // ">" must match at least one remaining token - } - if i >= len(s) { - return false - } - if tok != "*" && tok != s[i] { - return false - } - } - return len(p) == len(s) -} - -// workerSubjectTokenForTest mirrors the sanitizer both packages implement, so -// the negative assertion below names the exact prefix without reaching into -// either package's unexported copy. -func workerSubjectTokenForTest(nodeID string) string { - return strings.NewReplacer(".", "-", "*", "-", ">", "-", " ", "-", "\t", "-", "\n", "-").Replace(nodeID) -} - -func anyAllows(allow []string, subject string) bool { - for _, p := range allow { - if subjectMatches(p, subject) { - return true - } - } - return false -} - -var _ = Describe("WorkerPermissions subject coverage", func() { - // A node ID containing NATS-reserved characters exercises the (duplicated) - // sanitizer in pkg/natsauth against the canonical one in core/services/messaging. - // If the two ever diverge, the minted prefix stops matching the real subject - // and these assertions fail — guarding the copy noted in the review. - const nodeID = "host.a 1*b" - - Context("backend worker", func() { - pub, sub := natsauth.WorkerPermissions(nodeID, "backend") - - // A backend worker opens no connection at all on this build. Every verb - // a frontend gives it, the backend and model lifecycle ten plus the - // four file-staging verbs, is an HTTP route on its tunnelled control - // plane, so there is no subject left to cover. See - // core/services/workerctl. - It("no longer grants a backend worker its own node subtree to subscribe on", func() { - Expect(sub).ToNot(ContainElement("nodes."+workerSubjectTokenForTest(nodeID)+".>"), - "the node subtree went with the connection the worker no longer opens") - }) - - // The grant must be a grant of NOTHING and not an ABSENT grant: NATS - // treats an empty allow list as no restriction, so a branch that - // returned nil would silently widen every backend JWT the frontend - // still mints to the whole account. ConsistOf, not BeEmpty, is what - // tells those two apart. - It("grants a backend worker its own inbox and nothing else", func() { - Expect(sub).To(ConsistOf("_INBOX.>")) - Expect(sub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - // The negative half, and it is the one that would catch a verb quietly - // coming back to the bus: a backend worker is granted nothing to - // publish at all beyond its own inbox. File staging used to be the one - // exception and is not any more. - It("grants a backend worker no publish rights outside its inbox", func() { - Expect(pub).To(ConsistOf("_INBOX.>")) - Expect(pub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - It("no longer grants a backend worker the file-staging publish subtree", func() { - Expect(anyAllows(pub, "nodes."+workerSubjectTokenForTest(nodeID)+".files.stage")).To(BeFalse(), - "backend JWT pub allow-list %v still covers file staging", pub) - }) - }) - - Context("agent worker", func() { - // node_type "agent"; subjects from core/cli/agent_worker.go. - pub, sub := natsauth.WorkerPermissions(nodeID, "agent") - - // The subjects an agent worker still subscribes to. Both queue-group - // workloads have left this list: agent execution and MCP CI runs are - // streaming control verbs on the tunnel now, driven by a claim a - // frontend replica took off the job store. - subscribed := []string{ - messaging.SubjectAgentCancelWildcard, - } - - // The half that catches a narrowing going too far. NATS reads an EMPTY - // allow list as NO restriction, so a branch trimmed to nothing does not - // lock an agent worker down, it opens the whole account to it. - It("keeps the agent worker's allow lists non-empty", func() { - Expect(sub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - Expect(pub).ToNot(BeEmpty(), - "an empty allow list is unrestricted in NATS, not restrictive") - }) - - // MCP execution and discovery, backend.stop, and now agent execution - // and MCP CI runs, are all control RPCs on the worker's tunnel, - // addressed by the frontend rather than by a subject. An agent worker - // subscribes to none of them, so a JWT that still granted one would be - // granting a subscription nothing serves. Every entry is spelled out by - // hand rather than built from a deleted constant, which is the only way - // this can still name the subject that used to be granted. - for _, subject := range []string{ - "mcp.tools.execute", - "mcp.discovery", - "nodes." + workerSubjectTokenForTest(nodeID) + ".backend.stop", - "agent.execute", - "jobs.mcp-ci.new", - } { - It("no longer grants an agent worker "+subject, func() { - Expect(anyAllows(sub, subject)).To(BeFalse(), - "agent JWT sub allow-list %v still covers %s", sub, subject) - }) - } - - // The other half of the narrowing, and the one the phase's own trap - // makes necessary: an allow list trimmed to nothing is UNRESTRICTED in - // NATS. Naming what must survive is what tells a narrowing from an - // escalation, so the queue workloads an agent worker still consumes are - // asserted present just above, and the list is asserted non-empty here. - It("still grants the agent worker the broadcast subjects it lives on", func() { - Expect(sub).To(ContainElement("agent.*.cancel")) - Expect(sub).To(ContainElement("jobs.*.progress")) - Expect(sub).To(ContainElement("_INBOX.>")) - }) - for _, subject := range subscribed { - It("allows subscribing to "+subject, func() { - Expect(anyAllows(sub, subject)).To(BeTrue(), - "agent JWT sub allow-list %v does not cover %s — the agent worker subscribes to it", sub, subject) - }) - } - }) -}) - -// The "Documented NATS service-user permissions" suite that stood here read -// scripts/nats-auth-setup.sh and required its --allow-pub list to cover every -// subject the frontend publishes. Both the script and the frontend's bus -// connection are gone: cross-replica fan-out is a PostgreSQL NOTIFY, which has -// no allow-list to fall out of sync with. The property was retired with the -// artifact it guarded, not moved, and the surviving suites above still pin the -// MINTED credentials, which are Task 17's to remove. diff --git a/scripts/build/healthcheck_test.sh b/scripts/build/healthcheck_test.sh index 9dc811b8f..bcba9a460 100644 --- a/scripts/build/healthcheck_test.sh +++ b/scripts/build/healthcheck_test.sh @@ -135,8 +135,9 @@ echo "== a 503 from a still-preloading frontend is unhealthy, not a crash" run_hc 1 "local-ai run" STUB_CURL_EXIT=22 echo "== modes with no HTTP surface report healthy rather than false-unhealthy" -# agent-worker is NATS-only. Reporting `unhealthy` forever for a process that -# was never going to bind a port is the same bug as #10987, one mode over. +# agent-worker binds no port: it dials a tunnel out to a frontend and is +# reached over that. Reporting `unhealthy` forever for a process that was never +# going to bind a port is the same bug as #10987, one mode over. run_hc 0 "local-ai agent-worker" expect_url "" diff --git a/tests/e2e/distributed/cluster/cluster.go b/tests/e2e/distributed/cluster/cluster.go index fe48d5672..65672ac12 100644 --- a/tests/e2e/distributed/cluster/cluster.go +++ b/tests/e2e/distributed/cluster/cluster.go @@ -4,7 +4,8 @@ // process to kill, no second replica to race, and no real HTTP boundary between // a worker and the frontend it registered with. This package starts the same // binary an operator runs, one process per frontend replica and one per worker, -// against containerised Postgres and NATS. +// against containerised Postgres. Postgres is the only container: a distributed +// deployment needs it and the frontends' own HTTP listener, and nothing else. package cluster import ( @@ -23,6 +24,21 @@ import ( "github.com/phayes/freeport" ) +// StaleBusURL is the address of a broker that is not running, and is not meant +// to be. +// +// LOCALAI_NATS_URL is still accepted and ignored by every process this harness +// starts, so an operator's existing command line, unit file or Helm values file +// starts unchanged after the broker is shut down. This suite covers that +// promise by handing over THIS value: a value pointing at a live server would +// let a regression that dialled it pass unnoticed, while a dead one turns the +// same regression into a startup failure in every cluster spec. +// +// It is a constant here rather than an Options field, because there is nothing +// left for a caller to choose: no process reads the variable, so a per-cluster +// value would only be a knob that changes nothing. +const StaleBusURL = "nats://127.0.0.1:1" + // Options configures a cluster. Every field without a default is required. type Options struct { // Binary is the path to a built local-ai. @@ -31,9 +47,12 @@ type Options struct { // into each worker's backends directory as "mock-backend", which is the name // model YAML refers to (see tests/e2e/e2e_suite_test.go:75). MockBackend string - // PGDSN and NatsURL point at infrastructure the caller already started. - PGDSN string - NatsURL string + // PGDSN points at infrastructure the caller already started. + // + // There is no NatsURL beside it any more. Removing the FIELD rather than + // ignoring it means a spec that still sets one fails to compile, which is + // the only way a harness option stops being set by accident. + PGDSN string // LogDir receives one file per process. Never empty: a cluster failure is // unreadable without them. LogDir string @@ -275,7 +294,13 @@ func (c *Cluster) startFrontend(i int, port int) (*Process, error) { // the children need PATH, HOME and the Go/CI environment intact. cmd.Env = append(cmd.Environ(), "LOCALAI_DISTRIBUTED=true", - "LOCALAI_NATS_URL="+c.opts.NatsURL, + // Deliberately handed a broker URL, and deliberately a dead one. This + // is the shape of an operator's existing unit file on the day they shut + // the broker down, and the promise it covers is that such a command + // line still STARTS. A live address would let a regression that dialled + // it pass; this one turns such a regression into a startup failure in + // every cluster spec. + "LOCALAI_NATS_URL="+StaleBusURL, "LOCALAI_AUTH=true", "LOCALAI_AUTH_DATABASE_URL="+c.opts.PGDSN, "LOCALAI_ADMIN_EMAIL="+c.opts.AdminEmail, @@ -405,10 +430,11 @@ func (c *Cluster) startAgentWorker(i int) (*Process, error) { name := agentWorkerName(i) cmd := exec.Command(c.opts.Binary, "agent-worker") cmd.Env = append(cmd.Environ(), - // Still set, and still ignored. It is left here so this suite keeps - // covering the promise that an operator's existing --nats-url does not - // break an agent worker; nothing in the process reads it any more. - "LOCALAI_NATS_URL="+c.opts.NatsURL, + // Still set, and still ignored, on the same terms as the frontend's + // above: this suite keeps covering the promise that an operator's + // existing --nats-url does not break an agent worker. Nothing in the + // process reads it. + "LOCALAI_NATS_URL="+StaleBusURL, "LOCALAI_REGISTER_TO="+c.workerFrontendURL(i), "LOCALAI_NODE_NAME="+name, "LOCALAI_REGISTRATION_TOKEN="+c.opts.RegistrationToken, @@ -434,9 +460,28 @@ func (c *Cluster) WorkerEnviron(i int) ([]string, error) { if err := c.checkWorkerIndex(i); err != nil { return nil, err } - p := c.workers[i] + return processEnviron(c.workers[i]) +} + +// FrontendEnviron is the environment of frontend i's RUNNING PROCESS, read the +// same way. +// +// It exists so a spec proving a WORKER runs with no broker URL can show that +// the deployment it joined was handed one: without that half, a harness which +// simply stopped setting the variable anywhere would satisfy the worker +// assertion just as well. Read from /proc rather than from the harness options +// for the same reason the worker's is: it is a fact about the process. +func (c *Cluster) FrontendEnviron(i int) ([]string, error) { + if err := c.checkFrontendIndex(i); err != nil { + return nil, err + } + return processEnviron(c.frontends[i]) +} + +// processEnviron reads a running child's environment out of /proc. +func processEnviron(p *Process) ([]string, error) { if p == nil || p.Cmd == nil || p.Cmd.Process == nil { - return nil, fmt.Errorf("worker %d is not running", i) + return nil, fmt.Errorf("process is not running") } raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/environ", p.Cmd.Process.Pid)) if err != nil { @@ -476,8 +521,8 @@ const ( // starts at 32768 by default. That is not tidiness. Ports the kernel hands // out for outbound connections are exactly the ports a long-lived process // full of outbound connections is liable to be holding when a backend tries - // to bind one, and this suite's workers hold a tunnel, a NATS connection - // and a registration client each. + // to bind one, and this suite's workers hold a tunnel and a registration + // client each. workerPortFloor = 20000 workerPortCeiling = 31000 @@ -622,15 +667,6 @@ func (c *Cluster) FrontendBackendsDir(i int) (string, error) { return filepath.Join(c.frontendDir(i), "backends"), nil } -// NatsURL is the bus this cluster's frontends were given. -// -// It is exported for one assertion: a spec proving a WORKER runs with no bus -// has to show the deployment it joined has one, or "no NATS anywhere" would -// satisfy it just as well. -func (c *Cluster) NatsURL() string { - return c.opts.NatsURL -} - // WorkerName is the node name worker i registered under. func (c *Cluster) WorkerName(i int) string { return c.workers[i].Name diff --git a/tests/e2e/distributed/cluster/cluster_test.go b/tests/e2e/distributed/cluster/cluster_test.go index 2f63d0a28..5089c09e4 100644 --- a/tests/e2e/distributed/cluster/cluster_test.go +++ b/tests/e2e/distributed/cluster/cluster_test.go @@ -17,7 +17,6 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { _, err := cluster.Start(cluster.Options{ Binary: filepath.Join(os.TempDir(), "definitely-not-local-ai"), PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", - NatsURL: "nats://127.0.0.1:4222", LogDir: GinkgoT().TempDir(), Frontends: 1, }) @@ -29,7 +28,6 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { _, err := cluster.Start(cluster.Options{ Binary: "/bin/true", PGDSN: "postgres://test:test@127.0.0.1:5432/x?sslmode=disable", - NatsURL: "nats://127.0.0.1:4222", LogDir: GinkgoT().TempDir(), Frontends: 0, }) @@ -39,9 +37,9 @@ var _ = Describe("Cluster options", Label("Distributed"), func() { }) // The HTTP flow inside AdminSession and GetJSON cannot run here: it needs a -// built local-ai plus real Postgres and NATS, which arrive with the failover -// suites. These specs cover the argument validation that would otherwise panic -// on an out-of-range slice index inside a helper every later spec calls. +// built local-ai plus real Postgres, which arrives with the failover suites. +// These specs cover the argument validation that would otherwise panic on an +// out-of-range slice index inside a helper every later spec calls. var _ = Describe("Admin session", Label("Distributed"), func() { It("reports a clear error when the frontend index is out of range", func() { c := cluster.ForTestingEmpty() @@ -59,8 +57,8 @@ var _ = Describe("Admin session", Label("Distributed"), func() { }) // Like the admin specs above, these cover argument validation only. Killing, -// stopping and restarting a real replica needs a built local-ai plus Postgres -// and NATS, so those paths stay unexecuted until the failover suites land. +// stopping and restarting a real replica needs a built local-ai plus Postgres, +// so those paths stay unexecuted until the failover suites land. var _ = Describe("Failure primitives", Label("Distributed"), func() { It("rejects an out-of-range frontend index rather than panicking", func() { c := cluster.ForTestingEmpty() diff --git a/tests/e2e/distributed/cluster/failure.go b/tests/e2e/distributed/cluster/failure.go index fe808401c..49e6ecd35 100644 --- a/tests/e2e/distributed/cluster/failure.go +++ b/tests/e2e/distributed/cluster/failure.go @@ -51,7 +51,7 @@ func (c *Cluster) KillWorker(i int) error { // TIME_WAIT on an accepted connection does not block the bind. // // The data directory is wiped so the replica must rehydrate node, session and -// job state from the shared Postgres and NATS. Keeping it would model a pod +// job state from the shared Postgres. Keeping it would model a pod // with a persistent volume and would hide the very class of bug these tests // exist to find. This is only safe because startFrontend pins // LOCALAI_AUTH_HMAC_SECRET: the secret otherwise lives at diff --git a/tests/e2e/distributed/cluster_baseline_test.go b/tests/e2e/distributed/cluster_baseline_test.go index e6cb851c6..e85797e8e 100644 --- a/tests/e2e/distributed/cluster_baseline_test.go +++ b/tests/e2e/distributed/cluster_baseline_test.go @@ -288,13 +288,12 @@ func startClusterOnFreshDB(frontends, workers int, customise ...func(*cluster.Op Binary: binary, MockBackend: mockBackend, PGDSN: infra.PGURL, - // Deliberately a dead address. Frontends and agent workers are still - // handed LOCALAI_NATS_URL so this suite keeps covering the promise that - // an operator's existing command line starts unchanged after the broker - // is shut down. Pointing it at a running server would make a regression - // that dialled it invisible; pointing it at nothing makes such a - // regression a startup failure in every cluster spec. - NatsURL: staleBusURL, + // There is no bus URL to pass. Frontends and agent workers are still + // handed a dead LOCALAI_NATS_URL, so this suite keeps covering the + // promise that an operator's existing command line starts unchanged + // after the broker is shut down, but the value is the harness's own + // cluster.StaleBusURL rather than a caller's choice: no process reads + // it, so there was nothing left for a caller to choose. LogDir: logDir, Frontends: frontends, Workers: workers, diff --git a/tests/e2e/distributed/cluster_control_test.go b/tests/e2e/distributed/cluster_control_test.go index c6f213f83..fb363e9df 100644 --- a/tests/e2e/distributed/cluster_control_test.go +++ b/tests/e2e/distributed/cluster_control_test.go @@ -416,9 +416,14 @@ var _ = Describe("Control plane over the worker tunnel", Label("Distributed"), L // And the harness DOES still hand a LOCALAI_NATS_URL to the other // processes in this cluster, so the worker's lack of one is a property // of the worker and not of a harness that stopped setting the variable - // at all. There is no broker behind that URL any more, which is the - // point: nothing dials it, so nothing notices. - Expect(c.NatsURL()).ToNot(BeEmpty()) + // at all. Read from the FRONTEND's running process for the same reason + // the worker's absence is: the harness options would only say what was + // assembled. There is no broker behind that URL, which is the point: + // nothing dials it, so nothing notices. + frontendEnviron, err := c.FrontendEnviron(0) + Expect(err).ToNot(HaveOccurred()) + Expect(frontendEnviron).To(ContainElement(HavePrefix("LOCALAI_NATS_URL=")), + "the harness stopped handing a bus URL to anything, so the worker's lack of one proves nothing") probe := newRosterProbe(c, client, 0) Eventually(probe.healthyNames, nodeRosterTimeout, nodeRosterPoll). diff --git a/tests/e2e/distributed/finetune_distributed_test.go b/tests/e2e/distributed/finetune_distributed_test.go index b1a1409e3..c3c56de2c 100644 --- a/tests/e2e/distributed/finetune_distributed_test.go +++ b/tests/e2e/distributed/finetune_distributed_test.go @@ -103,8 +103,12 @@ var _ = Describe("Fine-Tune Distributed", Label("Distributed"), func() { Expect(appCfg.Distributed.Enabled).To(BeFalse()) // Without distributed mode, fine-tune jobs use local in-memory - // state tracking. No PostgreSQL or NATS needed. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // state tracking. No PostgreSQL needed. + // + // The "and no bus URL" half of this assertion is gone with the + // field it read: DistributedConfig has nowhere to hold one, which + // core/config's "broker surface" spec pins for every config rather + // than for this one. }) }) }) diff --git a/tests/e2e/distributed/foundation_test.go b/tests/e2e/distributed/foundation_test.go index cca2e271f..5623b0f3d 100644 --- a/tests/e2e/distributed/foundation_test.go +++ b/tests/e2e/distributed/foundation_test.go @@ -37,30 +37,28 @@ var _ = Describe("Phase 0: Foundation", Label("Distributed"), func() { Expect(appCfg.Auth.Enabled).To(BeFalse()) }) - It("leaves the inert bus URL empty when nothing sets it", func() { + // Two Its stood here: "leaves the inert bus URL empty when nothing sets + // it" and "should accept valid distributed configuration", which passed + // config.WithNatsURL and read the value back. Both are retired with the + // field and the option they used, and neither property is lost. + // + // "The value is never dialled" is no longer a promise about a stored + // value; there is nowhere to store one, which core/config's "broker + // surface" spec asserts by reflection so it cannot silently stop + // compiling when the field returns. "An existing command line still + // starts" moved DOWN a level, to where it is actually at risk: kong is + // what rejects an unknown flag, so core/cli's "frontend's broker flags" + // specs parse the real command line, and the cluster suite starts real + // frontends with a dead LOCALAI_NATS_URL in their environment. + It("should accept a valid distributed configuration", func() { appCfg := config.NewApplicationConfig( config.EnableDistributed, config.WithAuthEnabled(true), config.WithAuthDatabaseURL(infra.PGURL), ) - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) - }) - - It("should accept valid distributed configuration", func() { - // staleBusURL points at nothing on purpose. It is the shape of an - // operator's existing command line after the broker was shut down, - // and the promise this spec holds is that such a command line still - // STARTS: the value is parsed, stored and never dialled. Pointing - // it at a live server would let a regression that dialled it pass. - appCfg := config.NewApplicationConfig( - config.EnableDistributed, - config.WithAuthEnabled(true), - config.WithAuthDatabaseURL(infra.PGURL), - config.WithNatsURL(staleBusURL), - ) Expect(appCfg.Distributed.Enabled).To(BeTrue()) Expect(appCfg.Auth.Enabled).To(BeTrue()) - Expect(appCfg.Distributed.NatsURL).To(Equal(staleBusURL)) + Expect(appCfg.Distributed.Validate()).To(Succeed()) }) It("should generate unique frontend ID on startup", func() { diff --git a/tests/e2e/distributed/gallery_distributed_test.go b/tests/e2e/distributed/gallery_distributed_test.go index b5b4294e4..05babe279 100644 --- a/tests/e2e/distributed/gallery_distributed_test.go +++ b/tests/e2e/distributed/gallery_distributed_test.go @@ -171,7 +171,9 @@ var _ = Describe("Gallery Distributed", Label("Distributed"), func() { // Without distributed mode, gallery operations use the existing // in-memory galleryApplier map. No PostgreSQL needed. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/job_dispatch_test.go b/tests/e2e/distributed/job_dispatch_test.go index c1c74d8ee..070a186d4 100644 --- a/tests/e2e/distributed/job_dispatch_test.go +++ b/tests/e2e/distributed/job_dispatch_test.go @@ -239,7 +239,9 @@ var _ = Describe("Job Dispatch", Label("Distributed"), func() { // Without distributed mode, jobs use local in-process dispatch. // The JobStore can still be used standalone with SQLite or in-memory. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/model_routing_test.go b/tests/e2e/distributed/model_routing_test.go index cb64e4622..b70a2c137 100644 --- a/tests/e2e/distributed/model_routing_test.go +++ b/tests/e2e/distributed/model_routing_test.go @@ -117,7 +117,9 @@ var _ = Describe("Model Routing", Label("Distributed"), func() { // Without distributed mode, no SmartRouter is created. // The ModelLoader uses its local process management. // This test documents the design decision. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/skills_distributed_test.go b/tests/e2e/distributed/skills_distributed_test.go index 14929e640..0bc822c72 100644 --- a/tests/e2e/distributed/skills_distributed_test.go +++ b/tests/e2e/distributed/skills_distributed_test.go @@ -109,8 +109,10 @@ var _ = Describe("Skills Distributed", Label("Distributed"), func() { Expect(appCfg.Distributed.Enabled).To(BeFalse()) // Without distributed mode, skills are stored on the local - // filesystem. No PostgreSQL metadata or NATS cache invalidation. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // filesystem. No PostgreSQL metadata and no cache invalidation. + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/sse_routes_test.go b/tests/e2e/distributed/sse_routes_test.go index 2e7b64a9a..0accf5453 100644 --- a/tests/e2e/distributed/sse_routes_test.go +++ b/tests/e2e/distributed/sse_routes_test.go @@ -127,7 +127,9 @@ var _ = Describe("SSE Routes", Label("Distributed"), func() { // Without distributed mode, carrier-backed SSE routes are not registered. // Agent SSE events use the in-process LocalAGI SSE manager instead. // Job progress is tracked in-memory. - Expect(appCfg.Distributed.NatsURL).To(BeEmpty()) + // + // The bus-URL half of this assertion went with the field it read; + // core/config's "broker surface" spec pins its absence. }) }) }) diff --git a/tests/e2e/distributed/testhelpers_test.go b/tests/e2e/distributed/testhelpers_test.go index 0508f0675..d52d3cb31 100644 --- a/tests/e2e/distributed/testhelpers_test.go +++ b/tests/e2e/distributed/testhelpers_test.go @@ -37,14 +37,6 @@ type TestInfra struct { PGURL string } -// staleBusURL is the address of a broker that is not running, and is not meant -// to be. LOCALAI_NATS_URL and --nats-url are still accepted and ignored so an -// operator's existing command line, unit file or Helm values file starts -// unchanged after the broker is shut down; specs that exercise that promise -// hand over THIS value, because a value pointing at a live server would let a -// regression that dialled it pass unnoticed. -const staleBusURL = "nats://127.0.0.1:1" - // The container is suite-scoped, not spec-scoped. Starting a Postgres (~10s) per // spec cost roughly 36 minutes of pure startup across the 213 specs behind // SetupInfra, which is why this suite was never wired into CI. Isolation now diff --git a/website/layouts/index.html b/website/layouts/index.html index 22137a23c..4c014bb57 100644 --- a/website/layouts/index.html +++ b/website/layouts/index.html @@ -111,7 +111,7 @@

03 / DISTRIBUTED

Add a second machine.

Routing, VRAM-aware placement, prefix-cache affinity and failover are the runtime's problem. You add hardware, the cluster works out what to do with it.

-

Smart routing · autoscaling · P2P · NATS · federation

+

Smart routing · autoscaling · P2P · tunnels · federation