// SPDX-License-Identifier: MIT package application import ( "context" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/core/services/messaging" "github.com/mudler/LocalAI/core/services/nodes" "github.com/mudler/LocalAI/core/services/pgbus" "github.com/mudler/LocalAI/core/services/syncstate" "github.com/mudler/LocalAI/core/services/testutil" ) // The guard on the one setting that decides whether any broadcast in the // deployment is ever delivered. // // The carrier holds a pinned LISTEN connection opened from a DSN, and publishes // travel on a pooled handle opened from another. When those two name different // databases every publish succeeds, every subscribe succeeds, and nothing // arrives, on every replica, with no error anywhere. There is exactly one // legitimate DSN, and these specs are what say so in a way that fails when it // stops being true. var _ = Describe("opening the deployment's broadcast carrier", func() { It("listens on the same database URL the auth pool was built from", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn bus, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) // Equality with the field, not "is a PostgreSQL URL": the failure being // excluded is two databases, and any DSN passes a shape check. Expect(bus.DSN()).To(Equal(cfg.Auth.DatabaseURL)) }) It("migrates the spill table, so an oversized broadcast has somewhere to go", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn bus, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) Expect(db.Migrator().HasTable(&pgbus.BusMessage{})).To(BeTrue()) }) It("refuses to open a carrier whose DSN is not the pool's database", func() { db, _ := testutil.SetupTestDBWithDSN() _, otherDSN := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = otherDSN _, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).To(HaveOccurred()) }) }) // The partial pin on two wiring lines that cannot be reddened by a spec: the // newBroadcastBus call, and `Bus: bus` in the returned literal. Neither is a // compile error when deleted and initDistributed cannot be unit tested while it // opens NATS first, so what is available is a boot refusal, and this is what // keeps that refusal honest. var _ = Describe("refusing a deployment with no broadcast carrier", func() { It("accepts services that carry one", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn bus, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) Expect(requireBroadcastCarrier(&DistributedServices{Bus: bus})).To(Succeed()) }) It("refuses services whose carrier was never assigned, and says what it costs", func() { err := requireBroadcastCarrier(&DistributedServices{}) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("published between replicas")) Expect(err.Error()).To(ContainSubstring("shutdown")) }) It("refuses a nil deployment rather than dereferencing it", func() { Expect(requireBroadcastCarrier(nil)).ToNot(Succeed()) }) }) var _ = Describe("shutting the distributed services down", func() { It("closes the broadcast carrier", func() { // A pinned PostgreSQL session and the goroutine parked on it, per // replica restart. Nothing else in this process ever closes it, so the // line in the shutdown closure is the whole lifecycle. db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn bus, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).ToNot(HaveOccurred()) Expect(bus.IsConnected()).To(BeTrue()) (&DistributedServices{Bus: bus}).Shutdown() Expect(bus.IsConnected()).To(BeFalse()) }) }) // The one place the four state.*.delta families are told which carrier they // travel on. // // It was five field reads before this: the fine-tune service, the quantization // service, the agent-task setter on two startup paths, the per-user services // manager and the Open Responses store. Every one of them takes a // messaging.Broadcaster, which the broker client on the same struct satisfied // too, so a site left holding that field compiled, started, published and was // delivered onto a carrier only agent workers read, and nothing failed until // the broker went away. That field and its type are now gone; collapsing the // choice into one function is what keeps it a fact these specs can hold, rather // than a property that lasted only as long as there was one carrier. var _ = Describe("handing the broadcast carrier to its adopters", func() { It("returns the carrier the deployment opened", func() { db, dsn := testutil.SetupTestDBWithDSN() cfg := &config.ApplicationConfig{} cfg.Auth.DatabaseURL = dsn bus, err := newBroadcastBus(context.Background(), cfg, db) Expect(err).ToNot(HaveOccurred()) DeferCleanup(bus.Close) // Identity and not "is a Broadcaster". There is no second carrier on // this struct any more: the family that needed one, agent..cancel, // rides the agent worker's own tunnel now. The identity assertion stays // because what it pins is that adopters get THIS bus rather than // anything else that satisfies the interface. ds := &DistributedServices{Bus: bus} Expect(ds.Broadcast()).To(BeIdenticalTo(messaging.Broadcaster(bus))) }) It("returns an interface that reads as absent, not a typed nil, when there is no carrier", func() { // Every adopter branches on `bus == nil` to mean standalone. A nil // *pgbus.Bus placed in an interface is NOT nil, so that branch would be // skipped and the first Set would panic on a request rather than at // boot. // // Compared with == and not with BeNil(). Gomega's BeNil reports a nil // POINTER inside an interface as nil, so it passes on exactly the value // this spec exists to reject; the first draft of this spec did, and the // mutation that removed the guard stayed green. var ds *DistributedServices Expect(ds.Broadcast() == nil).To(BeTrue(), "a nil deployment must yield an interface that is itself nil") Expect((&DistributedServices{}).Broadcast() == nil).To(BeTrue(), "a deployment with no carrier must yield an interface that is itself nil, not one wrapping a nil *pgbus.Bus") }) It("gives an adopter a carrier-less map rather than one that panics on the first write", func() { // The consequence, driven through the component every adopter builds. // A typed nil satisfies `!= nil`, so Start subscribes on it and Set // publishes on it, and both dereference a nil *pgbus.Bus on a request // path rather than at boot. m := syncstate.New(syncstate.Config[string, string]{ Name: "test.jobs", Key: func(v string) string { return v }, Bus: (&DistributedServices{}).Broadcast(), }) Expect(m.Start(context.Background())).To(Succeed()) DeferCleanup(func() { Expect(m.Close()).To(Succeed()) }) Expect(func() { Expect(m.Set(context.Background(), "v")).To(Succeed()) }).ToNot(Panic()) }) }) // The registration of every per-node cache a node's departure evicts. // // Four subscribers, registered in one function, on a notifier the health // monitor is then handed. Every one of those is a line that compiles, starts // and serves when it is missing: a deployment whose departed nodes keep their // probe entries, staging rows, prefix affinity and per-node operation progress // does not fail, log or slow down, it just answers with state for a node that // left, for the life of the process. // // Asserted by NAME and not by count. A count says a cache was forgotten; only // the names say which, and "which" is the entire content of the failure. var _ = Describe("wiring the per-node caches a departure evicts", func() { // bootDistributed brings a real distributed deployment up against a fresh // database, which is what makes these assertions about production wiring // rather than about a notifier a spec assembled itself. bootDistributed := func(arm ...func(*config.ApplicationConfig)) *DistributedServices { GinkgoHelper() db, dsn := testutil.SetupTestDBWithDSN() ctx, cancel := context.WithCancel(context.Background()) DeferCleanup(cancel) cfg := &config.ApplicationConfig{DataPath: GinkgoT().TempDir(), Context: ctx} cfg.Auth.Enabled = true cfg.Auth.DatabaseURL = dsn cfg.Distributed.Enabled = true for _, a := range arm { a(cfg) } ds, err := initDistributed(cfg, db, nil, galleryop.NewGalleryService(cfg, nil)) Expect(err).ToNot(HaveOccurred()) DeferCleanup(ds.Shutdown) return ds } It("registers every one of them, on the notifier the health monitor fires", func() { // Reached through the health monitor and not through a local variable, // because registering the four on a DIFFERENT notifier than the one the // monitor was built with evicts nothing while every count still reads // four. ds := bootDistributed() Expect(ds.Health.Departures().SubscriberNames()).To(ConsistOf( departurePrefixCache, departureProbeCache, departureStagingTracker, departureGalleryNodes, departureControlClients, departureStagerClients, )) }) It("registers no prefix-cache eviction when prefix-cache routing is disabled", func() { // --distributed-prefix-cache=false stays a TRUE no-op: there is no // index to drop from, so nothing is registered rather than a hook // registered onto nothing. The other three are unaffected, which is the // half that makes this a statement about S1 and not about the feature // flag switching the whole mechanism off. ds := bootDistributed(func(cfg *config.ApplicationConfig) { cfg.Distributed.PrefixCacheDisabled = true }) Expect(ds.Health.Departures().SubscriberNames()).To(ConsistOf( departureProbeCache, departureStagingTracker, departureGalleryNodes, departureControlClients, departureStagerClients, )) }) It("refuses a deployment with no router, naming what its departed nodes would keep", func() { err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nil, galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("probe-freshness")) }) It("refuses a deployment with no gallery service", func() { err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), nil, specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("per-node breakdown")) }) It("refuses a deployment with no control client, naming the streams a departed node would keep", func() { err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), nil, specFileStager()) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("idle streams")) }) It("refuses a deployment with no file stager", func() { err := registerDepartureEvictions(nodes.NewDepartureNotifier(), nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), nil) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("cached HTTP client")) }) It("refuses a deployment with no notifier at all", func() { err := registerDepartureEvictions(nil, nil, nodes.NewSmartRouter(nil, nodes.SmartRouterOptions{}), galleryop.NewGalleryService(&config.ApplicationConfig{}, nil), specControlClient(), specFileStager()) Expect(err).To(HaveOccurred()) }) }) // specControlClient and specFileStager build the two per-node client caches a // registration needs. Neither is dialled: what the refusal specs assert is that // the wiring refuses a MISSING one, and what the registration specs assert is // that a subscriber was registered for it. func specControlClient() *nodes.ControlClient { return nodes.NewControlClient(nil, "") } func specFileStager() nodes.FileStager { return nodes.NewHTTPFileStager(func(string) (string, error) { return "", nil }, "", nil) }