mirror of
https://github.com/mudler/LocalAI.git
synced 2026-07-30 09:57:57 -04:00
* fix(distributed): make the remote LoadModel deadline configurable
The router hardcoded a 5 minute gRPC deadline for the remote LoadModel
call. Staging finishes before the timer starts, so those five minutes
cover only the worker backend's own checkpoint load and pipeline init.
A cold load of meituan-longcat/LongCat-Video-Avatar-1.5 (~83 GB) on an
ARM64 Thor worker fails at exactly 302s with DeadlineExceeded while the
backend process is still making progress (CPU time accumulating, RSS
moving as weights are mapped), so the load was cut short rather than
wedged.
Add LOCALAI_NATS_MODEL_LOAD_TIMEOUT / --model-load-timeout mirroring the
existing backend-install timeout knob, defaulting to 5m so unset
clusters keep today's behaviour.
The cold-load hold ceiling (which bounds how long one load may hold the
per-model advisory lock) was derived from the install timeout alone, so
raising the load deadline past it would have been silently clipped.
Derive it from both budgets via ModelLoadCeilingFor:
max(install + load + 5m staging margin, 25m)
With the defaults that is 15m + 5m + 5m = 25m, identical to the previous
constant, and the 25m floor means shrinking either budget can never
tighten the ceiling below what clusters relied on before.
Assisted-by: Claude:claude-opus-4-8 golangci-lint
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
* fix(distributed): reap the abandoned replica when a remote load times out
The gRPC deadline on the remote LoadModel call only cancels the client
side. A backend blocked in a synchronous weight load never observes its
cancelled handler context, so when scheduleAndLoad gave up it left the
worker loading with nobody waiting for the result.
Observed on an ARM64 Thor worker loading LongCat-Video-Avatar-1.5: the
client returned DeadlineExceeded at 302s, and the backend process was
still alive 30 minutes later having pulled ~57GB from HuggingFace. Every
retry stacked another multi-GB loader on the worker; they had to be
reaped by hand via POST /api/nodes/:id/models/unload.
Send backend.stop for the exact `modelID#replicaIndex` process key we
just abandoned. The exact key matters: a bare model ID stops every
replica on that node, including healthy ones serving traffic.
Only a deadline or cancellation triggers the reap. Any other LoadModel
failure is the backend answering, which means its handler returned and
the process is idle - stopping it there would discard a warm process and
its downloaded weights. The reap is best-effort and never replaces the
load error the caller is waiting on.
The `modelID#replicaIndex` format was already hand-rolled in two places
(the worker's buildProcessKey and pkg/model's log store). Rather than add
a third, export model.BackendProcessKey from pkg/model, the lowest common
dependency of both sides.
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Assisted-by: Claude:claude-opus-4-8 golangci-lint
---------
Signed-off-by: Ettore Di Giacinto <mudler@localai.io>
Co-authored-by: Ettore Di Giacinto <mudler@localai.io>
179 lines
6.4 KiB
Go
179 lines
6.4 KiB
Go
package config_test
|
|
|
|
import (
|
|
"time"
|
|
|
|
. "github.com/onsi/ginkgo/v2"
|
|
. "github.com/onsi/gomega"
|
|
|
|
"github.com/mudler/LocalAI/core/config"
|
|
)
|
|
|
|
var _ = Describe("DistributedConfig backend NATS timeouts", func() {
|
|
Context("BackendInstallTimeoutOrDefault", func() {
|
|
It("returns 15 minutes when unset", func() {
|
|
c := config.DistributedConfig{}
|
|
Expect(c.BackendInstallTimeoutOrDefault()).To(Equal(15 * time.Minute))
|
|
})
|
|
|
|
It("returns the configured value when set", func() {
|
|
c := config.DistributedConfig{BackendInstallTimeout: 42 * time.Minute}
|
|
Expect(c.BackendInstallTimeoutOrDefault()).To(Equal(42 * time.Minute))
|
|
})
|
|
})
|
|
|
|
Context("BackendUpgradeTimeoutOrDefault", func() {
|
|
It("returns 15 minutes when unset", func() {
|
|
c := config.DistributedConfig{}
|
|
Expect(c.BackendUpgradeTimeoutOrDefault()).To(Equal(15 * time.Minute))
|
|
})
|
|
|
|
It("returns the configured value when set", func() {
|
|
c := config.DistributedConfig{BackendUpgradeTimeout: 30 * time.Minute}
|
|
Expect(c.BackendUpgradeTimeoutOrDefault()).To(Equal(30 * time.Minute))
|
|
})
|
|
})
|
|
|
|
Context("ModelLoadTimeoutOrDefault", func() {
|
|
It("returns 5 minutes when unset so existing clusters keep today's behaviour", func() {
|
|
c := config.DistributedConfig{}
|
|
Expect(c.ModelLoadTimeoutOrDefault()).To(Equal(5 * time.Minute))
|
|
})
|
|
|
|
It("returns the configured value when set", func() {
|
|
c := config.DistributedConfig{ModelLoadTimeout: 45 * time.Minute}
|
|
Expect(c.ModelLoadTimeoutOrDefault()).To(Equal(45 * time.Minute))
|
|
})
|
|
})
|
|
})
|
|
|
|
var _ = Describe("DistributedConfig flag-name constants", func() {
|
|
// Pin the kebab-case strings so a rename of the Go field name (or a
|
|
// CLI flag naming convention change) forces the constant to update,
|
|
// keeping the Validate error messages and any future operator-facing
|
|
// surface in sync with the actual CLI flag.
|
|
DescribeTable("flag name constants",
|
|
func(actual, expected string) {
|
|
Expect(actual).To(Equal(expected))
|
|
},
|
|
Entry("MCP tool timeout", config.FlagMCPToolTimeout, "mcp-tool-timeout"),
|
|
Entry("MCP discovery timeout", config.FlagMCPDiscoveryTimeout, "mcp-discovery-timeout"),
|
|
Entry("worker wait timeout", config.FlagWorkerWaitTimeout, "worker-wait-timeout"),
|
|
Entry("drain timeout", config.FlagDrainTimeout, "drain-timeout"),
|
|
Entry("health check interval", config.FlagHealthCheckInterval, "health-check-interval"),
|
|
Entry("stale node threshold", config.FlagStaleNodeThreshold, "stale-node-threshold"),
|
|
Entry("MCP CI job timeout", config.FlagMCPCIJobTimeout, "mcp-ci-job-timeout"),
|
|
Entry("backend install timeout", config.FlagBackendInstallTimeout, "backend-install-timeout"),
|
|
Entry("backend upgrade timeout", config.FlagBackendUpgradeTimeout, "backend-upgrade-timeout"),
|
|
Entry("model load timeout", config.FlagModelLoadTimeout, "model-load-timeout"),
|
|
)
|
|
})
|
|
|
|
var _ = Describe("DistributedConfig.Validate negative-duration errors", func() {
|
|
It("rejects a negative BackendInstallTimeout with the flag name in the error", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
BackendInstallTimeout: -1 * time.Second,
|
|
}
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring(config.FlagBackendInstallTimeout))
|
|
Expect(err.Error()).To(ContainSubstring("must not be negative"))
|
|
})
|
|
|
|
It("rejects a negative BackendUpgradeTimeout with the flag name in the error", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
BackendUpgradeTimeout: -1 * time.Second,
|
|
}
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring(config.FlagBackendUpgradeTimeout))
|
|
})
|
|
|
|
It("rejects a negative ModelLoadTimeout with the flag name in the error", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
ModelLoadTimeout: -1 * time.Second,
|
|
}
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring(config.FlagModelLoadTimeout))
|
|
Expect(err.Error()).To(ContainSubstring("must not be negative"))
|
|
})
|
|
|
|
It("accepts all-zero durations as valid (defaults apply)", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
}
|
|
Expect(c.Validate()).To(Succeed())
|
|
})
|
|
})
|
|
|
|
var _ = Describe("DistributedConfig.Validate registration auth", func() {
|
|
It("rejects an empty registration token when RequireAuth is set", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
RegistrationRequireAuth: true,
|
|
}
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("LOCALAI_REGISTRATION_REQUIRE_AUTH"))
|
|
Expect(err.Error()).To(ContainSubstring("LOCALAI_REGISTRATION_TOKEN"))
|
|
})
|
|
|
|
It("accepts a set registration token when RequireAuth is set", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
RegistrationToken: "s3cret",
|
|
RegistrationRequireAuth: true,
|
|
}
|
|
Expect(c.Validate()).To(Succeed())
|
|
})
|
|
|
|
It("warns but succeeds with an empty token when RequireAuth is unset", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
}
|
|
Expect(c.Validate()).To(Succeed())
|
|
})
|
|
|
|
It("rejects an empty token when the umbrella RequireAuth is set", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
RequireAuth: true,
|
|
// Provide NATS creds so only the registration-token gap remains.
|
|
NatsServiceJWT: "jwt",
|
|
NatsServiceSeed: "seed",
|
|
NatsAccountSeed: "acct",
|
|
}
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("LOCALAI_DISTRIBUTED_REQUIRE_AUTH"))
|
|
Expect(err.Error()).To(ContainSubstring("LOCALAI_REGISTRATION_TOKEN"))
|
|
})
|
|
|
|
It("the umbrella implies NATS auth is required", func() {
|
|
c := config.DistributedConfig{
|
|
Enabled: true,
|
|
NatsURL: "nats://localhost:4222",
|
|
RegistrationToken: "tok", // registration layer satisfied
|
|
RequireAuth: true, // umbrella → NATS creds now required
|
|
}
|
|
Expect(c.NatsAuthRequired()).To(BeTrue())
|
|
Expect(c.RegistrationAuthRequired()).To(BeTrue())
|
|
// Missing NATS service JWT/seed must now be fatal.
|
|
err := c.Validate()
|
|
Expect(err).To(HaveOccurred())
|
|
Expect(err.Error()).To(ContainSubstring("LOCALAI_NATS_REQUIRE_AUTH"))
|
|
})
|
|
})
|