+
+ setConfirm(null)}
+ />
+
+ )
+}
+
+// ModelFailoverStatus is the Model Editor mount point: it renders the strip
+// only when the edited model is a failover chain, and owns the SSE
+// subscription so create mode never opens one.
+export function ModelFailoverStatus({ name, addToast }) {
+ const { t } = useTranslation('models')
+ const { isAdmin } = useAuth()
+ const { byName, refresh } = useFailoverChains()
+ const chain = byName[name]
+ if (!chain) return null
+
+ const act = async (call, errorKey) => {
+ try {
+ await call()
+ } catch (err) {
+ addToast?.(t(errorKey, { error: err.message }), 'error')
+ }
+ refresh()
+ }
+
+ return (
+ act(() => failoverApi.pin(name, target), 'failover.errors.pin')}
+ onUnpin={() => act(() => failoverApi.unpin(name), 'failover.errors.unpin')}
+ />
+ )
+}
diff --git a/core/http/react-ui/src/components/FailoverTargetsEditor.jsx b/core/http/react-ui/src/components/FailoverTargetsEditor.jsx
new file mode 100644
index 000000000..b52f276e8
--- /dev/null
+++ b/core/http/react-ui/src/components/FailoverTargetsEditor.jsx
@@ -0,0 +1,169 @@
+import { useMemo } from 'react'
+import { useFormContext } from '../contexts/FormContext'
+import SearchableModelSelect from './SearchableModelSelect'
+import Toggle from './Toggle'
+
+// FailoverTargetsEditor renders the ordered member list of a failover
+// chain. Each row binds a downstream model plus a warm flag that keeps a
+// local target loaded rather than cold-started the first time it serves.
+//
+// Schema mirrors core/config.FailoverTarget:
+// { model: string, warm: bool }
+//
+// Three conditions make the chain useless even though the YAML stays
+// valid, so they are surfaced as inline warnings rather than blocked
+// outright (the admin may be mid-edit):
+// - fewer than two targets: nothing to fail over TO.
+// - the same model listed twice: the prober would track one backend
+// under two identities.
+// - a target equal to the chain's OWN name: the alias would point at
+// itself and never resolve.
+//
+// The model list from useModels() (what SearchableModelSelect searches)
+// does not carry each entry's backend, so a remote/proxy target can't be
+// told apart from a local one here to disable warm for it. The server
+// already rejects/warns on warm for a backend that can't be kept loaded,
+// so this editor leaves the toggle enabled for every row.
+export default function FailoverTargetsEditor({ value, onChange }) {
+ const items = Array.isArray(value) ? value : []
+ const ownName = useOwnModelName()
+
+ const duplicateModels = useMemo(() => {
+ const seen = new Set()
+ const dup = new Set()
+ for (const it of items) {
+ const model = it?.model
+ if (!model) continue
+ if (seen.has(model)) dup.add(model)
+ else seen.add(model)
+ }
+ return dup
+ }, [items])
+
+ const update = (index, mut) => {
+ const next = items.map((it, i) => (i === index ? mut({ ...it }) : it))
+ onChange(next)
+ }
+ const remove = (index) => onChange(items.filter((_, i) => i !== index))
+ const move = (index, dir) => {
+ const j = index + dir
+ if (j < 0 || j >= items.length) return
+ const next = items.slice()
+ ;[next[index], next[j]] = [next[j], next[index]]
+ onChange(next)
+ }
+ const add = () => onChange([...items, { model: '', warm: false }])
+
+ return (
+
+ {items.length === 0 && (
+
+ No targets yet. Add at least two — the first healthy one answers, the rest take over in order.
+
+ )}
+
+ {items.length === 1 && (
+
+
+ Add at least one more target — a chain with a single target has nothing to fail over to.
+
+ )
+}
+
+function TargetRow({ index, total, row, duplicate, isOwnName, onChange, onRemove, onMove }) {
+ const error = isOwnName
+ ? "This is the chain's own name — it would point at itself and never resolve."
+ : duplicate
+ ? 'Duplicate target — the prober would track the same backend under two identities.'
+ : null
+
+ return (
+
)}
diff --git a/core/http/react-ui/src/pages/ModelEditor.jsx b/core/http/react-ui/src/pages/ModelEditor.jsx
index f16b06afd..dc424414c 100644
--- a/core/http/react-ui/src/pages/ModelEditor.jsx
+++ b/core/http/react-ui/src/pages/ModelEditor.jsx
@@ -11,6 +11,7 @@ import FieldBrowser from '../components/FieldBrowser'
import ConfigFieldRenderer from '../components/ConfigFieldRenderer'
import { FormContextProvider } from '../contexts/FormContext'
import TemplateSelector from '../components/TemplateSelector'
+import { ModelFailoverStatus } from '../components/FailoverChainStatus'
import MODEL_TEMPLATES from '../utils/modelTemplates'
import { useTranslation } from 'react-i18next'
@@ -19,7 +20,7 @@ const SECTION_ICONS = {
templates: 'fa-file-code', functions: 'fa-wrench', reasoning: 'fa-brain',
diffusers: 'fa-image', tts: 'fa-volume-up', pipeline: 'fa-code-branch',
grpc: 'fa-server', agent: 'fa-robot', mcp: 'fa-plug', router: 'fa-route', proxy: 'fa-cloud',
- mitm: 'fa-user-secret', pii: 'fa-user-shield', other: 'fa-ellipsis-h',
+ mitm: 'fa-user-secret', pii: 'fa-user-shield', failover: 'fa-shuffle', other: 'fa-ellipsis-h',
}
const SECTION_COLORS = {
@@ -28,7 +29,7 @@ const SECTION_COLORS = {
reasoning: 'var(--color-accent)', diffusers: 'var(--color-warning)', tts: 'var(--color-success)',
pipeline: 'var(--color-accent)', grpc: 'var(--color-text-muted)', agent: 'var(--color-primary)',
mcp: 'var(--color-accent)', router: 'var(--color-accent)', proxy: 'var(--color-info, var(--color-primary))',
- mitm: 'var(--color-warning)', pii: 'var(--color-error)', other: 'var(--color-text-muted)',
+ mitm: 'var(--color-warning)', pii: 'var(--color-error)', failover: 'var(--color-accent)', other: 'var(--color-text-muted)',
}
// flattenConfig turns a parsed YAML config into a flat { 'a.b.c': value }
@@ -462,6 +463,9 @@ export default function ModelEditor() {
+ {/* Live failover health; renders only when this model is a chain */}
+ {!isCreateMode && }
+
{/* Template selector (create mode, step 1) */}
{showTemplateSelector && }
diff --git a/core/http/react-ui/src/pages/Traces.jsx b/core/http/react-ui/src/pages/Traces.jsx
index 6e8f3e800..3a44b9c21 100644
--- a/core/http/react-ui/src/pages/Traces.jsx
+++ b/core/http/react-ui/src/pages/Traces.jsx
@@ -105,6 +105,7 @@ const TYPE_COLORS = {
vector_store: { bg: 'var(--color-accent-light)', color: 'var(--color-data-7)' },
token_classify: { bg: 'var(--color-info-light)', color: 'var(--color-data-3)' },
pattern_pii: { bg: 'var(--color-error-light)', color: 'var(--color-data-2)' },
+ failover: { bg: 'var(--color-warning-light)', color: 'var(--color-data-2)' },
}
function typeBadgeStyle(type) {
diff --git a/core/http/react-ui/src/router.jsx b/core/http/react-ui/src/router.jsx
index 136e0ddf6..c871e3b73 100644
--- a/core/http/react-ui/src/router.jsx
+++ b/core/http/react-ui/src/router.jsx
@@ -131,6 +131,7 @@ const NotFound = page(null, () => import('./pages/NotFound'))
const Usage = page('usage', () => import('./pages/Usage'))
const Users = page('users', () => import('./pages/Users'))
const Middleware = page('middleware', () => import('./pages/Middleware'))
+const Failover = page('failover', () => import('./pages/Failover'))
const Account = page('account', () => import('./pages/Account'))
import ConsoleLayout from './components/console/ConsoleLayout'
@@ -223,6 +224,7 @@ const appChildren = [
{ path: 'usage', element: },
{ path: 'users', element: },
{ path: 'middleware', element: },
+ { path: 'failover', element: },
],
},
diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js
index 0f6be7d64..8729876cb 100644
--- a/core/http/react-ui/src/utils/api.js
+++ b/core/http/react-ui/src/utils/api.js
@@ -600,6 +600,16 @@ export const quantizationApi = {
downloadUrl: (id) => apiUrl(`/api/quantization/jobs/${enc(id)}/download`),
}
+// Failover chains API. Health is pushed over /api/failover/events (SSE);
+// list() seeds the view and backs the periodic resync.
+export const failoverApi = {
+ list: () => fetchJSON(API_CONFIG.endpoints.failoverChains),
+ get: (name) => fetchJSON(API_CONFIG.endpoints.failoverChain(name)),
+ pin: (name, target) => postJSON(API_CONFIG.endpoints.failoverPin(name), { target }),
+ unpin: (name) => fetchJSON(API_CONFIG.endpoints.failoverPin(name), { method: 'DELETE' }),
+ eventsUrl: () => API_CONFIG.endpoints.failoverEvents,
+}
+
// Nodes API (distributed)
export const nodesApi = {
list: () => fetchJSON(API_CONFIG.endpoints.nodes),
diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js
index 00db3215a..9a6c69140 100644
--- a/core/http/react-ui/src/utils/config.js
+++ b/core/http/react-ui/src/utils/config.js
@@ -139,6 +139,12 @@ export const API_CONFIG = {
system: '/system',
corsProxy: '/api/cors-proxy',
+ // Failover chains
+ failoverChains: '/api/failover',
+ failoverChain: (name) => `/api/failover/${encodeURIComponent(name)}`,
+ failoverEvents: '/api/failover/events',
+ failoverPin: (name) => `/api/failover/${encodeURIComponent(name)}/pin`,
+
// Nodes (distributed)
nodes: '/api/nodes',
node: (id) => `/api/nodes/${id}`,
diff --git a/core/http/react-ui/src/utils/modelTemplates.js b/core/http/react-ui/src/utils/modelTemplates.js
index c3675f9db..321101a8f 100644
--- a/core/http/react-ui/src/utils/modelTemplates.js
+++ b/core/http/react-ui/src/utils/modelTemplates.js
@@ -142,6 +142,16 @@ const MODEL_TEMPLATES = [
],
},
},
+ {
+ id: 'failover',
+ label: 'Failover Chain',
+ icon: 'fa-shuffle',
+ description: 'Serve one model name from an ordered list of models. The first healthy one answers; the next takes over when it fails.',
+ fields: {
+ 'name': '',
+ 'failover.targets': [{ model: '' }, { model: '' }],
+ },
+ },
{
id: 'alias',
label: 'Alias / Routing',
diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go
index 1d3c7adaf..7a6f901ec 100644
--- a/core/http/routes/localai.go
+++ b/core/http/routes/localai.go
@@ -171,6 +171,16 @@ func RegisterLocalAIRoutes(router *echo.Echo,
return nil
}))
+ // Failover chains: reads and the event stream use standard auth (any
+ // authenticated caller may watch chain health); pin/unpin are admin-only
+ // since they override the routing decision for every caller of the chain.
+ fm := app.FailoverManager()
+ router.GET("/api/failover", localai.ListFailoverChainsEndpoint(fm))
+ router.GET("/api/failover/events", localai.FailoverEventsEndpoint(fm))
+ router.GET("/api/failover/:chain", localai.GetFailoverChainEndpoint(fm))
+ router.POST("/api/failover/:chain/pin", localai.PinFailoverTargetEndpoint(fm), adminMiddleware)
+ router.DELETE("/api/failover/:chain/pin", localai.UnpinFailoverTargetEndpoint(fm), adminMiddleware)
+
voiceProfiles := app.VoiceProfileStore()
router.GET("/api/voice-profiles", localai.ListVoiceProfilesEndpoint(voiceProfiles))
router.GET("/api/voice-profiles/:id/audio", localai.ServeVoiceProfileAudioEndpoint(voiceProfiles))
diff --git a/core/http/routes/middleware.go b/core/http/routes/middleware.go
index 0195f445e..de43ecf43 100644
--- a/core/http/routes/middleware.go
+++ b/core/http/routes/middleware.go
@@ -12,6 +12,7 @@ import (
"github.com/mudler/LocalAI/core/http/endpoints/localai"
"github.com/mudler/LocalAI/core/http/middleware"
"github.com/mudler/LocalAI/core/services/routing/router"
+ "github.com/mudler/LocalAI/pkg/model"
)
// RegisterMiddlewareRoutes wires the routing-module admin surface that
@@ -348,7 +349,7 @@ func buildPIIStatus(app *application.Application) map[string]any {
// Why is this on? backend default (cloud-proxy) vs an explicit YAML
// toggle. Helps admins understand the resolved state without
// reading source.
- "default_for_backend": !explicit && cfg.Backend == "cloud-proxy",
+ "default_for_backend": !explicit && cfg.Backend == model.CloudProxyBackend,
// The detectors came from the global default, not this model's YAML.
"detectors_from_default": enabled && len(ownDetectors) == 0 && len(detectors) > 0,
}
@@ -403,7 +404,7 @@ func buildPIIStatus(app *application.Application) map[string]any {
return map[string]any{
"enabled_globally": true,
- "default_enabled_for_backends": []string{"cloud-proxy"},
+ "default_enabled_for_backends": []string{model.CloudProxyBackend},
"models": models,
"detector_models": detectorModels,
"recent_event_count": recentCount,
diff --git a/core/services/advisorylock/held_lock.go b/core/services/advisorylock/held_lock.go
new file mode 100644
index 000000000..99ece9e22
--- /dev/null
+++ b/core/services/advisorylock/held_lock.go
@@ -0,0 +1,216 @@
+package advisorylock
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "fmt"
+ "sync"
+ "time"
+
+ "github.com/mudler/xlog"
+ "gorm.io/gorm"
+)
+
+// heldLockCheckTimeout bounds an acquire, the liveness check and the unlock,
+// so a hung database cannot stall the caller's loop. A var so tests can
+// shorten it.
+var heldLockCheckTimeout = 5 * time.Second
+
+// heldLockSessionSettings make the server notice a lock holder whose host
+// died without closing the connection (crash, power loss, partition) within
+// about 30 seconds, instead of after the OS keepalive default of over two
+// hours, during which nobody else could take the lock.
+var heldLockSessionSettings = []string{
+ "SET tcp_keepalives_idle = 10",
+ "SET tcp_keepalives_interval = 5",
+ "SET tcp_keepalives_count = 3",
+}
+
+// heldLockUserTimeout bounds how long unacknowledged data (for example a
+// keepalive reply to a dead host) may stay in flight. PostgreSQL 12 added it,
+// so it is applied separately and an older server's error is ignored.
+const heldLockUserTimeout = "SET tcp_user_timeout = 30000"
+
+// HeldLock is an advisory lock that stays taken across calls, for leader
+// election where leadership must be sticky. TryWithLockCtx releases the lock
+// when fn returns; with a short fn, every contender wins it in turn and
+// leadership flips on each tick.
+//
+// On PostgreSQL the lock belongs to one session, so HeldLock keeps a
+// dedicated connection out of the pool. The same session is reused for
+// later attempts while the lock is taken elsewhere, so a follower does not
+// open a connection per try. If the holding session dies, the server drops
+// the lock and another instance can take it. On other dialects it holds the
+// package's in-process lock for the key.
+type HeldLock struct {
+ db *gorm.DB
+ key int64
+
+ mu sync.Mutex
+ conn *sql.Conn // PostgreSQL: the dedicated session, holding the lock or not
+ held bool // PostgreSQL: conn holds the lock
+ local bool // other dialects: this lock holds the in-process slot
+ closed bool // Close was called; the lock is never taken again
+}
+
+// NewHeldLock returns a lock for key on db. It takes nothing until TryAcquire.
+func NewHeldLock(db *gorm.DB, key int64) *HeldLock {
+ return &HeldLock{db: db, key: key}
+}
+
+// TryAcquire takes the lock without blocking. It returns true if this
+// HeldLock holds the lock afterwards, including when it already held it.
+func (l *HeldLock) TryAcquire(ctx context.Context) (bool, error) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.closed {
+ return false, nil
+ }
+ if l.held || l.local {
+ return true, nil
+ }
+
+ if !isPostgres(l.db) {
+ select {
+ case localLockChan(l.key) <- struct{}{}:
+ l.local = true
+ return true, nil
+ default:
+ return false, nil
+ }
+ }
+
+ // Callers pass long-lived contexts (the application's, which never ends),
+ // and l.mu is held throughout: a hung database would otherwise block this
+ // caller and every other user of the lock for good. The deadline only
+ // covers obtaining the session, which stays usable afterwards.
+ ctx, cancel := context.WithTimeout(ctx, heldLockCheckTimeout)
+ defer cancel()
+ if l.conn == nil {
+ conn, err := l.openSession(ctx)
+ if err != nil {
+ return false, err
+ }
+ l.conn = conn
+ }
+ var acquired bool
+ if err := l.conn.QueryRowContext(ctx, "SELECT pg_try_advisory_lock($1)", l.key).Scan(&acquired); err != nil {
+ // The lock may have been granted before the error (a cancelled
+ // context, say); discarding the session is the only way to be sure
+ // it is not left held.
+ discardConn(l.conn)
+ l.conn = nil
+ return false, fmt.Errorf("pg_try_advisory_lock: %w", err)
+ }
+ l.held = acquired
+ return acquired, nil
+}
+
+// openSession takes a connection out of the pool for this lock and applies
+// the keepalive settings to it. The session never goes back to the pool, so
+// other pool users do not inherit those settings.
+func (l *HeldLock) openSession(ctx context.Context) (*sql.Conn, error) {
+ sqlDB, err := l.db.DB()
+ if err != nil {
+ return nil, fmt.Errorf("get sql.DB: %w", err)
+ }
+ conn, err := sqlDB.Conn(ctx)
+ if err != nil {
+ return nil, fmt.Errorf("advisory lock conn: %w", err)
+ }
+ for _, stmt := range heldLockSessionSettings {
+ if _, err := conn.ExecContext(ctx, stmt); err != nil {
+ discardConn(conn)
+ return nil, fmt.Errorf("advisory lock session %q: %w", stmt, err)
+ }
+ }
+ if _, err := conn.ExecContext(ctx, heldLockUserTimeout); err != nil {
+ xlog.Debug("advisory lock session: tcp_user_timeout not supported, relying on keepalives", "error", err)
+ }
+ return conn, nil
+}
+
+// Held reports whether this HeldLock believes it holds the lock. It does not
+// touch the database; use Verify for that.
+func (l *HeldLock) Held() bool {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ return l.held || l.local
+}
+
+// Verify reports whether the lock is still held. On PostgreSQL it checks
+// that the holding session is alive; if it is not, the lock is gone
+// server-side, so Verify drops it here too and returns false.
+func (l *HeldLock) Verify(ctx context.Context) bool {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.local {
+ return true
+ }
+ if !l.held {
+ return false
+ }
+ pctx, cancel := context.WithTimeout(ctx, heldLockCheckTimeout)
+ defer cancel()
+ if err := l.conn.PingContext(pctx); err != nil {
+ xlog.Warn("advisory lock session lost, giving up the lock", "key", l.key, "error", err)
+ // A ping that only timed out may leave the session (and the lock)
+ // alive; discarding closes it, so the server releases the lock.
+ discardConn(l.conn)
+ l.conn = nil
+ l.held = false
+ return false
+ }
+ return true
+}
+
+// Release gives up the lock; a later TryAcquire may take it again. It is safe
+// to call when the lock is not held.
+func (l *HeldLock) Release() {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.releaseLocked()
+}
+
+// Close gives up the lock for good: TryAcquire returns false afterwards. Use
+// it on shutdown, where a loop still running could otherwise take the lock
+// back right after Release (and, on the in-process fallback, keep it for the
+// rest of the process).
+func (l *HeldLock) Close() {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.closed = true
+ l.releaseLocked()
+}
+
+func (l *HeldLock) releaseLocked() {
+ if l.local {
+ <-localLockChan(l.key)
+ l.local = false
+ return
+ }
+ if l.conn == nil {
+ return
+ }
+ if l.held {
+ ctx, cancel := context.WithTimeout(context.Background(), heldLockCheckTimeout)
+ defer cancel()
+ if _, err := l.conn.ExecContext(ctx, "SELECT pg_advisory_unlock($1)", l.key); err != nil {
+ xlog.Warn("advisory lock unlock failed, closing its session instead", "key", l.key, "error", err)
+ }
+ }
+ // Never hand the session back to the pool: if the unlock failed it still
+ // holds the lock, and a later pool user would silently inherit it.
+ discardConn(l.conn)
+ l.conn = nil
+ l.held = false
+}
+
+// discardConn closes conn's underlying session instead of returning it to
+// the pool. database/sql drops a connection when Raw's callback returns
+// driver.ErrBadConn.
+func discardConn(conn *sql.Conn) {
+ _ = conn.Raw(func(any) error { return driver.ErrBadConn })
+ _ = conn.Close()
+}
diff --git a/core/services/advisorylock/held_lock_test.go b/core/services/advisorylock/held_lock_test.go
new file mode 100644
index 000000000..17f2c17b9
--- /dev/null
+++ b/core/services/advisorylock/held_lock_test.go
@@ -0,0 +1,183 @@
+package advisorylock
+
+import (
+ "context"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ "github.com/mudler/LocalAI/core/services/testutil"
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+)
+
+// expectCloseIsFinal checks that Close frees the lock for a rival and that the
+// closed HeldLock never takes it back, as a still-running loop would try to.
+func expectCloseIsFinal(db *gorm.DB, key int64) {
+ ctx := context.Background()
+ l, rival := NewHeldLock(db, key), NewHeldLock(db, key)
+ DeferCleanup(rival.Release)
+
+ ok, err := l.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+ l.Close()
+ Expect(l.Held()).To(BeFalse())
+
+ ok, err = l.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeFalse(), "a closed lock is never taken again")
+
+ ok, err = rival.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue(), "Close frees the lock for others")
+ l.Close() // idempotent
+}
+
+// expectStickyHandover checks the contract both backends share: the first
+// holder keeps the lock across repeated checks while a rival is denied, and
+// the rival gets it once the holder releases.
+func expectStickyHandover(db *gorm.DB, key int64) {
+ ctx := context.Background()
+ first, second := NewHeldLock(db, key), NewHeldLock(db, key)
+ DeferCleanup(first.Release)
+ DeferCleanup(second.Release)
+
+ ok, err := first.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+
+ for range 5 {
+ Expect(first.Held()).To(BeTrue())
+ Expect(first.Verify(ctx)).To(BeTrue())
+ ok, err = second.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeFalse(), "a rival must not take a held lock")
+ Expect(second.Held()).To(BeFalse())
+ }
+
+ first.Release()
+ Expect(first.Held()).To(BeFalse())
+ Expect(first.Verify(ctx)).To(BeFalse())
+
+ ok, err = second.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue(), "the lock is free once the holder releases it")
+ Expect(second.Verify(ctx)).To(BeTrue())
+}
+
+var _ = Describe("HeldLock (SQLite fallback)", Label("sqlite"), func() {
+ It("stays with its holder until Release", func() {
+ db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+ Expect(err).ToNot(HaveOccurred())
+ expectStickyHandover(db, 12101)
+ })
+
+ It("never takes the lock again after Close", func() {
+ db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+ Expect(err).ToNot(HaveOccurred())
+ expectCloseIsFinal(db, 12103)
+ })
+
+ It("is idempotent: acquiring twice and releasing twice is safe", func() {
+ db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+ Expect(err).ToNot(HaveOccurred())
+ l := NewHeldLock(db, 12102)
+ ok, err := l.TryAcquire(context.Background())
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+ ok, err = l.TryAcquire(context.Background())
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue(), "a holder asking again keeps the lock")
+ l.Release()
+ l.Release()
+ ok, err = l.TryAcquire(context.Background())
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+ l.Release()
+ })
+})
+
+var _ = Describe("HeldLock (PostgreSQL)", func() {
+ var db *gorm.DB
+
+ BeforeEach(func() {
+ db = testutil.SetupTestDB()
+ })
+
+ It("stays with its holder until Release, then hands over", func() {
+ expectStickyHandover(db, 12201)
+ })
+
+ It("never takes the lock again after Close", func() {
+ expectCloseIsFinal(db, 12203)
+ })
+
+ It("sets short TCP keepalives on its session so a dead host's lock expires", func() {
+ // Killing a host without closing its socket is not practical in a
+ // test; check the settings that make the server notice one.
+ ctx := context.Background()
+ l := NewHeldLock(db, 12204)
+ DeferCleanup(l.Release)
+ ok, err := l.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+
+ show := func(name string) string {
+ var v string
+ Expect(l.conn.QueryRowContext(ctx, "SHOW "+name).Scan(&v)).To(Succeed())
+ return v
+ }
+ Expect(show("tcp_keepalives_idle")).To(Equal("10"))
+ Expect(show("tcp_keepalives_interval")).To(Equal("5"))
+ Expect(show("tcp_keepalives_count")).To(Equal("3"))
+ Expect(show("tcp_user_timeout")).To(Equal("30000")) // milliseconds
+ })
+
+ It("reuses one session while the lock is taken elsewhere", func() {
+ ctx := context.Background()
+ holder, follower := NewHeldLock(db, 12205), NewHeldLock(db, 12205)
+ DeferCleanup(holder.Release)
+ DeferCleanup(follower.Release)
+ ok, err := holder.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+
+ pid := func() int {
+ var p int
+ Expect(follower.conn.QueryRowContext(ctx, "SELECT pg_backend_pid()").Scan(&p)).To(Succeed())
+ return p
+ }
+ ok, err = follower.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeFalse())
+ before := pid()
+ ok, err = follower.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeFalse())
+ Expect(pid()).To(Equal(before), "a follower must not open a connection per attempt")
+ })
+
+ It("drops leadership when the holding session dies, freeing the lock", func() {
+ ctx := context.Background()
+ first, second := NewHeldLock(db, 12202), NewHeldLock(db, 12202)
+ DeferCleanup(first.Release)
+ DeferCleanup(second.Release)
+
+ ok, err := first.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue())
+
+ // Kill the holder's backend, as a network cut or DB restart would.
+ var pid int
+ Expect(first.conn.QueryRowContext(ctx, "SELECT pg_backend_pid()").Scan(&pid)).To(Succeed())
+ Expect(db.Exec("SELECT pg_terminate_backend(?)", pid).Error).ToNot(HaveOccurred())
+
+ Expect(first.Verify(ctx)).To(BeFalse(), "a dead session no longer holds the lock")
+ Expect(first.Held()).To(BeFalse())
+
+ ok, err = second.TryAcquire(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(ok).To(BeTrue(), "the server released the dead session's lock")
+ })
+})
diff --git a/core/services/advisorylock/held_lock_timeout_test.go b/core/services/advisorylock/held_lock_timeout_test.go
new file mode 100644
index 000000000..0f8c6f0a3
--- /dev/null
+++ b/core/services/advisorylock/held_lock_timeout_test.go
@@ -0,0 +1,60 @@
+package advisorylock
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "errors"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "gorm.io/driver/postgres"
+ "gorm.io/gorm"
+)
+
+// hungConnector opens connections that accept session settings but never
+// answer a query, like a database host that stopped responding mid-session.
+type hungConnector struct{}
+
+func (hungConnector) Connect(context.Context) (driver.Conn, error) { return hungConn{}, nil }
+func (hungConnector) Driver() driver.Driver { return hungDriver{} }
+
+type hungDriver struct{}
+
+func (hungDriver) Open(string) (driver.Conn, error) { return hungConn{}, nil }
+
+type hungConn struct{}
+
+func (hungConn) Prepare(string) (driver.Stmt, error) { return nil, errors.New("not supported") }
+func (hungConn) Close() error { return nil }
+func (hungConn) Begin() (driver.Tx, error) { return nil, errors.New("not supported") }
+func (hungConn) ExecContext(context.Context, string, []driver.NamedValue) (driver.Result, error) {
+ return driver.RowsAffected(0), nil
+}
+func (hungConn) QueryContext(ctx context.Context, _ string, _ []driver.NamedValue) (driver.Rows, error) {
+ <-ctx.Done()
+ return nil, ctx.Err()
+}
+
+var _ = Describe("HeldLock against a hung database", func() {
+ It("gives up an acquire after a bounded time, even with a context that never ends", func() {
+ prev := heldLockCheckTimeout
+ heldLockCheckTimeout = 200 * time.Millisecond
+ DeferCleanup(func() { heldLockCheckTimeout = prev })
+
+ db, err := gorm.Open(postgres.New(postgres.Config{Conn: sql.OpenDB(hungConnector{})}), &gorm.Config{DisableAutomaticPing: true})
+ Expect(err).ToNot(HaveOccurred())
+ l := NewHeldLock(db, 4242)
+
+ done := make(chan error, 1)
+ go func() {
+ _, err := l.TryAcquire(context.Background())
+ done <- err
+ }()
+ var got error
+ Eventually(done, 5*time.Second).Should(Receive(&got))
+ Expect(got).To(HaveOccurred())
+ Expect(l.Held()).To(BeFalse())
+ })
+})
diff --git a/core/services/advisorylock/keys.go b/core/services/advisorylock/keys.go
index 277817229..76a1834ed 100644
--- a/core/services/advisorylock/keys.go
+++ b/core/services/advisorylock/keys.go
@@ -12,4 +12,7 @@ const (
KeySchemaMigrate int64 = 105
KeyBackendUpgradeCheck int64 = 106
KeyStateReconciler int64 = 107
+ // KeyFailoverProber elects the one frontend that probes failover
+ // targets, decides chains and preloads warm targets.
+ KeyFailoverProber int64 = 108
)
diff --git a/core/services/failover/classify.go b/core/services/failover/classify.go
new file mode 100644
index 000000000..ac499fa69
--- /dev/null
+++ b/core/services/failover/classify.go
@@ -0,0 +1,101 @@
+package failover
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "regexp"
+ "strconv"
+ "strings"
+
+ "github.com/labstack/echo/v4"
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+)
+
+// cloud-proxy translate mode reports upstream failures as plain text, so the
+// status is only visible in the message.
+var upstreamStatusRe = regexp.MustCompile(`upstream (\d{3})`)
+
+// Errors that the next target would reject in the same way.
+var requestErrorMarkers = []string{
+ "exceeds the available context size",
+ "is larger than the max context size",
+ "maximum context length",
+ // A payload over the gRPC message cap is ResourceExhausted too, but it
+ // is the request's size, not the target's load: every target has the
+ // same cap.
+ "larger than max",
+ "gRPC message limit exceeded",
+}
+
+// IsRetryable reports whether a failed attempt should move to the next
+// target. status is the HTTP status a handler wrote, or 0 when it returned err
+// without writing.
+func IsRetryable(err error, status int) bool {
+ if errors.Is(err, context.Canceled) {
+ return false
+ }
+ if status != 0 {
+ return retryableStatus(status)
+ }
+ if err == nil {
+ return false
+ }
+ var he *echo.HTTPError
+ if errors.As(err, &he) {
+ return retryableStatus(he.Code)
+ }
+ if errors.Is(err, context.DeadlineExceeded) {
+ return true
+ }
+ if st, ok := grpcstatus.FromError(err); ok {
+ switch st.Code() {
+ // ResourceExhausted (a rate-limited upstream, what localai-proxy
+ // returns for a 429, or a backend out of memory) is retried
+ // elsewhere and trips the target: a gap would skip a chronically
+ // exhausted target forever without moving traffic off it.
+ case codes.Unavailable, codes.Internal, codes.DeadlineExceeded, codes.Unknown, codes.ResourceExhausted:
+ return !isRequestError(st.Message())
+ default:
+ return false
+ }
+ }
+ msg := err.Error()
+ if m := upstreamStatusRe.FindStringSubmatch(msg); m != nil {
+ code, _ := strconv.Atoi(m[1])
+ return retryableStatus(code)
+ }
+ // Anything else is usually a dial or load failure of this target.
+ return !isRequestError(msg)
+}
+
+// IsCapabilityGap reports a target that cannot serve this kind of request at
+// all (gRPC Unimplemented, anywhere in the error chain, or the HTTP 501 that
+// the non-OpenAI endpoints map it to). The next target may serve it, and this
+// target is not broken: the failure carries no signal about its health, so
+// callers must skip it without tripping.
+func IsCapabilityGap(err error) bool {
+ if err == nil {
+ return false
+ }
+ var he *echo.HTTPError
+ if errors.As(err, &he) && he.Code == http.StatusNotImplemented {
+ return true
+ }
+ st, ok := grpcstatus.FromError(err)
+ return ok && st.Code() == codes.Unimplemented
+}
+
+func retryableStatus(code int) bool {
+ return code >= 500 && code != http.StatusNotImplemented
+}
+
+func isRequestError(msg string) bool {
+ for _, m := range requestErrorMarkers {
+ if strings.Contains(msg, m) {
+ return true
+ }
+ }
+ return false
+}
diff --git a/core/services/failover/classify_test.go b/core/services/failover/classify_test.go
new file mode 100644
index 000000000..08bc05f07
--- /dev/null
+++ b/core/services/failover/classify_test.go
@@ -0,0 +1,58 @@
+package failover
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "net/http"
+
+ "github.com/labstack/echo/v4"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+)
+
+var _ = DescribeTable("IsRetryable",
+ func(err error, status int, want bool) {
+ Expect(IsRetryable(err, status)).To(Equal(want))
+ },
+ Entry("nil error, no status", nil, 0, false),
+ Entry("held 503", nil, http.StatusServiceUnavailable, true),
+ Entry("held 500", nil, http.StatusInternalServerError, true),
+ Entry("held 501", nil, http.StatusNotImplemented, false),
+ Entry("client cancel", context.Canceled, 0, false),
+ Entry("wrapped client cancel", fmt.Errorf("predict: %w", context.Canceled), 0, false),
+ Entry("deadline", context.DeadlineExceeded, 0, true),
+ Entry("echo 502", echo.NewHTTPError(http.StatusBadGateway, "x"), 0, true),
+ Entry("echo 400", echo.NewHTTPError(http.StatusBadRequest, "x"), 0, false),
+ Entry("echo 404", echo.NewHTTPError(http.StatusNotFound, "x"), 0, false),
+ Entry("grpc unavailable", grpcstatus.Error(codes.Unavailable, "x"), 0, true),
+ Entry("grpc internal", grpcstatus.Error(codes.Internal, "x"), 0, true),
+ Entry("grpc deadline", grpcstatus.Error(codes.DeadlineExceeded, "x"), 0, true),
+ Entry("grpc unknown", grpcstatus.Error(codes.Unknown, "x"), 0, true),
+ Entry("grpc invalid argument", grpcstatus.Error(codes.InvalidArgument, "x"), 0, false),
+ Entry("grpc resource exhausted (rate limit, OOM)", grpcstatus.Error(codes.ResourceExhausted, "x"), 0, true),
+ Entry("cloud-proxy upstream 503", errors.New("cloud-proxy: upstream 503: no healthy nodes"), 0, true),
+ Entry("cloud-proxy upstream 429 stays 4xx", errors.New("cloud-proxy: upstream 429: slow down"), 0, false),
+ Entry("context overflow", errors.New("the request exceeds the available context size"), 0, false),
+ Entry("grpc received message over the size limit", grpcstatus.Error(codes.ResourceExhausted, "grpc: received message larger than max (60000000 vs. 52428800)"), 0, false),
+ Entry("grpc sent message over the size limit", grpcstatus.Error(codes.ResourceExhausted, "grpc: trying to send message larger than max (60000000 vs. 52428800)"), 0, false),
+ Entry("embeddings over the gRPC message limit", fmt.Errorf("conversation too long for per-token embeddings (gRPC message limit exceeded): %w",
+ grpcstatus.Error(codes.ResourceExhausted, "grpc: received message larger than max (1 vs. 0)")), 0, false),
+ Entry("dial error", errors.New("dial tcp 10.0.0.1:8080: connect: connection refused"), 0, true),
+)
+
+var _ = DescribeTable("IsCapabilityGap",
+ func(err error, want bool) {
+ Expect(IsCapabilityGap(err)).To(Equal(want))
+ },
+ Entry("nil error", nil, false),
+ Entry("grpc unimplemented", grpcstatus.Error(codes.Unimplemented, "x"), true),
+ Entry("wrapped grpc unimplemented", fmt.Errorf("call: %w", grpcstatus.Error(codes.Unimplemented, "x")), true),
+ Entry("echo 501 (a handler that mapped Unimplemented)", echo.NewHTTPError(http.StatusNotImplemented, "x"), true),
+ Entry("echo 500 is a failure, not a gap", echo.NewHTTPError(http.StatusInternalServerError, "x"), false),
+ Entry("grpc resource exhausted is a failure, not a gap", grpcstatus.Error(codes.ResourceExhausted, "x"), false),
+ Entry("grpc unavailable", grpcstatus.Error(codes.Unavailable, "x"), false),
+ Entry("plain error", errors.New("boom"), false),
+)
diff --git a/core/services/failover/distsync/distsync.go b/core/services/failover/distsync/distsync.go
new file mode 100644
index 000000000..825b495c6
--- /dev/null
+++ b/core/services/failover/distsync/distsync.go
@@ -0,0 +1,176 @@
+package distsync
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "reflect"
+ "time"
+
+ "github.com/mudler/LocalAI/core/services/failover"
+ "github.com/mudler/LocalAI/core/services/messaging"
+ "github.com/mudler/LocalAI/core/services/syncstate"
+ "github.com/mudler/xlog"
+)
+
+// compile-time assertions.
+var (
+ _ syncstate.Store[string, PinRecord] = (*PinStore)(nil)
+ _ failover.StateSync = (*Sync)(nil)
+)
+
+// pinReconcileInterval is how often the pins map re-reads the DB.
+const pinReconcileInterval = 30 * time.Second
+
+// Sync is a failover.StateSync backed by three syncstate.SyncedMaps:
+//
+// - failover.pins keeps the durable source of truth: Store-backed (when a
+// PinStore is given) so a late joiner hydrates every pin from the DB on
+// Start, not just from whatever peers happen to broadcast afterwards,
+// and re-reads it every pinReconcileInterval to repair a missed delta.
+// - failover.targets and failover.chains are ephemeral live-health state,
+// NATS-only with no Store and no Reconcile: with neither set, a Reconcile
+// tick's hydrate is a no-op (nothing durable to pull from), so it could
+// never help a late joiner catch up anyway. A late joiner instead catches
+// up from the leader's periodic Republish (see failover.Manager.Republish).
+type Sync struct {
+ pins *syncstate.SyncedMap[string, PinRecord]
+ targets *syncstate.SyncedMap[string, failover.TargetSnapshot]
+ chains *syncstate.SyncedMap[string, failover.ChainSnapshot]
+}
+
+// New builds and starts the three maps, then attaches the result to m via
+// SetStateSync so any already-durable pins hydrate onto m immediately.
+func New(ctx context.Context, nats messaging.MessagingClient, pins syncstate.Store[string, PinRecord], m *failover.Manager) (*Sync, error) {
+ s := &Sync{}
+
+ // pins is already typed as the Store interface (the brief fixes this
+ // signature), so a caller holding a nil *PinStore (e.g. standalone mode,
+ // no DB configured) and passing it straight through boxes it into a
+ // non-nil interface wrapping a nil pointer - "pins != nil" alone would
+ // not catch that, and the SyncedMap would then try to hydrate/write
+ // through a nil *gorm.DB. isNilStore catches both that and a literal nil
+ // argument, the same defense finetune/service.go gets for free by taking
+ // a concrete *distributed.FineTuneStore and nil-checking before boxing it.
+ var pinStore syncstate.Store[string, PinRecord]
+ if !isNilStore(pins) {
+ pinStore = pins
+ }
+
+ // A delta dropped without a reconnect would leave this map stale until
+ // the next reconnect; re-reading the DB repairs it, and the manager's
+ // periodic ReconcilePins carries the repair into the chains.
+ var reconcile time.Duration
+ if pinStore != nil {
+ reconcile = pinReconcileInterval
+ }
+ s.pins = syncstate.New(syncstate.Config[string, PinRecord]{
+ Name: "failover.pins",
+ Key: func(p PinRecord) string { return p.Chain },
+ Nats: nats,
+ Store: pinStore,
+ Reconcile: reconcile,
+ OnApply: func(op string, chain string, v PinRecord) {
+ if op == "delete" {
+ m.ApplyPin(chain, "")
+ return
+ }
+ m.ApplyPin(chain, v.Target)
+ },
+ })
+ if err := s.pins.Start(ctx); err != nil {
+ return nil, fmt.Errorf("distsync: starting pins map: %w", err)
+ }
+
+ s.targets = syncstate.New(syncstate.Config[string, failover.TargetSnapshot]{
+ Name: "failover.targets",
+ Key: func(t failover.TargetSnapshot) string { return t.Target },
+ Nats: nats,
+ OnApply: func(_ string, _ string, v failover.TargetSnapshot) {
+ m.ApplyTarget(v)
+ },
+ })
+ if err := s.targets.Start(ctx); err != nil {
+ _ = s.pins.Close()
+ return nil, fmt.Errorf("distsync: starting targets map: %w", err)
+ }
+
+ s.chains = syncstate.New(syncstate.Config[string, failover.ChainSnapshot]{
+ Name: "failover.chains",
+ Key: func(c failover.ChainSnapshot) string { return c.Chain },
+ Nats: nats,
+ OnApply: func(_ string, _ string, v failover.ChainSnapshot) {
+ m.ApplyChain(v)
+ },
+ })
+ if err := s.chains.Start(ctx); err != nil {
+ _ = s.targets.Close()
+ _ = s.pins.Close()
+ return nil, fmt.Errorf("distsync: starting chains map: %w", err)
+ }
+
+ m.SetStateSync(s)
+ // The pins map re-hydrates from the DB on reconnect without OnApply, so
+ // hand the manager the result. Registered after the map's own callback,
+ // which runs first.
+ if r, ok := nats.(interface{ OnReconnect(func()) }); ok {
+ r.OnReconnect(m.ReconcilePins)
+ }
+ return s, nil
+}
+
+// isNilStore reports whether pins is nil - either a literal nil argument, or
+// the classic Go footgun of a non-nil interface value wrapping a nil
+// pointer (e.g. a nil *PinStore passed in directly). Both must disable the
+// durable Store the same way, so syncstate hydrates from nothing rather than
+// panicking on a nil *gorm.DB the first time it dereferences it.
+func isNilStore(pins syncstate.Store[string, PinRecord]) bool {
+ if pins == nil {
+ return true
+ }
+ v := reflect.ValueOf(pins)
+ return v.Kind() == reflect.Ptr && v.IsNil()
+}
+
+// Close releases all three maps' subscriptions and background workers.
+func (s *Sync) Close() error {
+ return errors.Join(s.chains.Close(), s.targets.Close(), s.pins.Close())
+}
+
+// PublishTarget shares a target health transition. StateSync's methods
+// return no error to the manager, and a publish must never block it (the
+// manager calls this outside its lock precisely so a synchronous NATS echo
+// is safe) - so a failure here is logged and dropped; a missed publish
+// self-heals on the leader's next Republish.
+func (s *Sync) PublishTarget(t failover.TargetSnapshot) {
+ if err := s.targets.Set(context.Background(), t); err != nil {
+ xlog.Warn("distsync: publishing target state failed", "target", t.Target, "error", err)
+ }
+}
+
+// PublishChain shares the leader's decision for a chain.
+func (s *Sync) PublishChain(c failover.ChainSnapshot) {
+ if err := s.chains.Set(context.Background(), c); err != nil {
+ xlog.Warn("distsync: publishing chain state failed", "chain", c.Chain, "error", err)
+ }
+}
+
+// SetPin durably persists and broadcasts a pin.
+func (s *Sync) SetPin(chain, target string) error {
+ return s.pins.Set(context.Background(), PinRecord{Chain: chain, Target: target, UpdatedAt: time.Now()})
+}
+
+// ClearPin durably removes and broadcasts a pin's removal.
+func (s *Sync) ClearPin(chain string) error {
+ return s.pins.Delete(context.Background(), chain)
+}
+
+// Pins returns every known pin (chain -> target), for Manager.SetStateSync's
+// hydrate-on-attach.
+func (s *Sync) Pins() map[string]string {
+ out := make(map[string]string)
+ for chain, rec := range s.pins.Snapshot() {
+ out[chain] = rec.Target
+ }
+ return out
+}
diff --git a/core/services/failover/distsync/distsync_suite_test.go b/core/services/failover/distsync/distsync_suite_test.go
new file mode 100644
index 000000000..37a6d7092
--- /dev/null
+++ b/core/services/failover/distsync/distsync_suite_test.go
@@ -0,0 +1,13 @@
+package distsync_test
+
+import (
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+func TestDistsync(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "Distsync test suite")
+}
diff --git a/core/services/failover/distsync/distsync_test.go b/core/services/failover/distsync/distsync_test.go
new file mode 100644
index 000000000..4f4f37985
--- /dev/null
+++ b/core/services/failover/distsync/distsync_test.go
@@ -0,0 +1,330 @@
+package distsync_test
+
+import (
+ "context"
+ "errors"
+ "sync"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+
+ "gorm.io/driver/sqlite"
+ "gorm.io/gorm"
+
+ "github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/core/services/failover"
+ "github.com/mudler/LocalAI/core/services/failover/distsync"
+ "github.com/mudler/LocalAI/core/services/testutil"
+)
+
+// fakeSource is a minimal failover.ConfigSource with one chain "chain" of two
+// targets: "x" (remote, primary) and "y" (local warm, fallback). It is
+// shared across the managers in a spec, mirroring how frontends in a real
+// deployment read the same config loader.
+type fakeSource struct {
+ mu sync.Mutex
+ cfgs map[string]config.ModelConfig
+}
+
+func newChainSource() *fakeSource {
+ return &fakeSource{cfgs: map[string]config.ModelConfig{
+ "x": {Name: "x", Backend: "cloud-proxy"},
+ "y": {Name: "y", Backend: "llama-cpp"},
+ "chain": {Name: "chain", Failover: &config.FailoverConfig{Targets: []config.FailoverTarget{
+ {Model: "x"},
+ {Model: "y", Warm: true},
+ }}},
+ }}
+}
+
+func (s *fakeSource) GetModelConfig(n string) (config.ModelConfig, bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ c, ok := s.cfgs[n]
+ return c, ok
+}
+
+func (s *fakeSource) GetAllModelsConfigs() []config.ModelConfig {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ out := make([]config.ModelConfig, 0, len(s.cfgs))
+ for _, c := range s.cfgs {
+ out = append(out, c)
+ }
+ return out
+}
+
+// memPinStore is an in-memory syncstate.Store[string, distsync.PinRecord]
+// shared by several distsync.Sync instances the way a real DB would be, so a
+// spec can build a "late joiner" that hydrates from what earlier instances
+// already wrote through.
+type memPinStore struct {
+ mu sync.Mutex
+ data map[string]distsync.PinRecord
+ // down, when set, makes every call fail, simulating a database outage.
+ down error
+}
+
+func (s *memPinStore) setDown(err error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.down = err
+}
+
+func newMemPinStore() *memPinStore { return &memPinStore{data: map[string]distsync.PinRecord{}} }
+
+func (s *memPinStore) List(context.Context) ([]distsync.PinRecord, error) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.down != nil {
+ return nil, s.down
+ }
+ out := make([]distsync.PinRecord, 0, len(s.data))
+ for _, v := range s.data {
+ out = append(out, v)
+ }
+ return out, nil
+}
+
+func (s *memPinStore) Upsert(_ context.Context, v distsync.PinRecord) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.down != nil {
+ return s.down
+ }
+ s.data[v.Chain] = v
+ return nil
+}
+
+func (s *memPinStore) Delete(_ context.Context, k string) error {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ if s.down != nil {
+ return s.down
+ }
+ delete(s.data, k)
+ return nil
+}
+
+// leaderGateFor grants leadership to exactly one named manager at a time
+// (whatever *leader currently holds), mirroring an advisory-lock leader loop
+// where only one frontend probes and decides chains.
+func leaderGateFor(name string, leader *string) failover.LeaderGate {
+ return func(_ context.Context, fn func()) bool {
+ if *leader != name {
+ return false
+ }
+ fn()
+ return true
+ }
+}
+
+var errBoom = errors.New("boom")
+
+var _ = Describe("distsync", func() {
+ var (
+ ctx context.Context
+ bus *testutil.FakeBus
+ src *fakeSource
+ pinStore *memPinStore
+ leader string
+ )
+
+ BeforeEach(func() {
+ ctx = context.Background()
+ bus = testutil.NewFakeBus()
+ src = newChainSource()
+ pinStore = newMemPinStore()
+ leader = "a"
+ })
+
+ // newManager wires a fresh failover.Manager to a fresh distsync.Sync on
+ // the shared bus and pin store, named so leaderGateFor can grant or deny
+ // it leadership.
+ newManager := func(name string) (*failover.Manager, *distsync.Sync) {
+ m := failover.New(src, failover.WithLeaderGate(leaderGateFor(name, &leader)))
+ s, err := distsync.New(ctx, bus, pinStore, m)
+ Expect(err).ToNot(HaveOccurred())
+ return m, s
+ }
+
+ It("a pin on A is visible on B and survives a new instance C built from the same store", func() {
+ a, _ := newManager("a")
+ b, _ := newManager("b")
+ a.Tick(ctx)
+ b.Tick(ctx)
+
+ Expect(a.Pin("chain", "y")).To(Succeed())
+
+ stB, ok := b.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(stB.Pinned).ToNot(BeNil())
+ Expect(*stB.Pinned).To(Equal("y"))
+
+ // C is built after the pin was already written through to the shared
+ // store, and before ever ticking: SetStateSync (inside distsync.New)
+ // hydrates C's pins from s.Pins(), so the chain is created pinned the
+ // first time anything asks for it.
+ c, _ := newManager("c")
+ stC, ok := c.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(stC.Pinned).ToNot(BeNil())
+ Expect(*stC.Pinned).To(Equal("y"))
+ })
+
+ It("re-applies pins that changed while B missed the deltas, after a reconnect", func() {
+ a, _ := newManager("a")
+ b, _ := newManager("b")
+ a.Tick(ctx)
+ b.Tick(ctx)
+
+ // Written to the DB with no broadcast: B's map only learns it from
+ // the reconnect re-hydrate, which fires no OnApply.
+ Expect(pinStore.Upsert(ctx, distsync.PinRecord{Chain: "chain", Target: "y", UpdatedAt: time.Now()})).To(Succeed())
+ bus.TriggerReconnect()
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+
+ Expect(pinStore.Delete(ctx, "chain")).To(Succeed())
+ bus.TriggerReconnect()
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ st, _ = a.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ })
+
+ It("does not re-apply a pin whose write failed during a database outage", func() {
+ a, _ := newManager("a")
+ b, _ := newManager("b")
+ a.Tick(ctx)
+ b.Tick(ctx)
+
+ pinStore.setDown(errBoom)
+ Expect(a.Pin("chain", "y")).To(MatchError(errBoom))
+ st, _ := a.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil(), "a failed pin write must be rolled back")
+
+ // The periodic pin re-sync runs while the database is still down: it
+ // must not resurrect the pin the rollback just undid.
+ a.ReconcilePins()
+ st, _ = a.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil(), "the failed pin must not come back on this frontend")
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil(), "the failed pin must never reach a peer")
+
+ // Same for an unpin that fails: the pin stays in force everywhere.
+ pinStore.setDown(nil)
+ Expect(a.Pin("chain", "y")).To(Succeed())
+ pinStore.setDown(errBoom)
+ Expect(a.Unpin("chain")).To(MatchError(errBoom))
+ a.ReconcilePins()
+ st, _ = a.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ })
+
+ It("a trip on B makes A's plan skip the target", func() {
+ a, _ := newManager("a")
+ b, _ := newManager("b")
+ a.Tick(ctx)
+ b.Tick(ctx)
+
+ b.ReportFailure("x", errBoom)
+
+ att, err := a.Plan("chain")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(att.Target()).To(Equal("y"))
+ })
+
+ It("the leader's chain switch reaches the follower", func() {
+ a, _ := newManager("a") // leader
+ b, _ := newManager("b") // follower
+ a.Tick(ctx)
+ b.Tick(ctx)
+
+ a.ReportFailure("x", errBoom)
+
+ st, ok := b.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(st.Active).To(Equal("y"))
+ })
+
+ It("late joiner converges on heartbeat", func() {
+ a, _ := newManager("a") // leader
+ a.Tick(ctx)
+
+ a.ReportFailure("x", errBoom)
+
+ // C joins after the trip: failover.targets/chains are NATS-only with
+ // no Store, so C's hydrate on Start sees nothing and it starts out
+ // believing every target is healthy.
+ c, _ := newManager("c") // follower
+ c.Tick(ctx)
+
+ before, ok := c.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(before.Active).To(Equal("x"))
+
+ a.Republish()
+
+ after, ok := c.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(after.Active).To(Equal("y"))
+ })
+
+ It("guards against a typed-nil PinStore passed as the Store interface", func() {
+ var nilStore *distsync.PinStore // deliberately typed, deliberately nil
+ m := failover.New(src, failover.WithLeaderGate(leaderGateFor("a", &leader)))
+ _, err := distsync.New(ctx, bus, nilStore, m)
+ Expect(err).ToNot(HaveOccurred())
+ m.Tick(ctx)
+
+ Expect(m.Pin("chain", "y")).To(Succeed())
+ st, ok := m.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ })
+
+ It("Close stops all three maps without erroring", func() {
+ _, s := newManager("a")
+ Expect(s.Close()).To(Succeed())
+ })
+
+ It("PinStore round-trips through a real sqlite-backed gorm DB", func() {
+ db, err := gorm.Open(sqlite.Open("file::memory:?cache=shared"), &gorm.Config{})
+ Expect(err).ToNot(HaveOccurred())
+
+ store, err := distsync.NewPinStore(db)
+ Expect(err).ToNot(HaveOccurred())
+
+ recs, err := store.List(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(recs).To(BeEmpty())
+
+ Expect(store.Upsert(ctx, distsync.PinRecord{Chain: "chain", Target: "y", UpdatedAt: time.Now()})).To(Succeed())
+
+ recs, err = store.List(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(recs).To(HaveLen(1))
+ Expect(recs[0].Chain).To(Equal("chain"))
+ Expect(recs[0].Target).To(Equal("y"))
+
+ // Upsert again on the same key updates rather than duplicating.
+ Expect(store.Upsert(ctx, distsync.PinRecord{Chain: "chain", Target: "x", UpdatedAt: time.Now()})).To(Succeed())
+ recs, err = store.List(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(recs).To(HaveLen(1))
+ Expect(recs[0].Target).To(Equal("x"))
+
+ Expect(store.Delete(ctx, "chain")).To(Succeed())
+ recs, err = store.List(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(recs).To(BeEmpty())
+ })
+})
diff --git a/core/services/failover/distsync/pinstore.go b/core/services/failover/distsync/pinstore.go
new file mode 100644
index 000000000..eaa682431
--- /dev/null
+++ b/core/services/failover/distsync/pinstore.go
@@ -0,0 +1,63 @@
+// Package distsync wires failover.StateSync to syncstate.SyncedMap so target
+// health, chain decisions and pins are shared across frontends over NATS,
+// with pins durable in a small gorm-backed table.
+package distsync
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/mudler/LocalAI/core/services/advisorylock"
+ "gorm.io/gorm"
+)
+
+// PinRecord is the durable form of a chain pin. It doubles as the value type
+// of the "failover.pins" SyncedMap, so a hydrate (Store.List) needs no
+// conversion and the wire delta carries the exact row.
+type PinRecord struct {
+ Chain string `gorm:"primaryKey" json:"chain"`
+ Target string `json:"target"`
+ UpdatedAt time.Time `json:"updated_at"`
+}
+
+// TableName pins the table name independent of the Go type name.
+func (PinRecord) TableName() string { return "failover_pins" }
+
+// PinStore is gorm-backed durable storage for chain pins, implementing
+// syncstate.Store[string, PinRecord] (asserted in distsync.go).
+type PinStore struct {
+ db *gorm.DB
+}
+
+// NewPinStore migrates the failover_pins table under the schema-migrate
+// advisory lock - the same guard jobs.NewJobStore uses - so several
+// frontends starting at once do not race on the migration, and returns a
+// ready-to-use store.
+func NewPinStore(db *gorm.DB) (*PinStore, error) {
+ if err := advisorylock.WithLockCtx(context.Background(), db, advisorylock.KeySchemaMigrate, func() error {
+ return db.AutoMigrate(&PinRecord{})
+ }); err != nil {
+ return nil, fmt.Errorf("distsync: migrating pin table: %w", err)
+ }
+ return &PinStore{db: db}, nil
+}
+
+// List returns every durable pin, for hydrate on Start.
+func (s *PinStore) List(ctx context.Context) ([]PinRecord, error) {
+ var out []PinRecord
+ if err := s.db.WithContext(ctx).Find(&out).Error; err != nil {
+ return nil, err
+ }
+ return out, nil
+}
+
+// Upsert writes a pin through. Save inserts or updates by primary key.
+func (s *PinStore) Upsert(ctx context.Context, v PinRecord) error {
+ return s.db.WithContext(ctx).Save(&v).Error
+}
+
+// Delete removes a pin by chain name.
+func (s *PinStore) Delete(ctx context.Context, k string) error {
+ return s.db.WithContext(ctx).Delete(&PinRecord{Chain: k}).Error
+}
diff --git a/core/services/failover/failover_suite_test.go b/core/services/failover/failover_suite_test.go
new file mode 100644
index 000000000..1a7bce9d5
--- /dev/null
+++ b/core/services/failover/failover_suite_test.go
@@ -0,0 +1,13 @@
+package failover
+
+import (
+ "testing"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+func TestFailover(t *testing.T) {
+ RegisterFailHandler(Fail)
+ RunSpecs(t, "Failover test suite")
+}
diff --git a/core/services/failover/fakes_test.go b/core/services/failover/fakes_test.go
new file mode 100644
index 000000000..232b7b4df
--- /dev/null
+++ b/core/services/failover/fakes_test.go
@@ -0,0 +1,98 @@
+package failover
+
+import (
+ "sort"
+ "sync"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+)
+
+type fakeClock struct {
+ mu sync.Mutex
+ now time.Time
+}
+
+func newFakeClock() *fakeClock { return &fakeClock{now: time.Date(2026, 9, 26, 10, 0, 0, 0, time.UTC)} }
+func (c *fakeClock) Now() time.Time {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.now
+}
+func (c *fakeClock) Advance(d time.Duration) {
+ c.mu.Lock()
+ c.now = c.now.Add(d)
+ c.mu.Unlock()
+}
+
+type fakeSource struct {
+ mu sync.Mutex
+ cfgs map[string]config.ModelConfig
+ scans int // GetAllModelsConfigs calls
+}
+
+func newFakeSource(cfgs ...config.ModelConfig) *fakeSource {
+ s := &fakeSource{cfgs: map[string]config.ModelConfig{}}
+ for _, c := range cfgs {
+ s.cfgs[c.Name] = c
+ }
+ return s
+}
+func (s *fakeSource) Put(c config.ModelConfig) { s.mu.Lock(); s.cfgs[c.Name] = c; s.mu.Unlock() }
+func (s *fakeSource) Delete(name string) { s.mu.Lock(); delete(s.cfgs, name); s.mu.Unlock() }
+func (s *fakeSource) Scans() int { s.mu.Lock(); defer s.mu.Unlock(); return s.scans }
+func (s *fakeSource) GetModelConfig(n string) (config.ModelConfig, bool) {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ c, ok := s.cfgs[n]
+ return c, ok
+}
+func (s *fakeSource) GetAllModelsConfigs() []config.ModelConfig {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ s.scans++
+ out := make([]config.ModelConfig, 0, len(s.cfgs))
+ for _, c := range s.cfgs {
+ out = append(out, c)
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
+ return out
+}
+
+func local(name string) config.ModelConfig {
+ return config.ModelConfig{Name: name, Backend: "llama-cpp"}
+}
+func remote(name string) config.ModelConfig {
+ return config.ModelConfig{Name: name, Backend: "cloud-proxy"}
+}
+
+// chainCfg builds a chain; fc may be nil for defaults.
+func chainCfg(name string, fc *config.FailoverConfig, targets ...config.FailoverTarget) config.ModelConfig {
+ f := config.FailoverConfig{}
+ if fc != nil {
+ f = *fc
+ }
+ f.Targets = targets
+ return config.ModelConfig{Name: name, Failover: &f}
+}
+
+func t(model string) config.FailoverTarget { return config.FailoverTarget{Model: model} }
+func warmT(model string) config.FailoverTarget {
+ return config.FailoverTarget{Model: model, Warm: true}
+}
+
+// drain returns the events buffered so far without blocking.
+func drain(ch <-chan Event) []Event {
+ var out []Event
+ for {
+ select {
+ case ev, ok := <-ch:
+ if !ok {
+ return out
+ }
+ out = append(out, ev)
+ default:
+ return out
+ }
+ }
+}
diff --git a/core/services/failover/manager.go b/core/services/failover/manager.go
new file mode 100644
index 000000000..878c1c867
--- /dev/null
+++ b/core/services/failover/manager.go
@@ -0,0 +1,798 @@
+package failover
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "slices"
+ "sort"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/xlog"
+)
+
+var (
+ ErrChainNotFound = errors.New("failover chain not found")
+ ErrTargetNotInChain = errors.New("target is not in this failover chain")
+ ErrNoTarget = errors.New("failover chain has no usable target")
+)
+
+// ConfigSource is the part of ModelConfigLoader the manager reads.
+type ConfigSource interface {
+ GetModelConfig(name string) (config.ModelConfig, bool)
+ GetAllModelsConfigs() []config.ModelConfig
+}
+
+type Clock interface{ Now() time.Time }
+
+type realClock struct{}
+
+func (realClock) Now() time.Time { return time.Now() }
+
+type Option func(*Manager)
+
+func WithClock(c Clock) Option { return func(m *Manager) { m.clock = c } }
+func WithProber(p Prober) Option { return func(m *Manager) { m.prober = p } }
+
+// WithOnWarmChanged is called outside the manager lock when the set of warm
+// local targets changes. The application pins and preloads them.
+func WithOnWarmChanged(fn func(warm []string)) Option { return func(m *Manager) { m.onWarm = fn } }
+
+// Manager tracks health per target and the active target per chain.
+type Manager struct {
+ mu sync.Mutex
+ // id tells this manager's own target publishes apart when the sync
+ // layer echoes them back.
+ id string
+ src ConfigSource
+ clock Clock
+ prober Prober
+ onWarm func([]string)
+ // pins holds every known pin by chain name, including pins for chains
+ // or targets this frontend's config does not have yet: with a sync
+ // layer the pin is delivered once, and a chain that appears (or is
+ // rebuilt) later must still pick it up.
+ pins map[string]string
+ targets map[string]*targetState
+ chains map[string]*chainState
+ subs map[int]chan Event
+ nextSub int
+ warm []string
+ warmPending bool
+ closed bool
+ // hasChains mirrors len(chains) > 0 as of the last sync, so the request
+ // path can check it without the lock or a config-source scan.
+ hasChains atomic.Bool
+ // probes counts running probes; only tests wait on it.
+ probes sync.WaitGroup
+
+ // sync shares state with other frontends; nil when standalone.
+ sync StateSync
+ // gate grants probing and chain decisions to one frontend; nil means
+ // this manager is always the leader.
+ gate LeaderGate
+ leader bool
+ // applying is set while a peer's target state is applied, so the
+ // transition is not published back to the peers.
+ applying bool
+ // pending holds publishes queued under the lock; unlockAndFlush runs
+ // them after unlocking because the sync layer can call back into Apply*.
+ pending []func()
+ // ticks counts Run's ticks for the periodic republish; only Run uses it.
+ ticks int
+}
+
+type targetState struct {
+ name string
+ // serving is the model that serves this target: its own name, or the
+ // model an alias points to. Model loading and eviction only know it.
+ serving string
+ kind Kind
+ warm bool
+ state TargetState
+ failures []time.Time
+ consecutiveOK int
+ downSince time.Time
+ lastProbe time.Time
+ lastActivity time.Time
+ lastError string
+ // since and reason describe the last state change, for snapshots.
+ since time.Time
+ reason Reason
+ // probing is set while a probe runs, so the scheduler does not start a
+ // second one for the same target.
+ probing bool
+ // params come from the first chain, in name order, that lists the target.
+ params config.FailoverConfig
+}
+
+func (ts *targetState) cold() bool { return ts.kind == KindLocal && !ts.warm }
+
+type chainState struct {
+ name string
+ cfg config.FailoverConfig
+ targets []string
+ active int
+ activeSince time.Time
+ pinned string
+ state ChainState
+ // adopted is set once a follower received the leader's decision for this
+ // chain; from then on it stops choosing the active target itself.
+ adopted bool
+}
+
+func New(src ConfigSource, opts ...Option) *Manager {
+ m := &Manager{
+ id: uuid.NewString(),
+ src: src,
+ clock: realClock{},
+ targets: map[string]*targetState{},
+ chains: map[string]*chainState{},
+ pins: map[string]string{},
+ subs: map[int]chan Event{},
+ }
+ for _, o := range opts {
+ o(m)
+ }
+ m.leader = m.gate == nil
+ return m
+}
+
+// Sync reconciles chains with the config source. There is no config-change
+// hook in the loader, so this runs on every tick and on a lookup miss.
+func (m *Manager) Sync() {
+ m.mu.Lock()
+ m.syncLocked()
+ warm, deliver := m.takeWarmLocked()
+ m.unlockAndFlush()
+ if deliver && m.onWarm != nil {
+ m.onWarm(warm)
+ }
+}
+
+func (m *Manager) syncLocked() {
+ now := m.clock.Now()
+ seenChains := map[string]bool{}
+ claimed := map[string]bool{}
+ for _, c := range m.src.GetAllModelsConfigs() {
+ if !c.IsFailover() {
+ continue
+ }
+ seenChains[c.Name] = true
+ names := make([]string, 0, len(c.Failover.Targets))
+ for _, t := range c.Failover.Targets {
+ names = append(names, t.Model)
+ }
+ ch := m.chains[c.Name]
+ if ch == nil || !slices.Equal(ch.targets, names) {
+ pinned := m.pins[c.Name]
+ if !slices.Contains(names, pinned) {
+ if m.sync == nil {
+ // Standalone, the pin lives with the chain: a target
+ // dropped from it takes the pin along.
+ delete(m.pins, c.Name)
+ }
+ pinned = ""
+ }
+ ch = &chainState{name: c.Name, targets: names, activeSince: now, state: ChainPrimary, pinned: pinned}
+ m.chains[c.Name] = ch
+ }
+ ch.cfg = *c.Failover
+ for _, t := range c.Failover.Targets {
+ ts := m.targets[t.Model]
+ if ts == nil {
+ ts = &targetState{name: t.Model, state: StateHealthy}
+ m.targets[t.Model] = ts
+ }
+ if !claimed[t.Model] {
+ claimed[t.Model] = true
+ ts.params = *c.Failover
+ ts.warm = false
+ }
+ tc, ok := m.lookupTarget(t.Model)
+ if !ok {
+ m.setTargetLocked(ts, StateMissing, ReasonMissing, "target config not found")
+ continue
+ }
+ if tc.IsFailover() {
+ // Nested chains are rejected when a chain is saved, but a
+ // target edited into a chain later slips past that check.
+ // Serving its chain config would load a model with no
+ // backend, so treat it as unusable.
+ m.setTargetLocked(ts, StateMissing, ReasonMissing, "target is itself a failover chain")
+ continue
+ }
+ ts.kind = KindOf(tc)
+ ts.serving = tc.Name
+ if t.Warm && ts.kind == KindLocal {
+ ts.warm = true
+ }
+ if ts.state == StateMissing {
+ m.setTargetLocked(ts, StateHealthy, ReasonRecovery, "")
+ }
+ }
+ }
+ for name := range m.chains {
+ if !seenChains[name] {
+ delete(m.chains, name)
+ if m.sync == nil {
+ // With a sync layer the shared pin outlives a chain this
+ // frontend has not (re)loaded yet; standalone it does not.
+ delete(m.pins, name)
+ }
+ }
+ }
+ for name := range m.targets {
+ if !claimed[name] {
+ delete(m.targets, name)
+ }
+ }
+ m.hasChains.Store(len(m.chains) > 0)
+ for _, ch := range m.chains {
+ m.recomputeLocked(ch, "")
+ }
+ var warm []string
+ for _, ts := range m.targets {
+ if ts.warm && ts.serving != "" && !slices.Contains(warm, ts.serving) {
+ warm = append(warm, ts.serving)
+ }
+ }
+ sort.Strings(warm)
+ if !slices.Equal(warm, m.warm) {
+ m.warm = warm
+ m.warmPending = true
+ }
+}
+
+func (m *Manager) takeWarmLocked() ([]string, bool) {
+ if !m.warmPending {
+ return nil, false
+ }
+ m.warmPending = false
+ return slices.Clone(m.warm), true
+}
+
+// lookupTarget returns the config that serves a target, one alias hop deep.
+func (m *Manager) lookupTarget(name string) (config.ModelConfig, bool) {
+ c, ok := m.src.GetModelConfig(name)
+ if ok && c.IsAlias() {
+ return m.src.GetModelConfig(c.Alias)
+ }
+ return c, ok
+}
+
+// HasChains reports whether any failover chain was configured at the last
+// sync. The request path calls it on every request to skip chain bookkeeping
+// on installations without chains, so it reads a flag instead of scanning
+// the config source (which takes the loader's lock and copies every config).
+// A chain added since the last sync is still served, because Plan syncs on a
+// miss; only in-request retry is missing for it until the scheduler's next
+// tick, at most one second later.
+func (m *Manager) HasChains() bool {
+ if m == nil {
+ return false
+ }
+ return m.hasChains.Load()
+}
+
+func (m *Manager) chainLocked(name string) *chainState {
+ if ch := m.chains[name]; ch != nil {
+ return ch
+ }
+ m.syncLocked()
+ return m.chains[name]
+}
+
+// targetLocked looks up a target's health state, syncing lazily on a miss so
+// ReportFailure/ReportSuccess work before the first Plan or Sync call.
+func (m *Manager) targetLocked(name string) *targetState {
+ if ts := m.targets[name]; ts != nil {
+ return ts
+ }
+ m.syncLocked()
+ return m.targets[name]
+}
+
+// WarmTargets returns the warm local targets, sorted.
+func (m *Manager) WarmTargets() []string {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return slices.Clone(m.warm)
+}
+
+// targetStates snapshots each target's health state for the metrics gauge.
+func (m *Manager) targetStates() map[string]TargetState {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ out := make(map[string]TargetState, len(m.targets))
+ for name, ts := range m.targets {
+ out[name] = ts.state
+ }
+ return out
+}
+
+// Reevaluate recomputes every chain. Dwell-based fail-back needs no event, so
+// the scheduler calls this on every tick.
+func (m *Manager) Reevaluate() {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ for _, ch := range m.chains {
+ m.recomputeLocked(ch, "")
+ }
+}
+
+func (m *Manager) setTargetLocked(ts *targetState, to TargetState, reason Reason, errMsg string) {
+ if ts.state == to {
+ return
+ }
+ from := ts.state
+ now := m.clock.Now()
+ ts.state = to
+ ts.since, ts.reason = now, reason
+ switch to {
+ case StateDown:
+ ts.downSince = now
+ ts.consecutiveOK = 0
+ ts.failures = nil
+ case StateRecovering, StateHealthy:
+ ts.consecutiveOK = 0
+ ts.failures = nil
+ }
+ m.emitLocked(Event{Type: EventTargetState, Target: ts.name, From: string(from), To: string(to), Reason: reason, Error: errMsg, At: now})
+ m.queuePublishTargetLocked(ts, from)
+}
+
+// recomputeLocked picks the active target. override replaces the reason of a
+// resulting switch (pin and unpin are always "manual").
+func (m *Manager) recomputeLocked(ch *chainState, override Reason) {
+ if m.sync != nil && !m.leader && ch.adopted && ch.pinned == "" {
+ // The leader decides; deciding here too would let frontends serve
+ // different targets. A pin is exempt: it fixes the active target
+ // the same way on every frontend, and applying it at once gives the
+ // caller read-your-writes.
+ return
+ }
+ now := m.clock.Now()
+ prev := ch.active
+ next := prev
+ reason := ReasonTrip
+ best := -1
+ for i, name := range ch.targets {
+ if ts := m.targets[name]; ts != nil && ts.state == StateHealthy {
+ best = i
+ break
+ }
+ }
+ switch {
+ case ch.pinned != "":
+ next = slices.Index(ch.targets, ch.pinned)
+ reason = ReasonManual
+ case best == -1:
+ // Nothing is healthy: keep the active target, Plan tries all of them.
+ case best > prev:
+ next = best // the active target is not healthy
+ case best < prev:
+ cur := m.targets[ch.targets[prev]]
+ curHealthy := cur != nil && cur.state == StateHealthy
+ if !curHealthy {
+ next = best
+ } else if now.Sub(ch.activeSince) >= ch.cfg.MinDwell() {
+ next = best
+ reason = ReasonRecovery
+ }
+ }
+ if override != "" {
+ reason = override
+ }
+ var state ChainState
+ switch {
+ case ch.pinned == "" && best == -1:
+ state = ChainDegraded
+ case next == 0:
+ state = ChainPrimary
+ default:
+ state = ChainFallback
+ }
+ changed := next != prev || state != ch.state
+ switch {
+ case next != prev:
+ ch.active = next
+ ch.activeSince = now
+ m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: reason, At: now})
+ case state == ChainDegraded && ch.state != ChainDegraded:
+ // Entering degraded with no active-target change (every target is down).
+ m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: ReasonDegraded, At: now})
+ case state != ChainDegraded && ch.state == ChainDegraded:
+ // Leaving degraded with no active-target change (the active target
+ // itself recovered): SSE/realtime consumers watch chain.switched.state,
+ // so this must fire or they stay on "degraded" forever.
+ m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(state), Reason: ReasonRecovery, At: now})
+ }
+ ch.state = state
+ if changed {
+ pub := reason
+ if next == prev {
+ // Same reasons as the events above for a state-only change.
+ pub = ReasonRecovery
+ if state == ChainDegraded {
+ pub = ReasonDegraded
+ }
+ }
+ m.queuePublishChainLocked(ch, pub)
+ }
+}
+
+func (m *Manager) recomputeForLocked(target string) {
+ for _, ch := range m.chains {
+ if slices.Contains(ch.targets, target) {
+ m.recomputeLocked(ch, "")
+ }
+ }
+}
+
+// Attempt walks the targets of one request in order.
+type Attempt struct {
+ m *Manager
+ chain string
+ primary string
+ degraded bool
+ targets []string
+ i int
+}
+
+// Plan returns the attempt order for one request: the active target, then the
+// other healthy targets. A degraded chain tries every target in priority
+// order; a pinned chain only the pinned target.
+func (m *Manager) Plan(chain string) (*Attempt, error) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ ch := m.chainLocked(chain)
+ if ch == nil {
+ return nil, fmt.Errorf("%w: %q", ErrChainNotFound, chain)
+ }
+ att := &Attempt{m: m, chain: ch.name, primary: ch.targets[0], degraded: ch.state == ChainDegraded}
+ usable := func(name string) bool {
+ ts := m.targets[name]
+ if ts == nil || ts.state == StateMissing {
+ return false
+ }
+ return att.degraded || ts.state == StateHealthy
+ }
+ switch {
+ case ch.pinned != "":
+ att.targets = []string{ch.pinned}
+ case att.degraded:
+ for _, name := range ch.targets {
+ if usable(name) {
+ att.targets = append(att.targets, name)
+ }
+ }
+ default:
+ active := ch.targets[ch.active]
+ if usable(active) {
+ att.targets = append(att.targets, active)
+ }
+ for _, name := range ch.targets {
+ if name != active && usable(name) {
+ att.targets = append(att.targets, name)
+ }
+ }
+ }
+ if len(att.targets) == 0 {
+ return nil, fmt.Errorf("%w: %q", ErrNoTarget, chain)
+ }
+ return att, nil
+}
+
+func (a *Attempt) Chain() string { return a.chain }
+func (a *Attempt) Target() string { return a.targets[a.i] }
+func (a *Attempt) Primary() string { return a.primary }
+func (a *Attempt) Degraded() bool { return a.degraded }
+
+// Fail records err against the current target and moves to the next one. It
+// returns false when no target is left.
+func (a *Attempt) Fail(err error) bool {
+ a.m.ReportFailure(a.Target(), err)
+ if a.i+1 >= len(a.targets) {
+ return false
+ }
+ a.i++
+ return true
+}
+
+// Skip moves to the next target without recording a failure, for a target
+// that could not take this request although nothing is wrong with it (at
+// capacity, disabled). It returns false when no target is left.
+func (a *Attempt) Skip() bool {
+ if a.i+1 >= len(a.targets) {
+ return false
+ }
+ a.i++
+ return true
+}
+
+// Report records err against the current target without moving on: the
+// response was already committed, so nothing is left to retry.
+func (a *Attempt) Report(err error) { a.m.ReportFailure(a.Target(), err) }
+
+func (a *Attempt) Succeed() { a.m.ReportSuccess(a.Target()) }
+
+func (m *Manager) ReportFailure(target string, err error) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ ts := m.targetLocked(target)
+ if ts == nil {
+ return
+ }
+ msg := ""
+ if err != nil {
+ msg = err.Error()
+ }
+ m.recordFailureLocked(ts, msg)
+ m.recomputeForLocked(target)
+}
+
+func (m *Manager) recordFailureLocked(ts *targetState, msg string) {
+ now := m.clock.Now()
+ ts.lastError = msg
+ switch ts.state {
+ case StateRecovering:
+ m.setTargetLocked(ts, StateDown, ReasonTrip, msg)
+ case StateHealthy:
+ cut := now.Add(-ts.params.TripWindow())
+ kept := ts.failures[:0]
+ for _, f := range ts.failures {
+ if f.After(cut) {
+ kept = append(kept, f)
+ }
+ }
+ ts.failures = append(kept, now)
+ if len(ts.failures) >= ts.params.TripErrors() {
+ m.setTargetLocked(ts, StateDown, ReasonTrip, msg)
+ }
+ }
+}
+
+func (m *Manager) ReportSuccess(target string) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ ts := m.targetLocked(target)
+ if ts == nil {
+ return
+ }
+ ts.lastActivity = m.clock.Now()
+ m.recordPassLocked(ts)
+ m.recomputeForLocked(target)
+}
+
+// recordPassLocked counts a served request or a passed inference probe.
+func (m *Manager) recordPassLocked(ts *targetState) {
+ switch ts.state {
+ case StateHealthy:
+ ts.failures = nil
+ return
+ case StateMissing:
+ return
+ case StateDown:
+ if ts.cold() {
+ // Cold targets are never probed; a served request is proof enough.
+ m.setTargetLocked(ts, StateHealthy, ReasonRecovery, "")
+ return
+ }
+ m.setTargetLocked(ts, StateRecovering, ReasonRecovery, "")
+ }
+ ts.consecutiveOK++
+ if ts.consecutiveOK >= ts.params.RecoveryProbes() {
+ m.setTargetLocked(ts, StateHealthy, ReasonRecovery, "")
+ }
+}
+
+// Pin takes effect here at once (read-your-writes), then is shared with the
+// other frontends.
+func (m *Manager) Pin(chain, target string) error {
+ m.mu.Lock()
+ ch := m.chainLocked(chain)
+ if ch == nil {
+ m.unlockAndFlush()
+ return fmt.Errorf("%w: %q", ErrChainNotFound, chain)
+ }
+ if !slices.Contains(ch.targets, target) {
+ m.unlockAndFlush()
+ return fmt.Errorf("%w: %q", ErrTargetNotInChain, target)
+ }
+ prev := m.pins[chain]
+ ch.pinned = target
+ m.pins[chain] = target
+ m.recomputeLocked(ch, ReasonManual)
+ s := m.sync
+ m.unlockAndFlush()
+ if s == nil {
+ return nil
+ }
+ if err := s.SetPin(chain, target); err != nil {
+ m.rollbackPin(chain, target, prev)
+ return err
+ }
+ return nil
+}
+
+// rollbackPin restores prev after a pin change that the other frontends
+// never saw: serving it here alone would split the cluster. A newer change
+// made meanwhile is kept.
+func (m *Manager) rollbackPin(chain, applied, prev string) {
+ m.mu.Lock()
+ current := m.pins[chain]
+ m.mu.Unlock()
+ if current == applied {
+ m.ApplyPin(chain, prev)
+ }
+}
+
+func (m *Manager) Unpin(chain string) error {
+ m.mu.Lock()
+ ch := m.chainLocked(chain)
+ if ch == nil {
+ m.unlockAndFlush()
+ return fmt.Errorf("%w: %q", ErrChainNotFound, chain)
+ }
+ prev := m.pins[chain]
+ ch.pinned = ""
+ delete(m.pins, chain)
+ m.recomputeLocked(ch, ReasonManual)
+ s := m.sync
+ m.unlockAndFlush()
+ if s == nil {
+ return nil
+ }
+ if err := s.ClearPin(chain); err != nil {
+ m.rollbackPin(chain, "", prev)
+ return err
+ }
+ return nil
+}
+
+// Status returns every chain, sorted by name.
+func (m *Manager) Status() []ChainStatus {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ m.syncLocked()
+ names := make([]string, 0, len(m.chains))
+ for name := range m.chains {
+ names = append(names, name)
+ }
+ sort.Strings(names)
+ out := make([]ChainStatus, 0, len(names))
+ for _, name := range names {
+ out = append(out, m.statusLocked(m.chains[name]))
+ }
+ return out
+}
+
+func (m *Manager) ChainStatus(name string) (ChainStatus, bool) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ ch := m.chainLocked(name)
+ if ch == nil {
+ return ChainStatus{}, false
+ }
+ return m.statusLocked(ch), true
+}
+
+func (m *Manager) statusLocked(ch *chainState) ChainStatus {
+ cs := ChainStatus{Name: ch.name, State: ch.state, Active: ch.targets[ch.active], ActiveSince: ch.activeSince}
+ if ch.pinned != "" {
+ p := ch.pinned
+ cs.Pinned = &p
+ }
+ for _, name := range ch.targets {
+ st := TargetStatus{Model: name}
+ if ts := m.targets[name]; ts != nil {
+ st.Kind, st.Warm, st.State = ts.kind, ts.warm, ts.state
+ st.ConsecutiveOK, st.LastError = ts.consecutiveOK, ts.lastError
+ if !ts.lastProbe.IsZero() {
+ lp := ts.lastProbe
+ st.LastProbe = &lp
+ }
+ }
+ cs.Targets = append(cs.Targets, st)
+ }
+ return cs
+}
+
+// Subscribe returns a buffered event channel and a cancel func. A subscriber
+// that does not keep up loses events rather than blocking the manager.
+func (m *Manager) Subscribe(buffer int) (<-chan Event, func()) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ ch := make(chan Event, buffer)
+ if m.closed {
+ close(ch)
+ return ch, func() {}
+ }
+ id := m.nextSub
+ m.nextSub++
+ m.subs[id] = ch
+ var once sync.Once
+ return ch, func() {
+ once.Do(func() {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if c, ok := m.subs[id]; ok {
+ delete(m.subs, id)
+ close(c)
+ }
+ })
+ }
+}
+
+func (m *Manager) emitLocked(ev Event) {
+ // Every frontend emits the switch to its own subscribers, but only the
+ // one that decided it counts it, or the cluster-wide total would be N
+ // times the real one. Followers adopt the leader's switches (ApplyChain)
+ // and apply pins the leader applies too. Without a sync layer each
+ // frontend decides for itself, so each counts.
+ if ev.Type == EventChainSwitched && (m.leader || m.sync == nil) {
+ recordSwitch(ev)
+ }
+ for _, c := range m.subs {
+ select {
+ case c <- ev:
+ default:
+ xlog.Warn("failover: dropping event for a slow subscriber", "type", ev.Type, "chain", ev.Chain, "target", ev.Target)
+ }
+ }
+}
+
+func (m *Manager) close() {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.closed = true
+ for id, c := range m.subs {
+ close(c)
+ delete(m.subs, id)
+ }
+}
+
+// Do runs fn against the chain's targets in plan order. fn calls commit once
+// output has reached the client; after that a failure is not retried.
+func (m *Manager) Do(ctx context.Context, chain string, fn func(ctx context.Context, target string, commit func()) error) error {
+ att, err := m.Plan(chain)
+ if err != nil {
+ return err
+ }
+ for {
+ var committed atomic.Bool
+ err := fn(ctx, att.Target(), func() { committed.Store(true) })
+ switch {
+ case err == nil:
+ att.Succeed()
+ return nil
+ case IsCapabilityGap(err) && !committed.Load():
+ // This target cannot serve this kind of request at all; it is
+ // not broken, so move on without counting a failure.
+ if !att.Skip() {
+ return err
+ }
+ continue
+ case ctx.Err() != nil || !IsRetryable(err, 0):
+ return err
+ case committed.Load():
+ att.Report(err)
+ return err
+ case !att.Fail(err):
+ return err
+ }
+ }
+}
+
+// Prober checks targets. Implemented by DefaultProber (prober.go).
+type Prober interface {
+ // Liveness is the cheap steady-state check.
+ Liveness(ctx context.Context, target config.ModelConfig, kind Kind, warm bool) error
+ // Inference sends one minimal real request to confirm recovery.
+ Inference(ctx context.Context, target config.ModelConfig, kind Kind, warm bool) error
+}
diff --git a/core/services/failover/manager_test.go b/core/services/failover/manager_test.go
new file mode 100644
index 000000000..69326920c
--- /dev/null
+++ b/core/services/failover/manager_test.go
@@ -0,0 +1,353 @@
+package failover
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ . "github.com/onsi/gomega/gstruct"
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+)
+
+var errBoom = errors.New("dial tcp: connection refused")
+
+var _ = Describe("Manager", func() {
+ var (
+ clock *fakeClock
+ src *fakeSource
+ m *Manager
+ )
+
+ BeforeEach(func() {
+ clock = newFakeClock()
+ src = newFakeSource(remote("a"), local("b"), chainCfg("chain", nil, t("a"), t("b")))
+ m = New(src, WithClock(clock))
+ })
+
+ switched := func(evs []Event) []Event {
+ var out []Event
+ for _, e := range evs {
+ if e.Type == EventChainSwitched {
+ out = append(out, e)
+ }
+ }
+ return out
+ }
+
+ It("plans the primary first on a fresh chain", func() {
+ att, err := m.Plan("chain")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(att.Target()).To(Equal("a"))
+ Expect(att.Primary()).To(Equal("a"))
+ Expect(att.Degraded()).To(BeFalse())
+ st, ok := m.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(st.State).To(Equal(ChainPrimary))
+ Expect(st.Targets[0].Kind).To(Equal(KindRemote))
+ Expect(st.Targets[1].Kind).To(Equal(KindLocal))
+ })
+
+ It("skips to the next target without recording a failure", func() {
+ att, _ := m.Plan("chain")
+ Expect(att.Skip()).To(BeTrue())
+ Expect(att.Target()).To(Equal("b"))
+ Expect(att.Skip()).To(BeFalse())
+ Expect(att.Target()).To(Equal("b"))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Active).To(Equal("a"))
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ })
+
+ It("reports whether any chain was configured at the last sync", func() {
+ m.Sync()
+ Expect(m.HasChains()).To(BeTrue())
+ var nilManager *Manager
+ Expect(nilManager.HasChains()).To(BeFalse())
+ })
+
+ It("answers HasChains from the last sync without scanning the config source", func() {
+ empty := New(newFakeSource(remote("a")), WithClock(clock))
+ Expect(empty.HasChains()).To(BeFalse())
+ lateSrc := newFakeSource(remote("a"), local("b"))
+ late := New(lateSrc, WithClock(clock))
+ late.Sync()
+ scans := lateSrc.Scans()
+ for range 100 {
+ Expect(late.HasChains()).To(BeFalse())
+ }
+ Expect(lateSrc.Scans()).To(Equal(scans))
+ // A chain added since the last sync is seen at the next sync, or
+ // sooner by Plan, which syncs on a miss.
+ lateSrc.Put(chainCfg("chain", nil, t("a"), t("b")))
+ Expect(late.HasChains()).To(BeFalse())
+ _, err := late.Plan("chain")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(late.HasChains()).To(BeTrue())
+ lateSrc.Delete("chain")
+ late.Sync()
+ Expect(late.HasChains()).To(BeFalse())
+ })
+
+ It("returns ErrChainNotFound for an unknown chain", func() {
+ _, err := m.Plan("nope")
+ Expect(errors.Is(err, ErrChainNotFound)).To(BeTrue())
+ })
+
+ It("trips on the first failure by default and switches with an event", func() {
+ events, cancel := m.Subscribe(16)
+ defer cancel()
+ att, _ := m.Plan("chain")
+ Expect(att.Fail(errBoom)).To(BeTrue())
+ Expect(att.Target()).To(Equal("b"))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Active).To(Equal("b"))
+ Expect(st.State).To(Equal(ChainFallback))
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ Expect(st.Targets[0].LastError).To(ContainSubstring("connection refused"))
+ sw := switched(drain(events))
+ Expect(sw).To(HaveLen(1))
+ Expect(sw[0]).To(MatchFields(IgnoreExtras, Fields{
+ "Chain": Equal("chain"), "From": Equal("a"), "To": Equal("b"),
+ "State": Equal("fallback"), "Reason": Equal(ReasonTrip),
+ }))
+ })
+
+ It("counts failures inside the trip window only", func() {
+ src.Put(chainCfg("chain", &config.FailoverConfig{Trip: config.FailoverTrip{Errors: 2, Window: "30s"}}, t("a"), t("b")))
+ m.Sync()
+ m.ReportFailure("a", errBoom)
+ clock.Advance(31 * time.Second)
+ m.ReportFailure("a", errBoom)
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ clock.Advance(time.Second)
+ m.ReportFailure("a", errBoom)
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ })
+
+ It("fails back only after recovery probes and min_dwell", func() {
+ _, _ = m.Plan("chain")
+ m.ReportFailure("a", errBoom)
+ for i := 0; i < 3; i++ {
+ m.ReportSuccess("a") // a real success counts like a passed inference probe
+ }
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ Expect(st.Active).To(Equal("b"), "min_dwell has not passed")
+ clock.Advance(61 * time.Second)
+ events, cancel := m.Subscribe(16)
+ defer cancel()
+ m.Reevaluate()
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Active).To(Equal("a"))
+ Expect(switched(drain(events))[0].Reason).To(Equal(ReasonRecovery))
+ })
+
+ It("moves up at once when the active target itself goes down", func() {
+ src.Put(chainCfg("chain", nil, t("a"), t("b"), t("c")))
+ src.Put(local("c"))
+ m.Sync()
+ m.ReportFailure("a", errBoom) // active: b
+ for i := 0; i < 3; i++ {
+ m.ReportSuccess("a") // a healthy again, but dwell not passed
+ }
+ m.ReportFailure("b", errBoom) // b down: go to a now, not c
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Active).To(Equal("a"))
+ })
+
+ It("goes degraded when all targets are down and plans all of them in priority order", func() {
+ m.ReportFailure("a", errBoom)
+ m.ReportFailure("b", errBoom)
+ st, _ := m.ChainStatus("chain")
+ Expect(st.State).To(Equal(ChainDegraded))
+ att, err := m.Plan("chain")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(att.Degraded()).To(BeTrue())
+ Expect(att.Target()).To(Equal("a"))
+ Expect(att.Fail(errBoom)).To(BeTrue())
+ Expect(att.Target()).To(Equal("b"))
+ Expect(att.Fail(errBoom)).To(BeFalse())
+ })
+
+ It("emits chain.switched when leaving degraded without an active-target change", func() {
+ m.ReportFailure("a", errBoom) // active moves to b
+ m.ReportFailure("b", errBoom) // both down: degraded, active stays b
+ st, _ := m.ChainStatus("chain")
+ Expect(st.State).To(Equal(ChainDegraded))
+ Expect(st.Active).To(Equal("b"))
+ events, cancel := m.Subscribe(16)
+ defer cancel()
+ m.ReportSuccess("b") // b is cold local: one success recovers it in place
+ st, _ = m.ChainStatus("chain")
+ Expect(st.State).To(Equal(ChainFallback))
+ Expect(st.Active).To(Equal("b"), "the active target itself recovered, no switch needed")
+ sw := switched(drain(events))
+ Expect(sw).To(HaveLen(1), "leaving degraded must still notify chain.switched listeners")
+ Expect(sw[0]).To(MatchFields(IgnoreExtras, Fields{
+ "Chain": Equal("chain"), "State": Equal("fallback"), "Reason": Equal(ReasonRecovery),
+ }))
+ })
+
+ It("pins a target regardless of health", func() {
+ Expect(m.Pin("chain", "b")).To(Succeed())
+ att, _ := m.Plan("chain")
+ Expect(att.Target()).To(Equal("b"))
+ Expect(att.Fail(errBoom)).To(BeFalse(), "a pin allows only the pinned target")
+ st, _ := m.ChainStatus("chain")
+ Expect(*st.Pinned).To(Equal("b"))
+ Expect(st.Active).To(Equal("b"))
+ Expect(m.Unpin("chain")).To(Succeed())
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ Expect(errors.Is(m.Pin("chain", "zzz"), ErrTargetNotInChain)).To(BeTrue())
+ Expect(errors.Is(m.Pin("nope", "a"), ErrChainNotFound)).To(BeTrue())
+ })
+
+ It("shares target health across chains", func() {
+ src.Put(chainCfg("chain2", nil, t("a"), t("b")))
+ m.Sync()
+ m.ReportFailure("a", errBoom)
+ s1, _ := m.ChainStatus("chain")
+ s2, _ := m.ChainStatus("chain2")
+ Expect(s1.Active).To(Equal("b"))
+ Expect(s2.Active).To(Equal("b"))
+ })
+
+ It("marks a removed target missing and leaves it out of plans", func() {
+ _, _ = m.Plan("chain")
+ src.Delete("a")
+ m.Sync()
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateMissing))
+ att, _ := m.Plan("chain")
+ Expect(att.Target()).To(Equal("b"))
+ Expect(att.Fail(errBoom)).To(BeFalse())
+ })
+
+ It("resets a chain whose target list changed", func() {
+ m.ReportFailure("a", errBoom)
+ src.Put(local("c"))
+ src.Put(chainCfg("chain", nil, t("c"), t("b")))
+ m.Sync()
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Active).To(Equal("c"))
+ })
+
+ It("reports warm local targets and ignores warm on remote ones", func() {
+ var got []string
+ m = New(src, WithClock(clock), WithOnWarmChanged(func(w []string) { got = w }))
+ src.Put(chainCfg("chain", nil, warmT("a"), warmT("b")))
+ m.Sync()
+ Expect(got).To(Equal([]string{"b"}))
+ Expect(m.WarmTargets()).To(Equal([]string{"b"}))
+ })
+
+ It("reports the model an alias target serves from as warm, not the alias", func() {
+ var got []string
+ m = New(src, WithClock(clock), WithOnWarmChanged(func(w []string) { got = w }))
+ // The preloader and the eviction guard work on loaded model names; an
+ // alias is never loaded under its own name.
+ src.Put(config.ModelConfig{Name: "al", Alias: "b"})
+ src.Put(chainCfg("chain", nil, t("a"), warmT("al")))
+ m.Sync()
+ Expect(got).To(Equal([]string{"b"}))
+ Expect(m.WarmTargets()).To(Equal([]string{"b"}))
+ })
+
+ It("never plans a target that has since become a chain itself", func() {
+ // Validation rejects a nested chain when the outer chain is saved,
+ // but not when one of its targets is later edited into a chain.
+ src.Put(chainCfg("inner", nil, t("a")))
+ src.Put(chainCfg("chain", nil, t("a"), t("inner")))
+ m.Sync()
+ m.ReportFailure("a", errBoom)
+
+ st, ok := m.ChainStatus("chain")
+ Expect(ok).To(BeTrue())
+ Expect(st.Targets[1].State).To(Equal(StateMissing))
+ att, err := m.Plan("chain")
+ if err == nil {
+ Expect(att.Target()).NotTo(Equal("inner"))
+ }
+ })
+
+ It("closes a subscription on cancel", func() {
+ events, cancel := m.Subscribe(1)
+ cancel()
+ _, ok := <-events
+ Expect(ok).To(BeFalse())
+ cancel() // idempotent
+ })
+
+ Describe("Do", func() {
+ It("retries on the next target until commit", func() {
+ var tried []string
+ err := m.Do(context.Background(), "chain", func(_ context.Context, target string, commit func()) error {
+ tried = append(tried, target)
+ if target == "a" {
+ return errBoom
+ }
+ return nil
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(tried).To(Equal([]string{"a", "b"}))
+ })
+
+ It("does not retry after commit but still trips the target", func() {
+ var tried []string
+ err := m.Do(context.Background(), "chain", func(_ context.Context, target string, commit func()) error {
+ tried = append(tried, target)
+ commit()
+ return errBoom
+ })
+ Expect(err).To(MatchError(errBoom))
+ Expect(tried).To(Equal([]string{"a"}))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ })
+
+ It("does not retry or trip on a non-retryable error", func() {
+ bad := errors.New("the request exceeds the available context size")
+ err := m.Do(context.Background(), "chain", func(_ context.Context, _ string, _ func()) error { return bad })
+ Expect(err).To(MatchError(bad))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ })
+
+ It("skips an Unimplemented target without tripping it", func() {
+ var tried []string
+ err := m.Do(context.Background(), "chain", func(_ context.Context, target string, _ func()) error {
+ tried = append(tried, target)
+ if target == "a" {
+ return grpcstatus.Error(codes.Unimplemented, "localai-proxy: Rerank has no upstream counterpart")
+ }
+ return nil
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(tried).To(Equal([]string{"a", "b"}))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ })
+
+ It("fails a rate-limited (ResourceExhausted) target over and trips it", func() {
+ var tried []string
+ err := m.Do(context.Background(), "chain", func(_ context.Context, target string, _ func()) error {
+ tried = append(tried, target)
+ if target == "a" {
+ return grpcstatus.Error(codes.ResourceExhausted, "localai-proxy: upstream /v1/rerank returned 429: slow down")
+ }
+ return nil
+ })
+ Expect(err).ToNot(HaveOccurred())
+ Expect(tried).To(Equal([]string{"a", "b"}))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ })
+ })
+})
diff --git a/core/services/failover/metrics.go b/core/services/failover/metrics.go
new file mode 100644
index 000000000..99d1a8212
--- /dev/null
+++ b/core/services/failover/metrics.go
@@ -0,0 +1,54 @@
+package failover
+
+import (
+ "context"
+ "sync"
+
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/metric"
+)
+
+var (
+ metricsOnce sync.Once
+ switches metric.Int64Counter
+)
+
+func initMetrics() {
+ metricsOnce.Do(func() {
+ meter := otel.Meter("github.com/mudler/LocalAI")
+ switches, _ = meter.Int64Counter("localai_failover_switches_total",
+ metric.WithDescription("Failover chain switches between targets"))
+ })
+}
+
+func recordSwitch(ev Event) {
+ initMetrics()
+ if switches == nil {
+ return
+ }
+ switches.Add(context.Background(), 1, metric.WithAttributes(
+ attribute.String("chain", ev.Chain),
+ attribute.String("from", ev.From),
+ attribute.String("to", ev.To),
+ attribute.String("reason", string(ev.Reason)),
+ ))
+}
+
+// RegisterMetrics exports target health as a gauge. The application calls it
+// once for its manager; tests create many managers and skip it.
+func RegisterMetrics(m *Manager) {
+ meter := otel.Meter("github.com/mudler/LocalAI")
+ _, _ = meter.Int64ObservableGauge("localai_failover_target_up",
+ metric.WithDescription("1 when a failover target is healthy, 0 otherwise"),
+ metric.WithInt64Callback(func(_ context.Context, o metric.Int64Observer) error {
+ for name, state := range m.targetStates() {
+ v := int64(0)
+ if state == StateHealthy {
+ v = 1
+ }
+ o.Observe(v, metric.WithAttributes(attribute.String("target", name)))
+ }
+ return nil
+ }))
+}
diff --git a/core/services/failover/metrics_switch_test.go b/core/services/failover/metrics_switch_test.go
new file mode 100644
index 000000000..ae1b7957a
--- /dev/null
+++ b/core/services/failover/metrics_switch_test.go
@@ -0,0 +1,79 @@
+package failover
+
+import (
+ "context"
+ "sync"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "go.opentelemetry.io/otel"
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
+ "go.opentelemetry.io/otel/sdk/metric/metricdata"
+)
+
+var (
+ switchReaderOnce sync.Once
+ switchReader *sdkmetric.ManualReader
+)
+
+// switchCount reads localai_failover_switches_total. The global provider can
+// only be installed once per process, so every spec shares one reader and
+// compares counts before and after.
+func switchCount() int64 {
+ switchReaderOnce.Do(func() {
+ switchReader = sdkmetric.NewManualReader()
+ otel.SetMeterProvider(sdkmetric.NewMeterProvider(sdkmetric.WithReader(switchReader)))
+ })
+ var rm metricdata.ResourceMetrics
+ Expect(switchReader.Collect(context.Background(), &rm)).To(Succeed())
+ var n int64
+ for _, sm := range rm.ScopeMetrics {
+ for _, mt := range sm.Metrics {
+ if mt.Name != "localai_failover_switches_total" {
+ continue
+ }
+ if sum, ok := mt.Data.(metricdata.Sum[int64]); ok {
+ for _, dp := range sum.DataPoints {
+ n += dp.Value
+ }
+ }
+ }
+ }
+ return n
+}
+
+var _ = Describe("switch metric", func() {
+ It("counts a switch once across the cluster, on the frontend that decided it", func() {
+ clock := newFakeClock()
+ src := newFakeSource(remote("x"), local("y"), chainCfg("chain", nil, t("x"), t("y")))
+ bus := &loopSync{pins: map[string]string{}}
+ leaderIsA := true
+ gateFor := func(isA bool) LeaderGate {
+ return func(_ context.Context, fn func()) bool {
+ if isA != leaderIsA {
+ return false
+ }
+ fn()
+ return true
+ }
+ }
+ a := New(src, WithClock(clock), WithLeaderGate(gateFor(true)))
+ b := New(src, WithClock(clock), WithLeaderGate(gateFor(false)))
+ bus.add(a)
+ bus.add(b)
+ a.SetStateSync(bus)
+ b.SetStateSync(bus)
+ a.Tick(context.Background())
+ b.Tick(context.Background())
+
+ before := switchCount()
+ a.ReportFailure("x", errBoom) // the leader switches, the follower adopts
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Active).To(Equal("y"))
+ Expect(switchCount() - before).To(Equal(int64(1)))
+
+ before = switchCount()
+ Expect(b.Pin("chain", "x")).To(Succeed()) // applied on both frontends
+ Expect(switchCount() - before).To(Equal(int64(1)))
+ })
+})
diff --git a/core/services/failover/metrics_test.go b/core/services/failover/metrics_test.go
new file mode 100644
index 000000000..42dcd24b8
--- /dev/null
+++ b/core/services/failover/metrics_test.go
@@ -0,0 +1,19 @@
+package failover
+
+import (
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("metrics", func() {
+ It("registers and records without a meter provider", func() {
+ src := newFakeSource(remote("a"), local("b"), chainCfg("chain", nil, t("a"), t("b")))
+ m := New(src, WithClock(newFakeClock()))
+ Expect(func() { RegisterMetrics(m) }).ToNot(Panic())
+ Expect(func() { m.ReportFailure("a", errBoom) }).ToNot(Panic())
+ })
+ It("records an attempt trace only when enabled", func() {
+ Expect(func() { RecordAttemptTrace(false, "chain", "a", errBoom) }).ToNot(Panic())
+ Expect(func() { RecordAttemptTrace(true, "chain", "a", errBoom) }).ToNot(Panic())
+ })
+})
diff --git a/core/services/failover/prober.go b/core/services/failover/prober.go
new file mode 100644
index 000000000..cb7a53183
--- /dev/null
+++ b/core/services/failover/prober.go
@@ -0,0 +1,311 @@
+package failover
+
+import (
+ "bytes"
+ "context"
+ "encoding/binary"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "net/url"
+ "strings"
+ "sync"
+
+ "github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/pkg/grpc"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ "github.com/mudler/xlog"
+)
+
+// ErrNotLoaded is what Inference returns for a local target whose backend is
+// not running. It neither confirms nor fails recovery: the manager judges the
+// target like a cold one, by real requests once min_dwell has passed.
+var ErrNotLoaded = errors.New("failover: target is not loaded")
+
+// LoadedFunc returns the running backend of a local target, or nil when it is
+// not loaded. It must never load the model: a probe that loads blocks until
+// the load ends (while the warm preload loads the same model) and then judges
+// the target on an expired context.
+type LoadedFunc func(cfg config.ModelConfig) grpc.Backend
+
+// DefaultProber probes remote targets over the upstream's OpenAI-compatible
+// API and local targets through their gRPC backend.
+type DefaultProber struct {
+ HTTP *http.Client
+ Loaded LoadedFunc
+ EnvLookup func(string) string
+
+ // unresolvableEnvWarned tracks which api_key_env names already got the
+ // "no lookup configured" warning below, so a chain with no working
+ // credential lookup does not re-log on every probe tick.
+ unresolvableEnvWarned sync.Map // map[string]struct{}
+}
+
+func NewProber(loaded LoadedFunc, envLookup func(string) string) *DefaultProber {
+ return &DefaultProber{HTTP: &http.Client{
+ // A redirect is a failed probe, not something to follow: Go resends
+ // custom headers such as x-api-key to any host, and the target's
+ // API key must reach only the configured upstream.
+ CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse },
+ }, Loaded: loaded, EnvLookup: envLookup}
+}
+
+func (p *DefaultProber) Liveness(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
+ switch {
+ case kind == KindRemote:
+ return p.remoteLiveness(ctx, cfg)
+ case warm:
+ return p.localHealth(ctx, cfg)
+ }
+ // A cold target is judged only by real requests: loading it only to probe
+ // it could evict other models, and no cheaper check is reliable. A missing
+ // model file is not one: models download on first use, some backends need
+ // no file, and a dotted name like "Phi-3.5-mini" looks like a file path. A
+ // false "down" would take away the very fallback the chain exists for,
+ // while a real load failure still trips the target and the request moves on.
+ return nil
+}
+
+func (p *DefaultProber) Inference(ctx context.Context, cfg config.ModelConfig, kind Kind, warm bool) error {
+ if kind == KindRemote {
+ return p.remoteInference(ctx, cfg)
+ }
+ return p.localInference(ctx, cfg)
+}
+
+// UpstreamBase strips the endpoint path from a cloud-proxy upstream_url:
+// everything from "/v1" on, so a path prefix before it survives.
+func UpstreamBase(raw string) (string, error) {
+ u, err := url.Parse(raw)
+ if err != nil || u.Scheme == "" || u.Host == "" {
+ return "", fmt.Errorf("invalid upstream_url %q", raw)
+ }
+ path := u.Path
+ if i := strings.Index(path, "/v1"); i >= 0 {
+ path = path[:i]
+ }
+ return u.Scheme + "://" + u.Host + strings.TrimSuffix(path, "/"), nil
+}
+
+// UpstreamModel is the model name the upstream knows the target by.
+func UpstreamModel(cfg config.ModelConfig) string {
+ if cfg.Proxy.UpstreamModel != "" {
+ return cfg.Proxy.UpstreamModel
+ }
+ return cfg.Name
+}
+
+// PrepareTarget readies a copy of a target's config to serve a chain request.
+// A remote target gets its upstream model set explicitly: left empty,
+// passthrough forwards the client's "model" (the chain name) and translate
+// falls back to it, so the upstream would answer 404 for a model the liveness
+// probe (which checks UpstreamModel) just found.
+func PrepareTarget(cfg *config.ModelConfig) {
+ if KindOf(*cfg) == KindRemote {
+ cfg.Proxy.UpstreamModel = UpstreamModel(*cfg)
+ }
+}
+
+func (p *DefaultProber) authorize(req *http.Request, cfg config.ModelConfig) error {
+ if cfg.Proxy.APIKeyEnv != "" && p.EnvLookup == nil {
+ if _, warned := p.unresolvableEnvWarned.LoadOrStore(cfg.Proxy.APIKeyEnv, struct{}{}); !warned {
+ xlog.Warn("failover: remote target has api_key_env set but no credential lookup is configured; liveness and requests will fail authorization", "target", cfg.Name, "api_key_env", cfg.Proxy.APIKeyEnv)
+ }
+ }
+ key, err := cfg.Proxy.ResolveAPIKey(p.EnvLookup)
+ if err != nil || key == "" {
+ return err
+ }
+ if cfg.Proxy.Provider == config.ProxyProviderAnthropic {
+ req.Header.Set("x-api-key", key)
+ req.Header.Set("anthropic-version", "2023-06-01")
+ return nil
+ }
+ req.Header.Set("Authorization", "Bearer "+key)
+ return nil
+}
+
+func (p *DefaultProber) do(req *http.Request, cfg config.ModelConfig) (*http.Response, error) {
+ if err := p.authorize(req, cfg); err != nil {
+ return nil, err
+ }
+ resp, err := p.HTTP.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if resp.StatusCode/100 != 2 {
+ _ = resp.Body.Close()
+ return nil, fmt.Errorf("upstream %s: HTTP %d", req.URL.Path, resp.StatusCode)
+ }
+ return resp, nil
+}
+
+func (p *DefaultProber) remoteLiveness(ctx context.Context, cfg config.ModelConfig) error {
+ base, err := UpstreamBase(cfg.Proxy.UpstreamURL)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/v1/models", nil)
+ if err != nil {
+ return err
+ }
+ resp, err := p.do(req, cfg)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = resp.Body.Close() }()
+ var list struct {
+ Data []struct {
+ ID string `json:"id"`
+ } `json:"data"`
+ }
+ if err := json.NewDecoder(io.LimitReader(resp.Body, 4<<20)).Decode(&list); err != nil {
+ return fmt.Errorf("upstream /v1/models: %w", err)
+ }
+ want := UpstreamModel(cfg)
+ for _, d := range list.Data {
+ if d.ID == want {
+ return nil
+ }
+ }
+ return fmt.Errorf("upstream does not list model %q", want)
+}
+
+func (p *DefaultProber) remoteInference(ctx context.Context, cfg config.ModelConfig) error {
+ base, err := UpstreamBase(cfg.Proxy.UpstreamURL)
+ if err != nil {
+ return err
+ }
+ model := UpstreamModel(cfg)
+ ping := []map[string]string{{"role": "user", "content": "ping"}}
+ switch {
+ case cfg.HasUsecases(config.FLAG_CHAT) || cfg.HasUsecases(config.FLAG_COMPLETION):
+ if cfg.Proxy.Provider == config.ProxyProviderAnthropic {
+ return p.postJSON(ctx, cfg, base+"/v1/messages", map[string]any{"model": model, "max_tokens": 1, "messages": ping})
+ }
+ return p.postJSON(ctx, cfg, base+"/v1/chat/completions", map[string]any{"model": model, "max_tokens": 1, "messages": ping})
+ case cfg.HasUsecases(config.FLAG_EMBEDDINGS):
+ return p.postJSON(ctx, cfg, base+"/v1/embeddings", map[string]any{"model": model, "input": "ping"})
+ case cfg.HasUsecases(config.FLAG_TRANSCRIPT):
+ return p.postTranscription(ctx, cfg, base, model)
+ case cfg.HasUsecases(config.FLAG_TTS):
+ return p.postJSON(ctx, cfg, base+"/v1/audio/speech", map[string]any{"model": model, "input": "ok"})
+ }
+ // Image, video and other costly usecases: liveness is the confirmation.
+ return p.remoteLiveness(ctx, cfg)
+}
+
+func (p *DefaultProber) postJSON(ctx context.Context, cfg config.ModelConfig, endpoint string, body any) error {
+ b, err := json.Marshal(body)
+ if err != nil {
+ return err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(b))
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := p.do(req, cfg)
+ if err != nil {
+ return err
+ }
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
+ return resp.Body.Close()
+}
+
+func (p *DefaultProber) postTranscription(ctx context.Context, cfg config.ModelConfig, base, model string) error {
+ var buf bytes.Buffer
+ mw := multipart.NewWriter(&buf)
+ _ = mw.WriteField("model", model)
+ fw, err := mw.CreateFormFile("file", "probe.wav")
+ if err != nil {
+ return err
+ }
+ _, _ = fw.Write(silenceWAV())
+ if err := mw.Close(); err != nil {
+ return err
+ }
+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, base+"/v1/audio/transcriptions", &buf)
+ if err != nil {
+ return err
+ }
+ req.Header.Set("Content-Type", mw.FormDataContentType())
+ resp, err := p.do(req, cfg)
+ if err != nil {
+ return err
+ }
+ _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, 1<<20))
+ return resp.Body.Close()
+}
+
+// silenceWAV is 200 ms of 16 kHz mono 16-bit silence.
+func silenceWAV() []byte {
+ const rate, samples = 16000, 3200
+ data := samples * 2
+ b := make([]byte, 44+data)
+ copy(b[0:], "RIFF")
+ binary.LittleEndian.PutUint32(b[4:], uint32(36+data))
+ copy(b[8:], "WAVE")
+ copy(b[12:], "fmt ")
+ binary.LittleEndian.PutUint32(b[16:], 16)
+ binary.LittleEndian.PutUint16(b[20:], 1) // PCM
+ binary.LittleEndian.PutUint16(b[22:], 1) // mono
+ binary.LittleEndian.PutUint32(b[24:], rate)
+ binary.LittleEndian.PutUint32(b[28:], rate*2)
+ binary.LittleEndian.PutUint16(b[32:], 2)
+ binary.LittleEndian.PutUint16(b[34:], 16)
+ copy(b[36:], "data")
+ binary.LittleEndian.PutUint32(b[40:], uint32(data))
+ return b
+}
+
+func (p *DefaultProber) loaded(cfg config.ModelConfig) grpc.Backend {
+ if p.Loaded == nil {
+ return nil
+ }
+ return p.Loaded(cfg)
+}
+
+// localHealth checks a warm target's running backend. A target that is not
+// loaded passes: the warm preload is loading it, or a crash removed it and
+// the next real request loads it again and judges it.
+func (p *DefaultProber) localHealth(ctx context.Context, cfg config.ModelConfig) error {
+ b := p.loaded(cfg)
+ if b == nil {
+ return nil
+ }
+ return healthCheck(ctx, b)
+}
+
+func healthCheck(ctx context.Context, b grpc.Backend) error {
+ ok, err := b.HealthCheck(ctx)
+ if err != nil {
+ return err
+ }
+ if !ok {
+ return errors.New("backend health check failed")
+ }
+ return nil
+}
+
+func (p *DefaultProber) localInference(ctx context.Context, cfg config.ModelConfig) error {
+ b := p.loaded(cfg)
+ if b == nil {
+ return ErrNotLoaded
+ }
+ var err error
+ switch {
+ case cfg.HasUsecases(config.FLAG_CHAT) || cfg.HasUsecases(config.FLAG_COMPLETION):
+ _, err = b.Predict(ctx, &pb.PredictOptions{Prompt: "ping", Tokens: 1})
+ return err
+ case cfg.HasUsecases(config.FLAG_EMBEDDINGS):
+ _, err = b.Embeddings(ctx, &pb.PredictOptions{Embeddings: "ping"})
+ return err
+ }
+ // A backend process that answers HealthCheck rarely fails only for TTS or
+ // transcription, so a real request adds little here.
+ return healthCheck(ctx, b)
+}
diff --git a/core/services/failover/prober_test.go b/core/services/failover/prober_test.go
new file mode 100644
index 000000000..8e66d2be3
--- /dev/null
+++ b/core/services/failover/prober_test.go
@@ -0,0 +1,269 @@
+package failover
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+ "io"
+ "log/slog"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "sync"
+
+ "github.com/mudler/LocalAI/core/config"
+ "github.com/mudler/LocalAI/pkg/grpc"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ "github.com/mudler/xlog"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ ggrpc "google.golang.org/grpc"
+)
+
+// captureXlog redirects the package-wide xlog logger to buf for the duration
+// of a test and restores the suite's default on cleanup. xlog exposes no
+// getter for the current logger, so this restores the same default the
+// entrypoint installs rather than the prior value (same pattern as
+// core/config/model_artifact_fallback_test.go).
+func captureXlog(buf *bytes.Buffer) {
+ handler := slog.NewTextHandler(buf, &slog.HandlerOptions{Level: slog.LevelWarn})
+ xlog.SetLogger(xlog.NewLoggerWithHandler(handler, xlog.LogLevelWarn))
+ DeferCleanup(func() {
+ xlog.SetLogger(xlog.NewLogger(xlog.LogLevel("info"), "text"))
+ })
+}
+
+type fakeUpstream struct {
+ mu sync.Mutex
+ srv *httptest.Server
+ models []string
+ status int
+ paths []string
+ auth string
+}
+
+func newFakeUpstream() *fakeUpstream {
+ u := &fakeUpstream{status: http.StatusOK}
+ u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ u.mu.Lock()
+ u.paths = append(u.paths, r.Method+" "+r.URL.Path)
+ u.auth = r.Header.Get("Authorization")
+ status, models := u.status, u.models
+ u.mu.Unlock()
+ _, _ = io.Copy(io.Discard, r.Body)
+ if status != http.StatusOK {
+ w.WriteHeader(status)
+ return
+ }
+ if r.URL.Path == "/v1/models" {
+ var data []map[string]string
+ for _, m := range models {
+ data = append(data, map[string]string{"id": m})
+ }
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data})
+ return
+ }
+ _, _ = w.Write([]byte(`{}`))
+ }))
+ return u
+}
+
+type fakeBackend struct {
+ grpc.Backend
+ healthy bool
+ predictErr error
+ predicted bool
+}
+
+func (b *fakeBackend) HealthCheck(context.Context) (bool, error) { return b.healthy, nil }
+func (b *fakeBackend) Predict(context.Context, *pb.PredictOptions, ...ggrpc.CallOption) (*pb.Reply, error) {
+ b.predicted = true
+ return &pb.Reply{}, b.predictErr
+}
+
+var _ = Describe("DefaultProber", func() {
+ var (
+ up *fakeUpstream
+ p *DefaultProber
+ ctx = context.Background()
+ )
+
+ BeforeEach(func() {
+ up = newFakeUpstream()
+ DeferCleanup(up.srv.Close)
+ p = NewProber(nil, os.Getenv)
+ })
+
+ proxied := func(name, upstreamModel string, usecases ...string) config.ModelConfig {
+ c := config.ModelConfig{Name: name, Backend: "cloud-proxy", KnownUsecaseStrings: usecases}
+ c.KnownUsecases = config.GetUsecasesFromYAML(usecases)
+ c.Proxy.UpstreamURL = up.srv.URL + "/v1/chat/completions"
+ c.Proxy.UpstreamModel = upstreamModel
+ return c
+ }
+
+ DescribeTable("UpstreamBase",
+ func(in, want string) {
+ got, err := UpstreamBase(in)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(got).To(Equal(want))
+ },
+ Entry("full endpoint", "https://h:8080/v1/chat/completions", "https://h:8080"),
+ Entry("path prefix", "https://h/api/v1/chat/completions", "https://h/api"),
+ Entry("bare host", "https://h", "https://h"),
+ Entry("bare host slash", "https://h/", "https://h"),
+ )
+
+ It("passes liveness when the upstream lists the model", func() {
+ up.models = []string{"big-llm"}
+ Expect(p.Liveness(ctx, proxied("argus-llm", "big-llm"), KindRemote, false)).To(Succeed())
+ Expect(up.paths).To(ContainElement("GET /v1/models"))
+ })
+
+ It("uses the target name when upstream_model is empty", func() {
+ up.models = []string{"argus-llm"}
+ Expect(p.Liveness(ctx, proxied("argus-llm", ""), KindRemote, false)).To(Succeed())
+ })
+
+ It("fails liveness when the model is not listed or the upstream errors", func() {
+ up.models = []string{"other"}
+ Expect(p.Liveness(ctx, proxied("argus-llm", ""), KindRemote, false)).To(MatchError(ContainSubstring("does not list")))
+ up.status = http.StatusServiceUnavailable
+ Expect(p.Liveness(ctx, proxied("argus-llm", ""), KindRemote, false)).To(MatchError(ContainSubstring("503")))
+ })
+
+ It("sends the API key as a bearer token", func() {
+ GinkgoT().Setenv("FAILOVER_PROBE_KEY", "sekret")
+ up.models = []string{"argus-llm"}
+ c := proxied("argus-llm", "")
+ c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY"
+ Expect(p.Liveness(ctx, c, KindRemote, false)).To(Succeed())
+ Expect(up.auth).To(Equal("Bearer sekret"))
+ })
+
+ It("does not follow a redirect, so the API key never leaves the upstream", func() {
+ GinkgoT().Setenv("FAILOVER_PROBE_KEY", "sekret")
+ other := newFakeUpstream()
+ DeferCleanup(other.srv.Close)
+ other.models = []string{"argus-llm"}
+ redirect := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ http.Redirect(w, r, other.srv.URL+r.URL.Path, http.StatusFound)
+ }))
+ DeferCleanup(redirect.Close)
+ c := proxied("argus-llm", "")
+ c.Proxy.UpstreamURL = redirect.URL + "/v1/chat/completions"
+ c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY"
+ c.Proxy.Provider = config.ProxyProviderAnthropic
+ Expect(p.Liveness(ctx, c, KindRemote, false)).To(MatchError(ContainSubstring("302")))
+ Expect(other.paths).To(BeEmpty())
+ })
+
+ It("warns once when a remote target has api_key_env set but no credential lookup is configured", func() {
+ var buf bytes.Buffer
+ captureXlog(&buf)
+
+ noLookup := NewProber(nil, nil)
+ c := proxied("argus-llm", "")
+ c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil)
+ Expect(err).ToNot(HaveOccurred())
+
+ // authorize surfaces the misconfiguration as an error on every call
+ // (liveness/inference must not silently proceed unauthenticated)...
+ Expect(noLookup.authorize(req, c)).To(HaveOccurred())
+ Expect(noLookup.authorize(req, c)).To(HaveOccurred())
+
+ // ...but only logs the warning once per api_key_env, so a chain with
+ // no lookup configured does not spam the log on every probe tick.
+ Expect(strings.Count(buf.String(), "no credential lookup is configured")).To(Equal(1))
+ Expect(buf.String()).To(ContainSubstring("FAILOVER_PROBE_KEY"))
+ })
+
+ It("does not warn when a lookup is configured, even if the env var itself is unset", func() {
+ var buf bytes.Buffer
+ captureXlog(&buf)
+
+ c := proxied("argus-llm", "")
+ c.Proxy.APIKeyEnv = "FAILOVER_PROBE_KEY_UNSET"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, up.srv.URL+"/v1/models", nil)
+ Expect(err).ToNot(HaveOccurred())
+
+ // p (from BeforeEach) has a real lookup (os.Getenv); the env var is
+ // simply unset, which is a different, already-reported failure mode.
+ Expect(p.authorize(req, c)).To(HaveOccurred())
+ Expect(buf.String()).ToNot(ContainSubstring("no credential lookup is configured"))
+ })
+
+ DescribeTable("remote inference hits the usecase endpoint",
+ func(usecase, path string) {
+ Expect(p.Inference(ctx, proxied("m", "", usecase), KindRemote, false)).To(Succeed())
+ Expect(up.paths).To(ContainElement("POST " + path))
+ },
+ Entry("chat", "chat", "/v1/chat/completions"),
+ Entry("embeddings", "embeddings", "/v1/embeddings"),
+ Entry("transcription", "transcript", "/v1/audio/transcriptions"),
+ Entry("tts", "tts", "/v1/audio/speech"),
+ )
+
+ It("uses HealthCheck for warm local liveness and Predict for local chat inference", func() {
+ b := &fakeBackend{healthy: true}
+ p = NewProber(func(config.ModelConfig) grpc.Backend { return b }, nil)
+ c := config.ModelConfig{Name: "gemma", Backend: "llama-cpp", KnownUsecaseStrings: []string{"chat"}}
+ c.KnownUsecases = config.GetUsecasesFromYAML(c.KnownUsecaseStrings)
+ Expect(p.Liveness(ctx, c, KindLocal, true)).To(Succeed())
+ b.healthy = false
+ Expect(p.Liveness(ctx, c, KindLocal, true)).To(HaveOccurred())
+ Expect(p.Inference(ctx, c, KindLocal, true)).To(Succeed())
+ Expect(b.predicted).To(BeTrue())
+ b.predictErr = errors.New("boom")
+ Expect(p.Inference(ctx, c, KindLocal, true)).To(HaveOccurred())
+ })
+
+ It("passes warm liveness for a target that is not loaded, and leaves recovery unconfirmed", func() {
+ // The warm preload loads the model; a probe that loaded it too would
+ // block until the load finished and then judge it on an expired ctx.
+ asked := 0
+ p = NewProber(func(config.ModelConfig) grpc.Backend { asked++; return nil }, nil)
+ for _, uc := range []string{"chat", "tts"} {
+ c := config.ModelConfig{Name: "gemma", Backend: "llama-cpp", KnownUsecaseStrings: []string{uc}}
+ c.KnownUsecases = config.GetUsecasesFromYAML(c.KnownUsecaseStrings)
+ Expect(p.Liveness(ctx, c, KindLocal, true)).To(Succeed())
+ Expect(p.Inference(ctx, c, KindLocal, true)).To(MatchError(ErrNotLoaded), uc)
+ }
+ Expect(asked).To(Equal(4))
+ })
+
+ It("passes cold local liveness without a model file and without loading", func() {
+ p = NewProber(func(config.ModelConfig) grpc.Backend {
+ Fail("cold liveness must not look up the backend")
+ return nil
+ }, nil)
+ // None of these files exist: a missing file says nothing about whether
+ // the target can serve (download on first use, dotted names, backends
+ // that need no file). Only a real request may trip a cold target.
+ for _, model := range []string{"weights.gguf", "Phi-3.5-mini", "org/some-hf-repo", ""} {
+ c := config.ModelConfig{Name: "cold", Backend: "llama-cpp"}
+ c.Model = model
+ Expect(p.Liveness(ctx, c, KindLocal, false)).To(Succeed(), model)
+ }
+ })
+})
+
+var _ = Describe("PrepareTarget", func() {
+ It("names the upstream model of a remote target the way the probe does", func() {
+ plain := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"}
+ PrepareTarget(&plain)
+ Expect(plain.Proxy.UpstreamModel).To(Equal("argus-llm"))
+
+ mapped := config.ModelConfig{Name: "argus-llm", Backend: "cloud-proxy"}
+ mapped.Proxy.UpstreamModel = "big-llm"
+ PrepareTarget(&mapped)
+ Expect(mapped.Proxy.UpstreamModel).To(Equal("big-llm"))
+
+ local := config.ModelConfig{Name: "gemma", Backend: "llama-cpp"}
+ PrepareTarget(&local)
+ Expect(local.Proxy.UpstreamModel).To(BeEmpty())
+ })
+})
diff --git a/core/services/failover/schedule.go b/core/services/failover/schedule.go
new file mode 100644
index 000000000..c851db835
--- /dev/null
+++ b/core/services/failover/schedule.go
@@ -0,0 +1,231 @@
+package failover
+
+import (
+ "context"
+ "errors"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+)
+
+// Run drives probes and dwell-based fail-back until ctx ends. Only this
+// scheduler goroutine calls Sync: onWarm callbacks run after the manager's
+// lock is released, and a concurrent Sync from elsewhere could reorder them.
+func (m *Manager) Run(ctx context.Context) {
+ ticker := time.NewTicker(time.Second)
+ defer ticker.Stop()
+ m.Tick(ctx)
+ for {
+ select {
+ case <-ctx.Done():
+ m.close()
+ return
+ case <-ticker.C:
+ m.Tick(ctx)
+ // Publishes are fire-and-forget, so a frontend that missed one
+ // (restart, dropped message) converges within ten seconds. Pins
+ // are not republished: every frontend re-reads the shared set.
+ m.ticks++
+ if m.ticks%10 == 0 {
+ m.ReconcilePins()
+ if m.IsLeader() {
+ m.Republish()
+ }
+ }
+ }
+ }
+}
+
+// Tick runs one pass: sync configs, start due probes, recompute chains. It is
+// exported so tests can drive the manager without a real ticker. Like Run, it
+// must only be called from the scheduler goroutine (see Run's comment on Sync).
+//
+// Tick does not wait for the probes it starts: one slow target (a probe that
+// hangs until its timeout) must not delay probing and fail-back of every
+// other chain. Each probe applies its own result, and a target whose probe is
+// still running is skipped until it ends.
+//
+// With a leader gate, only the leader probes and decides chains; followers
+// still recompute, which only moves chains that have not yet adopted a
+// leader decision.
+func (m *Manager) Tick(ctx context.Context) {
+ m.Sync()
+ m.mu.Lock()
+ gate := m.gate // SetLeaderGate may replace it after construction
+ m.mu.Unlock()
+ if gate == nil {
+ m.lead(ctx)
+ return
+ }
+ if gate(ctx, func() { m.lead(ctx) }) {
+ return
+ }
+ m.mu.Lock()
+ m.leader = false
+ m.mu.Unlock()
+ m.Reevaluate()
+}
+
+// lead is the leader's share of a tick.
+func (m *Manager) lead(ctx context.Context) {
+ m.mu.Lock()
+ became := !m.leader
+ m.leader = true
+ if became {
+ // The previous leader owned the warm-set callback's effects;
+ // deliver the set again so this frontend takes them over.
+ m.warmPending = true
+ }
+ warm, deliver := m.takeWarmLocked()
+ m.unlockAndFlush()
+ if deliver && m.onWarm != nil {
+ m.onWarm(warm)
+ }
+ if became {
+ // Followers hold the old leader's view; send ours at once instead of
+ // letting them wait for the periodic republish.
+ m.Republish()
+ }
+ for _, j := range m.dueProbes() {
+ m.probes.Add(1)
+ go func(j probeJob) {
+ defer m.probes.Done()
+ m.runProbe(ctx, j)
+ }(j)
+ }
+ m.Reevaluate()
+}
+
+// waitProbes waits for the probes started so far. Tests use it to see a
+// tick's results; the scheduler never waits.
+func (m *Manager) waitProbes() { m.probes.Wait() }
+
+type probeJob struct {
+ target string
+ cfg config.ModelConfig
+ kind Kind
+ warm bool
+ inference bool
+ timeout time.Duration
+}
+
+func (m *Manager) dueProbes() []probeJob {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ now := m.clock.Now()
+ var jobs []probeJob
+ for _, ts := range m.targets {
+ if ts.probing {
+ continue
+ }
+ interval := ts.params.ProbeInterval()
+ inference := false
+ switch ts.state {
+ case StateMissing:
+ continue
+ case StateHealthy:
+ // A served request is as good as a liveness probe.
+ if now.Sub(ts.lastActivity) < interval || now.Sub(ts.lastProbe) < interval {
+ continue
+ }
+ case StateDown:
+ if ts.cold() {
+ // Loading a cold model only to probe it could evict others.
+ if now.Sub(ts.downSince) >= ts.params.MinDwell() {
+ m.setTargetLocked(ts, StateHealthy, ReasonRecovery, "")
+ m.recomputeForLocked(ts.name)
+ }
+ continue
+ }
+ if now.Sub(ts.lastProbe) < interval {
+ continue
+ }
+ case StateRecovering:
+ if ts.cold() || now.Sub(ts.lastProbe) < interval {
+ continue
+ }
+ inference = true
+ }
+ if m.prober == nil {
+ continue
+ }
+ cfg, ok := m.lookupTarget(ts.name)
+ if !ok {
+ continue
+ }
+ ts.lastProbe = now
+ ts.probing = true
+ jobs = append(jobs, probeJob{
+ target: ts.name, cfg: cfg, kind: ts.kind, warm: ts.warm,
+ inference: inference, timeout: ts.params.ProbeTimeout(),
+ })
+ }
+ return jobs
+}
+
+func (m *Manager) runProbe(ctx context.Context, j probeJob) {
+ pctx, cancel := context.WithTimeout(ctx, j.timeout)
+ defer cancel()
+ var err error
+ if j.inference {
+ err = m.prober.Inference(pctx, j.cfg, j.kind, j.warm)
+ } else {
+ err = m.prober.Liveness(pctx, j.cfg, j.kind, j.warm)
+ }
+ if ctx.Err() != nil {
+ m.endProbe(j.target)
+ return // shutting down: a cancelled probe says nothing about the target
+ }
+ m.applyProbe(j, err)
+}
+
+func (m *Manager) endProbe(target string) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if ts := m.targets[target]; ts != nil {
+ ts.probing = false
+ }
+}
+
+func (m *Manager) applyProbe(j probeJob, err error) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ ts := m.targets[j.target]
+ if ts == nil {
+ return
+ }
+ ts.probing = false
+ if ts.state == StateMissing {
+ return
+ }
+ if errors.Is(err, ErrNotLoaded) {
+ // Nothing running to confirm recovery against: judge the target like
+ // a cold one, by real requests once min_dwell has passed.
+ if ts.state == StateRecovering && m.clock.Now().Sub(ts.downSince) >= ts.params.MinDwell() {
+ m.setTargetLocked(ts, StateHealthy, ReasonRecovery, "")
+ m.recomputeForLocked(ts.name)
+ }
+ return
+ }
+ if err != nil {
+ if ts.state == StateDown {
+ ts.lastError = err.Error()
+ } else {
+ m.recordFailureLocked(ts, err.Error())
+ }
+ m.recomputeForLocked(ts.name)
+ return
+ }
+ switch ts.state {
+ case StateHealthy:
+ ts.lastActivity = m.clock.Now()
+ ts.failures = nil
+ case StateDown:
+ m.setTargetLocked(ts, StateRecovering, ReasonRecovery, "")
+ case StateRecovering:
+ if j.inference {
+ m.recordPassLocked(ts)
+ }
+ }
+ m.recomputeForLocked(ts.name)
+}
diff --git a/core/services/failover/schedule_test.go b/core/services/failover/schedule_test.go
new file mode 100644
index 000000000..a05e2f982
--- /dev/null
+++ b/core/services/failover/schedule_test.go
@@ -0,0 +1,237 @@
+package failover
+
+import (
+ "context"
+ "slices"
+ "sync"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+type probeCall struct {
+ target string
+ inference bool
+}
+
+type fakeProber struct {
+ mu sync.Mutex
+ calls []probeCall
+ fail map[string]error // target -> error returned by every probe
+ block map[string]chan struct{} // target -> probes wait until it is closed
+}
+
+func (p *fakeProber) record(target string, inference bool) error {
+ p.mu.Lock()
+ p.calls = append(p.calls, probeCall{target, inference})
+ err, block := p.fail[target], p.block[target]
+ p.mu.Unlock()
+ if block != nil {
+ <-block
+ }
+ return err
+}
+func (p *fakeProber) Liveness(_ context.Context, c config.ModelConfig, _ Kind, _ bool) error {
+ return p.record(c.Name, false)
+}
+func (p *fakeProber) Inference(_ context.Context, c config.ModelConfig, _ Kind, _ bool) error {
+ return p.record(c.Name, true)
+}
+func (p *fakeProber) take() []probeCall {
+ p.mu.Lock()
+ defer p.mu.Unlock()
+ out := p.calls
+ p.calls = nil
+ return out
+}
+
+var _ = Describe("Manager probes", func() {
+ var (
+ clock *fakeClock
+ src *fakeSource
+ prober *fakeProber
+ m *Manager
+ ctx = context.Background()
+ )
+
+ BeforeEach(func() {
+ clock = newFakeClock()
+ prober = &fakeProber{fail: map[string]error{}, block: map[string]chan struct{}{}}
+ src = newFakeSource(remote("a"), local("b"), local("cold"),
+ chainCfg("chain", nil, t("a"), warmT("b")))
+ m = New(src, WithClock(clock), WithProber(prober))
+ })
+
+ // tick runs one scheduler pass and waits for the probes it started, so
+ // each spec sees their results.
+ tick := func() {
+ m.Tick(ctx)
+ m.waitProbes()
+ }
+
+ It("probes idle targets on the first tick and not again before the interval", func() {
+ tick()
+ Expect(prober.take()).To(ConsistOf(probeCall{"a", false}, probeCall{"b", false}))
+ clock.Advance(5 * time.Second)
+ tick()
+ Expect(prober.take()).To(BeEmpty())
+ })
+
+ It("skips the liveness probe for a target with recent traffic", func() {
+ tick()
+ prober.take()
+ clock.Advance(14 * time.Second)
+ m.ReportSuccess("a")
+ clock.Advance(2 * time.Second)
+ tick()
+ Expect(prober.take()).To(ConsistOf(probeCall{"b", false}))
+ })
+
+ It("trips a target whose liveness probe fails", func() {
+ prober.fail["a"] = errBoom
+ tick()
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ Expect(st.Active).To(Equal("b"))
+ })
+
+ It("recovers through liveness, then inference probes, then fails back after dwell", func() {
+ prober.fail["a"] = errBoom
+ tick()
+ delete(prober.fail, "a")
+ prober.take()
+
+ clock.Advance(15 * time.Second)
+ tick() // liveness passes: recovering
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateRecovering))
+
+ for i := 0; i < 3; i++ {
+ clock.Advance(15 * time.Second)
+ tick()
+ }
+ calls := prober.take()
+ Expect(calls).To(ContainElement(probeCall{"a", true}))
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ Expect(st.Active).To(Equal("a"), "60s min_dwell passed during the 4 ticks")
+ })
+
+ It("sends a recovering target back down when an inference probe fails", func() {
+ m.ReportFailure("a", errBoom)
+ clock.Advance(15 * time.Second)
+ tick() // liveness passes: recovering
+ prober.fail["a"] = errBoom
+ clock.Advance(15 * time.Second)
+ tick()
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ })
+
+ It("never probes a down cold target and restores it after min_dwell", func() {
+ src.Put(chainCfg("chain", nil, t("cold"), warmT("b")))
+ m.Sync()
+ m.ReportFailure("cold", errBoom)
+ prober.take()
+ clock.Advance(30 * time.Second)
+ tick()
+ for _, c := range prober.take() {
+ Expect(c.target).ToNot(Equal("cold"))
+ }
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateDown))
+ clock.Advance(31 * time.Second)
+ tick()
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ })
+
+ It("probes a target shared by two chains once per tick", func() {
+ src.Put(chainCfg("chain2", nil, t("a"), warmT("b")))
+ tick()
+ calls := prober.take()
+ n := 0
+ for _, c := range calls {
+ if c.target == "a" {
+ n++
+ }
+ }
+ Expect(n).To(Equal(1))
+ })
+
+ It("does not hold other targets' probes behind a slow one", func() {
+ src.Put(remote("c"))
+ src.Put(chainCfg("chain2", nil, t("c"), warmT("b")))
+ release := make(chan struct{})
+ DeferCleanup(func() { close(release); m.waitProbes() })
+ prober.block["a"] = release
+ m.ReportFailure("c", errBoom)
+ prober.take()
+
+ m.Tick(ctx) // a hangs; c's liveness still runs and starts its recovery
+ Eventually(func() TargetState {
+ st, _ := m.ChainStatus("chain2")
+ return st.Targets[0].State
+ }).Should(Equal(StateRecovering))
+
+ clock.Advance(15 * time.Second)
+ m.Tick(ctx) // a is still in flight: no second probe for it
+ Eventually(func() []probeCall { return prober.take() }).Should(ContainElement(probeCall{"c", true}))
+ })
+
+ It("does not probe a target again while its probe is in flight", func() {
+ release := make(chan struct{})
+ prober.block["a"] = release
+ m.Tick(ctx)
+ Eventually(func() []probeCall {
+ prober.mu.Lock()
+ defer prober.mu.Unlock()
+ return slices.Clone(prober.calls)
+ }).Should(ContainElement(probeCall{"a", false}))
+ for range 3 {
+ clock.Advance(15 * time.Second)
+ m.Tick(ctx)
+ }
+ close(release)
+ m.waitProbes()
+ n := 0
+ for _, c := range prober.take() {
+ if c.target == "a" {
+ n++
+ }
+ }
+ Expect(n).To(Equal(1))
+ })
+
+ It("restores a warm target that is not loaded like a cold one, after min_dwell", func() {
+ m.ReportFailure("b", errBoom)
+ prober.fail["b"] = ErrNotLoaded
+ prober.take()
+ clock.Advance(15 * time.Second)
+ prober.fail["b"] = nil
+ tick() // liveness passes: recovering
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Targets[1].State).To(Equal(StateRecovering))
+ prober.fail["b"] = ErrNotLoaded
+ clock.Advance(15 * time.Second)
+ tick() // nothing to confirm against yet, and no trip
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Targets[1].State).To(Equal(StateRecovering))
+ clock.Advance(31 * time.Second)
+ tick()
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Targets[1].State).To(Equal(StateHealthy))
+ })
+
+ It("closes subscriptions when Run stops", func() {
+ events, _ := m.Subscribe(1)
+ rctx, cancel := context.WithCancel(ctx)
+ done := make(chan struct{})
+ go func() { m.Run(rctx); close(done) }()
+ cancel()
+ Eventually(done).Should(BeClosed())
+ Eventually(events).Should(BeClosed())
+ })
+})
diff --git a/core/services/failover/statesync.go b/core/services/failover/statesync.go
new file mode 100644
index 000000000..14e9de431
--- /dev/null
+++ b/core/services/failover/statesync.go
@@ -0,0 +1,273 @@
+package failover
+
+import (
+ "context"
+ "slices"
+ "time"
+
+ "github.com/mudler/xlog"
+)
+
+// TargetSnapshot is one target's health as shared between frontends.
+type TargetSnapshot struct {
+ Target string `json:"target"`
+ State TargetState `json:"state"`
+ Reason Reason `json:"reason"`
+ Error string `json:"error,omitempty"`
+ ConsecutiveOK int `json:"consecutive_ok"`
+ Since time.Time `json:"since"`
+ // Origin is the publishing manager, so it can drop its own echoes.
+ Origin string `json:"origin,omitempty"`
+}
+
+// ChainSnapshot is one chain's active target as decided by the leader.
+type ChainSnapshot struct {
+ Chain string `json:"chain"`
+ Active string `json:"active"`
+ ActiveSince time.Time `json:"active_since"`
+ State ChainState `json:"state"`
+ Reason Reason `json:"reason"`
+}
+
+// StateSync shares failover state between frontends. Implementations may
+// deliver a publish back to the publisher synchronously (NATS echoes), so the
+// manager never calls it while holding its lock.
+type StateSync interface {
+ PublishTarget(TargetSnapshot)
+ PublishChain(ChainSnapshot)
+ SetPin(chain, target string) error
+ ClearPin(chain string) error
+ Pins() map[string]string
+}
+
+// LeaderGate runs fn only on the one frontend that holds leadership and
+// reports whether it did. Probing and chain decisions happen on the leader
+// only, so N frontends do not probe every target N times or disagree on the
+// active target.
+type LeaderGate func(ctx context.Context, fn func()) bool
+
+// WithLeaderGate makes the manager probe and decide chains only while the gate
+// grants leadership. Without it the manager is always the leader.
+func WithLeaderGate(g LeaderGate) Option { return func(m *Manager) { m.gate = g } }
+
+// SetLeaderGate is WithLeaderGate for a manager that already exists: the
+// application builds the manager before distributed init, where the gate's
+// database becomes known. Leadership is reset to match New, so the first tick
+// that wins the gate counts as becoming leader (warm set redelivered, state
+// republished at once).
+func (m *Manager) SetLeaderGate(g LeaderGate) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ m.gate = g
+ m.leader = g == nil
+}
+
+// SetStateSync attaches the sync layer and hydrates pins from it. Pins are
+// read outside the lock because the store may need I/O.
+func (m *Manager) SetStateSync(s StateSync) {
+ m.mu.Lock()
+ m.sync = s
+ m.mu.Unlock()
+ if s == nil {
+ return
+ }
+ m.ReconcilePins()
+}
+
+// ReconcilePins makes this frontend's pins match the shared pin set. Pins
+// normally arrive as deltas, but a re-hydrate of the shared set (after a NATS
+// reconnect, or a missed delta repaired from the DB) changes it without
+// delivering them; a frontend left with a stale pin would serve it while the
+// others do not. The scheduler runs this periodically on every frontend.
+func (m *Manager) ReconcilePins() {
+ m.mu.Lock()
+ s := m.sync
+ m.mu.Unlock()
+ if s == nil {
+ return
+ }
+ // Read outside the lock: the store may need I/O.
+ want := s.Pins()
+ m.mu.Lock()
+ var stale []string
+ for chain := range m.pins {
+ if _, ok := want[chain]; !ok {
+ stale = append(stale, chain)
+ }
+ }
+ m.mu.Unlock()
+ for _, chain := range stale {
+ m.ApplyPin(chain, "")
+ }
+ for chain, target := range want {
+ m.ApplyPin(chain, target)
+ }
+}
+
+// IsLeader reports whether this manager probed and decided chains at its last
+// tick. A standalone manager (no gate) is always the leader.
+func (m *Manager) IsLeader() bool {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ return m.leader
+}
+
+// unlockAndFlush releases the lock and then runs the publishes queued while
+// it was held: the sync layer may call straight back into Apply*, which takes
+// the lock again.
+func (m *Manager) unlockAndFlush() {
+ pending := m.pending
+ m.pending = nil
+ m.mu.Unlock()
+ for _, f := range pending {
+ f()
+ }
+}
+
+func (m *Manager) targetSnapshotLocked(ts *targetState) TargetSnapshot {
+ since := ts.since
+ if ts.state == StateDown {
+ since = ts.downSince
+ }
+ return TargetSnapshot{
+ Target: ts.name, State: ts.state, Reason: ts.reason, Error: ts.lastError,
+ ConsecutiveOK: ts.consecutiveOK, Since: since, Origin: m.id,
+ }
+}
+
+func (m *Manager) chainSnapshotLocked(ch *chainState, reason Reason) ChainSnapshot {
+ return ChainSnapshot{
+ Chain: ch.name, Active: ch.targets[ch.active], ActiveSince: ch.activeSince,
+ State: ch.state, Reason: reason,
+ }
+}
+
+// queuePublishTargetLocked shares a local target transition. Missing is a fact
+// about this frontend's config, not about the target, so it is never shared.
+func (m *Manager) queuePublishTargetLocked(ts *targetState, from TargetState) {
+ if m.sync == nil || m.applying || ts.state == StateMissing || from == StateMissing {
+ return
+ }
+ s, snap := m.sync, m.targetSnapshotLocked(ts)
+ m.pending = append(m.pending, func() { s.PublishTarget(snap) })
+}
+
+func (m *Manager) queuePublishChainLocked(ch *chainState, reason Reason) {
+ if m.sync == nil || !m.leader {
+ return
+ }
+ s, snap := m.sync, m.chainSnapshotLocked(ch, reason)
+ m.pending = append(m.pending, func() { s.PublishChain(snap) })
+}
+
+// ApplyTarget takes a target state published by another frontend. The echo
+// of an own publish is dropped: this manager already holds that state or a
+// newer one, and publishes are snapshotted when queued, so an echo can arrive
+// after a later local transition (down -> recovering -> healthy under one
+// lock queues two) and would roll the target back.
+func (m *Manager) ApplyTarget(s TargetSnapshot) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ if s.Origin != "" && s.Origin == m.id {
+ return
+ }
+ ts := m.targetLocked(s.Target)
+ if ts == nil || ts.state == StateMissing || s.State == StateMissing {
+ return
+ }
+ if ts.state == s.State {
+ return
+ }
+ m.applying = true
+ m.setTargetLocked(ts, s.State, s.Reason, s.Error)
+ m.applying = false
+ // Keep the publisher's clock, so dwell timers (cold recovery, fail-back)
+ // run from when the target actually changed, not from when we heard.
+ if !s.Since.IsZero() {
+ ts.since = s.Since
+ if s.State == StateDown {
+ ts.downSince = s.Since
+ }
+ }
+ ts.consecutiveOK = s.ConsecutiveOK
+ if s.Error != "" {
+ ts.lastError = s.Error
+ }
+ m.recomputeForLocked(ts.name)
+}
+
+// ApplyChain adopts the leader's decision for a chain. The leader ignores it:
+// it is the source of these decisions, and a late publish from a previous
+// leader must not undo its own.
+func (m *Manager) ApplyChain(s ChainSnapshot) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ if m.leader {
+ return
+ }
+ ch := m.chainLocked(s.Chain)
+ if ch == nil {
+ return
+ }
+ next := slices.Index(ch.targets, s.Active)
+ if next < 0 {
+ // The frontends disagree on the chain's targets while a config
+ // change propagates; keep deciding locally until they agree.
+ xlog.Debug("failover: ignoring chain state for an unknown target", "chain", s.Chain, "target", s.Active)
+ return
+ }
+ prev := ch.active
+ changed := next != prev || s.State != ch.state
+ ch.active, ch.activeSince, ch.state, ch.adopted = next, s.ActiveSince, s.State, true
+ if changed {
+ m.emitLocked(Event{Type: EventChainSwitched, Chain: ch.name, From: ch.targets[prev], To: ch.targets[next], State: string(s.State), Reason: s.Reason, At: m.clock.Now()})
+ }
+}
+
+// ApplyPin sets (target != "") or clears a pin set on any frontend. A pin for
+// a chain or target this frontend does not know yet is kept and applied by
+// syncLocked once the config catches up.
+func (m *Manager) ApplyPin(chain, target string) {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ if target == "" {
+ delete(m.pins, chain)
+ } else {
+ m.pins[chain] = target
+ }
+ ch := m.chainLocked(chain)
+ if ch == nil {
+ return
+ }
+ if target != "" && !slices.Contains(ch.targets, target) {
+ xlog.Debug("failover: deferring pin to a target not in the chain yet", "chain", chain, "target", target)
+ return
+ }
+ if ch.pinned == target {
+ return
+ }
+ ch.pinned = target
+ m.recomputeLocked(ch, ReasonManual)
+}
+
+// Republish sends every target and chain state, so frontends that missed a
+// publish (joined late, dropped a message) converge. Only the leader's view
+// is authoritative, so followers do nothing.
+func (m *Manager) Republish() {
+ m.mu.Lock()
+ defer m.unlockAndFlush()
+ if m.sync == nil || !m.leader {
+ return
+ }
+ s := m.sync
+ for _, ts := range m.targets {
+ if ts.state == StateMissing {
+ continue
+ }
+ snap := m.targetSnapshotLocked(ts)
+ m.pending = append(m.pending, func() { s.PublishTarget(snap) })
+ }
+ for _, ch := range m.chains {
+ m.queuePublishChainLocked(ch, ReasonInitial)
+ }
+}
diff --git a/core/services/failover/statesync_test.go b/core/services/failover/statesync_test.go
new file mode 100644
index 000000000..6021636de
--- /dev/null
+++ b/core/services/failover/statesync_test.go
@@ -0,0 +1,345 @@
+package failover
+
+import (
+ "context"
+ "sync"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+// loopSync is an in-process StateSync that delivers every publish to all
+// managers synchronously, including the publisher (like NATS echo).
+type loopSync struct {
+ mu sync.Mutex
+ peers []*Manager
+ pins map[string]string
+}
+
+func (l *loopSync) add(m *Manager) { l.mu.Lock(); l.peers = append(l.peers, m); l.mu.Unlock() }
+func (l *loopSync) each(f func(*Manager)) {
+ l.mu.Lock()
+ ps := append([]*Manager(nil), l.peers...)
+ l.mu.Unlock()
+ for _, p := range ps {
+ f(p)
+ }
+}
+func (l *loopSync) PublishTarget(s TargetSnapshot) { l.each(func(m *Manager) { m.ApplyTarget(s) }) }
+func (l *loopSync) PublishChain(s ChainSnapshot) { l.each(func(m *Manager) { m.ApplyChain(s) }) }
+func (l *loopSync) SetPin(c, t string) error {
+ l.mu.Lock()
+ l.pins[c] = t
+ l.mu.Unlock()
+ l.each(func(m *Manager) { m.ApplyPin(c, t) })
+ return nil
+}
+func (l *loopSync) ClearPin(c string) error {
+ l.mu.Lock()
+ delete(l.pins, c)
+ l.mu.Unlock()
+ l.each(func(m *Manager) { m.ApplyPin(c, "") })
+ return nil
+}
+func (l *loopSync) Pins() map[string]string {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ out := map[string]string{}
+ for k, v := range l.pins {
+ out[k] = v
+ }
+ return out
+}
+
+// failingPinSync is a StateSync whose pin writes fail (the DB is down).
+type failingPinSync struct{ *loopSync }
+
+func (failingPinSync) SetPin(string, string) error { return errBoom }
+func (failingPinSync) ClearPin(string) error { return errBoom }
+
+var _ = Describe("Manager state sync", func() {
+ var (
+ clock *fakeClock
+ src *fakeSource
+ bus *loopSync
+ a, b *Manager
+ leaderIsA bool
+ gateFor func(isA bool) LeaderGate
+ ctx = context.Background()
+ )
+
+ BeforeEach(func() {
+ clock = newFakeClock()
+ src = newFakeSource(remote("x"), local("y"), chainCfg("chain", nil, t("x"), t("y")))
+ bus = &loopSync{pins: map[string]string{}}
+ leaderIsA = true
+ gateFor = func(isA bool) LeaderGate {
+ return func(_ context.Context, fn func()) bool {
+ if isA != leaderIsA {
+ return false
+ }
+ fn()
+ return true
+ }
+ }
+ a = New(src, WithClock(clock), WithLeaderGate(gateFor(true)))
+ b = New(src, WithClock(clock), WithLeaderGate(gateFor(false)))
+ bus.add(a)
+ bus.add(b)
+ a.SetStateSync(bus)
+ b.SetStateSync(bus)
+ a.Tick(ctx)
+ b.Tick(ctx)
+ })
+
+ It("echo of own publish is a no-op and emits one event", func() {
+ events, cancel := a.Subscribe(16)
+ defer cancel()
+ a.ReportFailure("x", errBoom)
+ n := 0
+ for _, e := range drain(events) {
+ if e.Type == EventTargetState && e.Target == "x" {
+ n++
+ }
+ }
+ Expect(n).To(Equal(1))
+ })
+
+ It("does not let the echo of its own earlier publish undo a newer local state", func() {
+ // One recovery probe: a success moves x down -> recovering -> healthy
+ // under one lock, which queues two publishes. Their echoes arrive
+ // after x is already healthy here.
+ src.Put(chainCfg("chain", &config.FailoverConfig{Recovery: config.FailoverRecovery{Probes: 1}}, t("x"), t("y")))
+ a.Sync()
+ b.Sync()
+ a.ReportFailure("x", errBoom)
+ st, _ := a.ChainStatus("chain")
+ Expect(st.Active).To(Equal("y"))
+
+ clock.Advance(10 * time.Minute) // past any dwell
+ a.ReportSuccess("x")
+
+ st, _ = a.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ Expect(st.Active).To(Equal("x"), "the leader must stay failed back to x")
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Targets[0].State).To(Equal(StateHealthy))
+ Expect(st.Active).To(Equal("x"))
+ })
+
+ It("a trip on one frontend is skipped by the other's plan", func() {
+ b.ReportFailure("x", errBoom)
+ att, err := a.Plan("chain")
+ Expect(err).ToNot(HaveOccurred())
+ Expect(att.Target()).To(Equal("y"))
+ })
+
+ It("followers adopt the leader's chain state and emit the switch", func() {
+ events, cancel := b.Subscribe(16)
+ defer cancel()
+ a.ReportFailure("x", errBoom) // leader recomputes and publishes chain state
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Active).To(Equal("y"))
+ var sw []Event
+ for _, e := range drain(events) {
+ if e.Type == EventChainSwitched {
+ sw = append(sw, e)
+ }
+ }
+ Expect(sw).ToNot(BeEmpty())
+ })
+
+ It("a pin on one frontend applies on all", func() {
+ Expect(b.Pin("chain", "y")).To(Succeed())
+ st, _ := a.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ Expect(a.Unpin("chain")).To(Succeed())
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ })
+
+ It("hydrates pins when the sync is attached", func() {
+ bus.pins["chain"] = "y"
+ c := New(src, WithClock(clock), WithLeaderGate(gateFor(false)))
+ c.SetStateSync(bus)
+ st, _ := c.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ })
+
+ It("ReconcilePins converges a frontend that missed a pin and an unpin", func() {
+ // The shared pin set changes without B hearing the delta, as after a
+ // NATS reconnect whose re-hydrate fires no OnApply.
+ bus.mu.Lock()
+ bus.pins["chain"] = "y"
+ bus.mu.Unlock()
+ b.ReconcilePins()
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ Expect(st.Active).To(Equal("y"))
+
+ bus.mu.Lock()
+ delete(bus.pins, "chain")
+ bus.mu.Unlock()
+ b.ReconcilePins()
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ })
+
+ It("ReconcilePins leaves a pin for a chain this frontend does not know yet", func() {
+ bus.mu.Lock()
+ bus.pins["later"] = "y"
+ bus.mu.Unlock()
+ b.ReconcilePins()
+ src.Put(chainCfg("later", nil, t("x"), t("y")))
+ b.Sync()
+ st, ok := b.ChainStatus("later")
+ Expect(ok).To(BeTrue())
+ Expect(st.Pinned).ToNot(BeNil())
+ })
+
+ It("rolls a pin back when the shared write fails", func() {
+ m := New(src, WithClock(clock))
+ m.SetStateSync(failingPinSync{bus})
+ Expect(m.Pin("chain", "y")).To(MatchError(errBoom))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ })
+
+ It("restores the previous pin when a re-pin or an unpin fails to share", func() {
+ m := New(src, WithClock(clock))
+ m.SetStateSync(bus)
+ Expect(m.Pin("chain", "y")).To(Succeed())
+ m.SetStateSync(failingPinSync{bus})
+
+ Expect(m.Pin("chain", "x")).To(MatchError(errBoom))
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+
+ Expect(m.Unpin("chain")).To(MatchError(errBoom))
+ st, _ = m.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ Expect(st.Active).To(Equal("y"))
+ })
+
+ It("SetLeaderGate gates a manager built without one", func() {
+ // Production builds the manager before distributed init, so the gate
+ // arrives through the setter rather than the option.
+ p := &fakeProber{fail: map[string]error{}}
+ m := New(src, WithClock(clock), WithProber(p))
+ Expect(m.IsLeader()).To(BeTrue())
+ m.SetLeaderGate(gateFor(false))
+ Expect(m.IsLeader()).To(BeFalse(), "a gated manager is not the leader until the gate grants it")
+ m.Tick(ctx)
+ Consistently(func() int { return len(p.take()) }, 200*time.Millisecond).Should(Equal(0), "a follower must not probe")
+ })
+
+ It("only the leader probes", func() {
+ pa, pb := &fakeProber{fail: map[string]error{}}, &fakeProber{fail: map[string]error{}}
+ a = New(src, WithClock(clock), WithProber(pa), WithLeaderGate(gateFor(true)))
+ b = New(src, WithClock(clock), WithProber(pb), WithLeaderGate(gateFor(false)))
+ a.SetStateSync(bus)
+ b.SetStateSync(bus)
+ a.Tick(ctx)
+ b.Tick(ctx)
+ Eventually(func() int { return len(pa.take()) }).Should(BeNumerically(">", 0))
+ Consistently(func() int { return len(pb.take()) }, 200*time.Millisecond).Should(Equal(0))
+ Expect(a.IsLeader()).To(BeTrue())
+ Expect(b.IsLeader()).To(BeFalse())
+ })
+
+ It("new leader keeps activeSince across a leadership move", func() {
+ a.ReportFailure("x", errBoom)
+ before, _ := b.ChainStatus("chain")
+ leaderIsA = false
+ clock.Advance(5 * time.Second)
+ a.Tick(ctx)
+ b.Tick(ctx)
+ after, _ := b.ChainStatus("chain")
+ Expect(after.ActiveSince).To(Equal(before.ActiveSince))
+ Expect(b.IsLeader()).To(BeTrue())
+ })
+
+ It("republish sends every target and chain", func() {
+ c := New(src, WithClock(clock), WithLeaderGate(gateFor(false)))
+ bus.add(c)
+ c.SetStateSync(bus)
+ a.ReportFailure("x", errBoom)
+ a.Republish()
+ st, _ := c.ChainStatus("chain")
+ Expect(st.Active).To(Equal("y"))
+ })
+
+ It("keeps a pin for a chain it does not know yet and applies it when the chain appears", func() {
+ b.ApplyPin("later", "y")
+ _, ok := b.ChainStatus("later")
+ Expect(ok).To(BeFalse())
+ src.Put(chainCfg("later", nil, t("x"), t("y")))
+ b.Sync()
+ st, ok := b.ChainStatus("later")
+ Expect(ok).To(BeTrue())
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ Expect(st.Active).To(Equal("y"))
+ })
+
+ It("re-applies a shared pin when the chain is removed and re-added", func() {
+ Expect(a.Pin("chain", "y")).To(Succeed())
+ src.Delete("chain")
+ b.Sync()
+ _, ok := b.ChainStatus("chain")
+ Expect(ok).To(BeFalse())
+ src.Put(chainCfg("chain", nil, t("x"), t("y")))
+ b.Sync()
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ })
+
+ It("applies a deferred pin once its target joins the chain", func() {
+ src.Put(local("z"))
+ b.ApplyPin("chain", "z")
+ st, _ := b.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ src.Put(chainCfg("chain", nil, t("x"), t("y"), t("z")))
+ b.Sync()
+ st, _ = b.ChainStatus("chain")
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("z"))
+ })
+
+ It("hydrates a pin for an unknown chain and applies it when the chain appears", func() {
+ bus.pins["later"] = "y"
+ c := New(src, WithClock(clock), WithLeaderGate(gateFor(false)))
+ c.SetStateSync(bus)
+ src.Put(chainCfg("later", nil, t("x"), t("y")))
+ c.Sync()
+ st, ok := c.ChainStatus("later")
+ Expect(ok).To(BeTrue())
+ Expect(st.Pinned).ToNot(BeNil())
+ Expect(*st.Pinned).To(Equal("y"))
+ })
+
+ It("standalone, a removed chain drops its pin", func() {
+ m := New(src, WithClock(clock))
+ Expect(m.Pin("chain", "y")).To(Succeed())
+ src.Delete("chain")
+ m.Sync()
+ src.Put(chainCfg("chain", nil, t("x"), t("y")))
+ m.Sync()
+ st, _ := m.ChainStatus("chain")
+ Expect(st.Pinned).To(BeNil())
+ })
+
+ It("standalone manager (no sync, no gate) is always leader", func() {
+ m := New(src, WithClock(clock))
+ m.Tick(ctx)
+ Expect(m.IsLeader()).To(BeTrue())
+ })
+})
diff --git a/core/services/failover/trace.go b/core/services/failover/trace.go
new file mode 100644
index 000000000..8eb6618f1
--- /dev/null
+++ b/core/services/failover/trace.go
@@ -0,0 +1,23 @@
+package failover
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/mudler/LocalAI/core/trace"
+)
+
+// RecordAttemptTrace shows in the Traces UI why a target was skipped.
+func RecordAttemptTrace(enabled bool, chain, target string, err error) {
+ if !enabled || err == nil {
+ return
+ }
+ trace.RecordBackendTrace(trace.BackendTrace{
+ Timestamp: time.Now(),
+ Type: trace.BackendTraceFailover,
+ ModelName: target,
+ Summary: fmt.Sprintf("failover chain %s: %s failed, trying the next target", chain, target),
+ Error: err.Error(),
+ Data: map[string]any{"chain": chain},
+ })
+}
diff --git a/core/services/failover/types.go b/core/services/failover/types.go
new file mode 100644
index 000000000..428123c00
--- /dev/null
+++ b/core/services/failover/types.go
@@ -0,0 +1,106 @@
+// Package failover serves a model name from an ordered chain of target
+// models, moving to the next target when one fails and back when it
+// recovers.
+package failover
+
+import (
+ "slices"
+ "time"
+
+ "github.com/mudler/LocalAI/core/config"
+)
+
+type TargetState string
+
+const (
+ StateHealthy TargetState = "healthy"
+ StateDown TargetState = "down"
+ StateRecovering TargetState = "recovering"
+ StateMissing TargetState = "missing"
+)
+
+type ChainState string
+
+const (
+ ChainPrimary ChainState = "primary"
+ ChainFallback ChainState = "fallback"
+ ChainDegraded ChainState = "degraded"
+)
+
+type Kind string
+
+const (
+ KindLocal Kind = "local"
+ KindRemote Kind = "remote"
+)
+
+type Reason string
+
+const (
+ ReasonTrip Reason = "trip"
+ ReasonRecovery Reason = "recovery"
+ ReasonManual Reason = "manual"
+ ReasonDegraded Reason = "degraded"
+ ReasonMissing Reason = "missing"
+ ReasonInitial Reason = "initial"
+)
+
+type EventType string
+
+const (
+ EventChainSwitched EventType = "chain.switched"
+ EventTargetState EventType = "target.state"
+)
+
+// Event is one change of a target state or of a chain's active target.
+type Event struct {
+ Type EventType `json:"type"`
+ Chain string `json:"chain,omitempty"`
+ Target string `json:"target,omitempty"`
+ From string `json:"from"`
+ To string `json:"to"`
+ State string `json:"state,omitempty"`
+ Reason Reason `json:"reason"`
+ Error string `json:"error,omitempty"`
+ At time.Time `json:"at"`
+}
+
+type TargetStatus struct {
+ Model string `json:"model"`
+ Kind Kind `json:"kind"`
+ Warm bool `json:"warm"`
+ State TargetState `json:"state"`
+ ConsecutiveOK int `json:"consecutive_ok"`
+ LastProbe *time.Time `json:"last_probe,omitempty"`
+ LastError string `json:"last_error,omitempty"`
+}
+
+type ChainStatus struct {
+ Name string `json:"name"`
+ State ChainState `json:"state"`
+ Active string `json:"active"`
+ ActiveSince time.Time `json:"active_since"`
+ Pinned *string `json:"pinned"`
+ Targets []TargetStatus `json:"targets"`
+}
+
+// KindOf decides how a target is probed: proxy backends forward to another
+// server and are checked over HTTP, everything else runs in this instance.
+func KindOf(cfg config.ModelConfig) Kind {
+ if cfg.IsRemoteProxy() {
+ return KindRemote
+ }
+ return KindLocal
+}
+
+// MergePinned adds warm failover targets to the config-pinned model list, so
+// the watchdog never evicts them.
+func MergePinned(pinned, warm []string) []string {
+ out := slices.Clone(pinned)
+ for _, w := range warm {
+ if !slices.Contains(out, w) {
+ out = append(out, w)
+ }
+ }
+ return out
+}
diff --git a/core/services/failover/types_test.go b/core/services/failover/types_test.go
new file mode 100644
index 000000000..abbe0bd09
--- /dev/null
+++ b/core/services/failover/types_test.go
@@ -0,0 +1,22 @@
+package failover
+
+import (
+ "github.com/mudler/LocalAI/core/config"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("KindOf", func() {
+ It("treats proxy backends as remote", func() {
+ Expect(KindOf(config.ModelConfig{Backend: "cloud-proxy"})).To(Equal(KindRemote))
+ Expect(KindOf(config.ModelConfig{Backend: "localai-proxy"})).To(Equal(KindRemote))
+ Expect(KindOf(config.ModelConfig{Backend: "llama-cpp"})).To(Equal(KindLocal))
+ })
+})
+
+var _ = Describe("MergePinned", func() {
+ It("adds warm targets without duplicates", func() {
+ Expect(MergePinned([]string{"a", "b"}, []string{"b", "c"})).To(Equal([]string{"a", "b", "c"}))
+ Expect(MergePinned(nil, nil)).To(BeEmpty())
+ })
+})
diff --git a/core/services/modeladmin/config.go b/core/services/modeladmin/config.go
index f85f29ee9..4471b51c7 100644
--- a/core/services/modeladmin/config.go
+++ b/core/services/modeladmin/config.go
@@ -169,6 +169,9 @@ func (s *ConfigService) patchConfig(ctx context.Context, name string, patch map[
if err := s.Loader.ValidateAliasTarget(&updated); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
+ if err := s.Loader.ValidateFailoverTargets(&updated); err != nil {
+ return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
+ }
var result *PatchResult
err = s.withMutationRollback([]string{configPath}, func() error {
if err := writeFileAtomic(configPath, yamlData, 0644); err != nil {
@@ -286,6 +289,9 @@ func (s *ConfigService) editYAML(ctx context.Context, name string, body []byte)
if err := s.Loader.ValidateAliasTarget(&req); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
}
+ if err := s.Loader.ValidateFailoverTargets(&req); err != nil {
+ return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
+ }
configPath := existing.GetModelConfigFile()
modelsPath := s.modelsPath()
diff --git a/core/services/syncstate/syncstate.go b/core/services/syncstate/syncstate.go
index 809177d40..5aa69470f 100644
--- a/core/services/syncstate/syncstate.go
+++ b/core/services/syncstate/syncstate.go
@@ -54,7 +54,7 @@ type delta[K comparable, V any] struct {
}
// SyncedMap is a cross-replica in-memory map. A local write (Set/Delete) updates
-// memory, the optional durable Store, then broadcasts a delta to peers. A peer's
+// the optional durable Store, then memory, then broadcasts a delta to peers. A peer's
// delta updates memory only and fires OnApply - it never re-broadcasts and never
// writes the Store. That structural split is the echo-loop guard (same pattern as
// galleryop.mergeStatus / OpCache.applyStart): receiving your own broadcast just
@@ -141,34 +141,40 @@ func (m *SyncedMap[K, V]) Close() error {
return nil
}
-// Set updates the value locally, writes through the Store, then broadcasts.
-// Per the data-flow contract the Store write happens under the lock so memory and
-// durable state move together; the broadcast is best-effort after unlocking.
+// Set writes through the Store, then updates the value locally, then
+// broadcasts. The Store write comes first and happens under the lock so memory
+// and durable state move together: when it fails, Set returns the error with
+// memory and peers untouched. Keeping an unpersisted value in memory would let
+// this replica serve it (and a caller that re-reads the map re-apply it) while
+// the Store and every other replica disagree, until the next re-hydrate.
+// The broadcast is best-effort after unlocking.
func (m *SyncedMap[K, V]) Set(ctx context.Context, v V) error {
k := m.cfg.Key(v)
m.mu.Lock()
- m.data[k] = v
if m.cfg.Store != nil {
if err := m.cfg.Store.Upsert(ctx, v); err != nil {
m.mu.Unlock()
return err
}
}
+ m.data[k] = v
m.mu.Unlock()
m.publish(opSet, k, v)
return nil
}
-// Delete removes the key locally, deletes it from the Store, then broadcasts.
+// Delete deletes the key from the Store, then removes it locally, then
+// broadcasts. A failed Store delete leaves memory and peers untouched, for the
+// same reason as Set.
func (m *SyncedMap[K, V]) Delete(ctx context.Context, k K) error {
m.mu.Lock()
- delete(m.data, k)
if m.cfg.Store != nil {
if err := m.cfg.Store.Delete(ctx, k); err != nil {
m.mu.Unlock()
return err
}
}
+ delete(m.data, k)
m.mu.Unlock()
var zero V
m.publish(opDelete, k, zero)
diff --git a/core/services/syncstate/syncstate_test.go b/core/services/syncstate/syncstate_test.go
index 1e31db41b..e8daf6014 100644
--- a/core/services/syncstate/syncstate_test.go
+++ b/core/services/syncstate/syncstate_test.go
@@ -2,6 +2,7 @@ package syncstate_test
import (
"context"
+ "errors"
"sync"
. "github.com/onsi/ginkgo/v2"
@@ -34,6 +35,8 @@ type fakeStore struct {
upsertCalls int
deleteCalls int
listCalls int
+ // fail, when set, makes Upsert and Delete return it without writing.
+ fail error
}
func newFakeStore(seed ...*job) *fakeStore {
@@ -59,6 +62,9 @@ func (s *fakeStore) Upsert(_ context.Context, j *job) error {
s.mu.Lock()
defer s.mu.Unlock()
s.upsertCalls++
+ if s.fail != nil {
+ return s.fail
+ }
s.data[j.ID] = j
return nil
}
@@ -67,6 +73,9 @@ func (s *fakeStore) Delete(_ context.Context, k string) error {
s.mu.Lock()
defer s.mu.Unlock()
s.deleteCalls++
+ if s.fail != nil {
+ return s.fail
+ }
delete(s.data, k)
return nil
}
@@ -207,6 +216,41 @@ var _ = Describe("SyncedMap", func() {
})
})
+ Describe("failed Store write", func() {
+ It("leaves memory and peers untouched when Set or Delete cannot persist", func() {
+ bus := testutil.NewFakeBus()
+ storeA := newFakeStore(&job{ID: "kept", Status: "running"})
+ a := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus, Store: storeA})
+ b := syncstate.New(syncstate.Config[string, *job]{Name: stateName, Key: jobKey, Nats: bus})
+ Expect(a.Start(ctx)).To(Succeed())
+ Expect(b.Start(ctx)).To(Succeed())
+ defer func() {
+ Expect(a.Close()).To(Succeed())
+ Expect(b.Close()).To(Succeed())
+ }()
+
+ errDown := errors.New("database down")
+ storeA.mu.Lock()
+ storeA.fail = errDown
+ storeA.mu.Unlock()
+
+ Expect(a.Set(ctx, &job{ID: "new", Status: "running"})).To(MatchError(errDown))
+ _, ok := a.Get("new")
+ Expect(ok).To(BeFalse(), "a value that was not persisted must not be served")
+ _, ok = b.Get("new")
+ Expect(ok).To(BeFalse(), "a value that was not persisted must not be broadcast")
+
+ Expect(a.Set(ctx, &job{ID: "kept", Status: "done"})).To(MatchError(errDown))
+ got, ok := a.Get("kept")
+ Expect(ok).To(BeTrue())
+ Expect(got.Status).To(Equal("running"), "a failed overwrite must keep the persisted value")
+
+ Expect(a.Delete(ctx, "kept")).To(MatchError(errDown))
+ _, ok = a.Get("kept")
+ Expect(ok).To(BeTrue(), "a failed delete must keep the persisted value")
+ })
+ })
+
Describe("OnApply hook", func() {
It("fires with the correct op and key on an applied delta", func() {
bus := testutil.NewFakeBus()
diff --git a/core/trace/backend_trace.go b/core/trace/backend_trace.go
index 072e41ebf..e14370c45 100644
--- a/core/trace/backend_trace.go
+++ b/core/trace/backend_trace.go
@@ -48,6 +48,7 @@ const (
BackendTraceTokenClassify BackendTraceType = "token_classify"
BackendTracePatternPII BackendTraceType = "pattern_pii"
BackendTraceVectorStore BackendTraceType = "vector_store"
+ BackendTraceFailover BackendTraceType = "failover"
)
const (
diff --git a/docs/content/features/model-aliases.md b/docs/content/features/model-aliases.md
index ed52f0977..292c05374 100644
--- a/docs/content/features/model-aliases.md
+++ b/docs/content/features/model-aliases.md
@@ -38,6 +38,9 @@ That is the whole config: a `name` (the alias clients call) and an `alias` key
- Usage accounting records both sides: requested `gpt-4`, served `my-llama-3`.
- Aliases work for every modality (chat, embeddings, audio, images, and so on).
+To serve a name from several models with automatic fallback, use a [failover
+chain]({{%relref "features/model-failover" %}}).
+
## Managing aliases
You can create, swap, and remove aliases from any of the management surfaces.
diff --git a/docs/content/features/model-failover.md b/docs/content/features/model-failover.md
new file mode 100644
index 000000000..aa6db3a94
--- /dev/null
+++ b/docs/content/features/model-failover.md
@@ -0,0 +1,304 @@
+
++++
+disableToc = false
+title = "Model Failover"
+weight = 15
+url = "/features/model-failover/"
++++
+
+A **failover chain** is a model name that is served by an ordered list of
+other models. LocalAI sends each request to the first healthy target. When a
+target fails, the request moves to the next target, and later requests stay
+there until the first target has recovered.
+
+Use it to serve a model from a remote LocalAI or another OpenAI-compatible
+provider, and to fall back to a local model when the remote one is down.
+
+## Declaring a chain
+
+```yaml
+name: assistant-llm
+failover:
+ targets:
+ - model: argus-llm # for example a localai-proxy or cloud-proxy model
+ - model: gemma-local
+ warm: true # keep it loaded
+```
+
+Clients call `assistant-llm`. Each target is a normal model config. A chain
+has no `backend` and no `parameters.model`.
+
+Optional settings, with their defaults:
+
+```yaml
+failover:
+ probe:
+ interval: 15s # how often an idle target is checked
+ timeout: 5s
+ trip:
+ errors: 1 # failures within the window that mark a target down
+ window: 30s
+ recovery:
+ probes: 3 # test requests a target must pass before it is used again
+ min_dwell: 60s # minimum time on a lower target before moving back
+```
+
+Rules:
+
+- A chain needs at least 2 targets. A target can be an alias, but not another
+ chain.
+- A chain cannot also set `alias` or `backend`.
+- Responses name the chain as the model. The `X-LocalAI-Served-Model` header
+ names the target that served the request.
+- A remote (`localai-proxy` or `cloud-proxy`) target receives its own model
+ name, never the chain name: `proxy.upstream_model`, or the target name when
+ `upstream_model` is empty. The health check looks for the same name.
+
+## How the target is chosen
+
+- The active target is the first healthy target in the list.
+- When a target fails, LocalAI marks it down and moves to the next target at
+ once.
+- LocalAI moves back to a higher target only when that target has passed
+ `recovery.probes` test requests **and** the current target has been active
+ for at least `recovery.min_dwell`. This stops an unstable upstream from
+ moving traffic back and forth.
+- When all targets are down, the chain is `degraded`. Each request still tries
+ every target in order.
+
+## Retry inside a request
+
+When a target fails before the response starts, LocalAI sends the same
+request to the next target. The client does not see the failure.
+
+- LocalAI does not retry after the first byte of a response is sent (for
+ example after the first streamed token). The request fails, the target is
+ marked down, and the next request uses the next target.
+- A target that is at its concurrency limit (an admission rejection) or that
+ is disabled is skipped for that request without being marked down.
+- LocalAI does not retry client errors (4xx), such as a prompt that is too
+ long, because the next target would reject it too. A 4xx counts neither as
+ a success nor as a failure for the target.
+- Request bodies larger than 32 MiB are not retried.
+
+When the primary did not serve the request, the response has the header
+`X-LocalAI-Failover: fallback`, or `X-LocalAI-Failover: degraded` when all
+targets were down.
+
+## Health checks
+
+| Target | Regular check | Check before moving back |
+|---|---|---|
+| Remote (`localai-proxy`, `cloud-proxy`) | `GET /v1/models` on the upstream lists the model | one small real request, for example a 1-token completion |
+| Local, `warm: true` | the backend answers a health check. A check never loads the model: while it is not loaded, the check passes and real requests judge it | one small real request. While the model is not loaded, the target is used again after `min_dwell` |
+| Local, not warm | none: judged only by real requests; it is never loaded only to check it | none: the target is used again after `min_dwell` |
+
+A request that succeeds counts as a check, so a busy target is almost never
+probed.
+
+When a target is in more than one chain, its check settings come from the
+first of those chains in name order.
+
+## Warm targets
+
+`warm: true` loads a local target at startup and protects it from idle and
+LRU eviction, so a switch does not wait for the model to load. Warm targets
+count toward the active backend limit (`--max-active-backends`) like any
+pinned model: LocalAI never evicts them to make room, and if they fill the
+limit, a new model still loads rather than being blocked.
+
+`warm` applies only to local targets. On a remote (`localai-proxy` or
+`cloud-proxy`) target it has no effect, and LocalAI logs a warning when it
+loads the chain.
+
+## Realtime pipelines
+
+A pipeline stage can name a chain:
+
+```yaml
+name: assistant
+pipeline:
+ vad: silero-vad
+ transcription: whisper-chain
+ llm: assistant-llm
+ tts: voice-chain
+```
+
+LocalAI resolves the chain for every call of the stage, in full pipelines and
+in transcription-only and sound-detection-only sessions. When a chain switches,
+the session stays open and keeps its conversation. The next turn uses the new
+target.
+
+The session receives a `localai.model.failover` event for each chain stage when
+it starts (`reason: initial`) and each time a chain switches:
+
+```json
+{"type":"localai.model.failover","chain":"assistant-llm","stage":"llm",
+ "from":"argus-llm","to":"gemma-local","state":"fallback","reason":"trip"}
+```
+
+### Example: stages on a remote LocalAI
+
+This pipeline runs its transcription, LLM and TTS stages on a remote LocalAI
+(`argus`) through [`localai-proxy`]({{% relref "operations/cloud-proxy" %}})
+models, and uses local models when the remote instance is down. Each stage has
+its own chain, so one stage can fail over while the others stay remote.
+
+```yaml
+# Remote targets: each one names the model on the upstream LocalAI.
+name: argus-stt
+backend: localai-proxy
+known_usecases: [transcript]
+options:
+ - realtime_pipeline:asr-pipeline # upstream pipeline for live transcription
+proxy:
+ upstream_url: http://argus.lan:8080
+ upstream_model: parakeet
+---
+name: argus-llm
+backend: localai-proxy
+known_usecases: [chat]
+proxy:
+ upstream_url: http://argus.lan:8080
+ upstream_model: gemma-3-12b
+---
+name: argus-tts
+backend: localai-proxy
+known_usecases: [tts]
+proxy:
+ upstream_url: http://argus.lan:8080
+ upstream_model: kokoro
+---
+# One chain per stage, remote first, local second.
+name: stt-chain
+failover:
+ targets: [{model: argus-stt}, {model: whisper-local, warm: true}]
+---
+name: llm-chain
+failover:
+ targets: [{model: argus-llm}, {model: gemma-local, warm: true}]
+---
+name: tts-chain
+failover:
+ targets: [{model: argus-tts}, {model: piper-local}]
+---
+name: assistant
+pipeline:
+ vad: silero-vad
+ transcription: stt-chain
+ llm: llm-chain
+ tts: tts-chain
+```
+
+The example shows the configs as one YAML stream; put each config in its own
+file in the models directory. When `argus` stops
+answering, the next call of each stage fails over to the local model and the
+session receives a `localai.model.failover` event for that stage. A remote
+target that does not support a call (it returns `Unimplemented`) is skipped for
+that call and is not marked down.
+
+Limits:
+
+- After a `session.update` that changes the pipeline, `localai.model.failover`
+ events keep describing the chains from session start.
+- A chain used as a router candidate, or as the classifier-mode scoring model,
+ is not resolved per call.
+
+## Watching failover
+
+- `GET /api/failover` lists every chain, its active target and the state of
+ each target.
+- `GET /api/failover/{chain}` returns one chain.
+- `GET /api/failover/events` is a server-sent event stream. The first event is
+ `snapshot` with the full state. Then `chain.switched` and `target.state`
+ events follow.
+- Metrics: `localai_failover_switches_total{chain,from,to,reason}` and
+ `localai_failover_target_up{target}`.
+- With tracing on, each skipped target appears in the Traces view with the
+ error that made LocalAI skip it.
+
+## Pinning a target
+
+An admin can force a chain to one target, for example during maintenance:
+
+```bash
+curl -X POST http://localhost:8080/api/failover/assistant-llm/pin \
+ -H 'Content-Type: application/json' -d '{"target":"gemma-local"}'
+curl -X DELETE http://localhost:8080/api/failover/assistant-llm/pin
+```
+
+While a chain is pinned, only the pinned target serves it. Health checks
+continue. On a single LocalAI instance, a restart removes the pin. In
+[distributed mode](#distributed-mode), pins persist.
+
+## In the UI
+
+- The model editor's template picker has a **Failover Chain** template that
+ starts a new model with two empty targets. Any model can also add a
+ `failover.targets` field directly: each row picks a target model and a
+ **warm** toggle, with move up/down and remove controls. The toggle is
+ available on every row; `warm` has no effect on a remote (`localai-proxy` or
+ `cloud-proxy`) target, and LocalAI logs a warning when it loads a chain with
+ one set. The field flags fewer than two targets, a duplicate target, and the
+ chain naming itself as one of its own targets.
+- Opening an existing chain in the model editor shows a health strip: the
+ chain's status pill, its active target and how long it has been active,
+ and a table of every target with its kind, warm flag, status, last probe
+ time and last error. An admin sees a **Pin** button on each target and an
+ **Unpin** action for the chain, both behind a confirmation dialog.
+- The **Failover** page (`/app/failover`, admin only, linked from the
+ console navigation) lists every chain with its status pill, active target,
+ a small pill per target, and time since the last switch. It links each
+ chain name to its model editor page and shows an empty state linking to
+ the failover template when no chains exist yet.
+- The Installed Models list badges a model that belongs to a chain with
+ `chain → `, next to the alias badge.
+- All of the above update live from the same event stream as
+ `GET /api/failover/events` — the strip, the overview page and the badge do
+ not need a page refresh to reflect a `chain.switched` or `target.state`
+ event.
+
+## Assistant and MCP
+
+The LocalAI Assistant and `local-ai mcp-server` offer `list_failover_chains`,
+`pin_failover_target` and `unpin_failover_target`. Create and edit chains with
+the model config tools, like any other model.
+
+## Distributed mode
+
+In [distributed mode]({{%relref "features/distributed-mode" %}}), all frontends
+share one failover state:
+
+- Pins apply to the whole cluster. LocalAI stores them in the database, so
+ they persist across restarts. A pin set on one frontend applies on all.
+- Target health and the active target of each chain are shared over NATS, so
+ all frontends converge on the same target for a chain.
+- One frontend, the probe leader, runs the health checks, decides fail-over
+ and fail-back, and loads warm targets. The leader holds a PostgreSQL
+ advisory lock and keeps it until it stops or its database connection fails.
+ Then another frontend takes the lock and becomes the leader: immediately
+ when the leader shuts down or its process exits, and within about 30 seconds
+ when the leader's host or network fails.
+- Warm targets stay loaded on the workers. The router and the replica
+ reconciler treat them like pinned models and do not evict them.
+- A frontend that starts late gets the current state within 10 seconds,
+ because the leader sends its full state again every 10 seconds.
+- If PostgreSQL is not available, no frontend holds the lock. Health checks
+ and fail-back stop until the database is back. Requests still fail over to
+ the next target when a target fails during the request.
+- If a frontend cannot start the shared state, it logs an error and manages
+ failover alone, as a single LocalAI instance does.
+- Leadership follows the database lock, not NATS. A leader that loses its NATS
+ connection but keeps its database connection stays the leader. Until NATS
+ recovers, the other frontends keep the last decisions they received from it,
+ and its new decisions do not reach them.
+- Each frontend reads the pins from the database again every 30 seconds and
+ after a NATS reconnect, and applies any change within 10 seconds. A frontend
+ that missed a pin or an unpin catches up in this way. If a pin cannot be written to the database, LocalAI returns an
+ error and restores the previous pin.
+
+## Limits
+
+- Chains do not nest.
+- See also [model aliases]({{%relref "features/model-aliases" %}}) and the
+ [realtime API]({{%relref "features/openai-realtime" %}}).
diff --git a/docs/content/features/openai-realtime.md b/docs/content/features/openai-realtime.md
index 54bec58ab..ac7afb484 100644
--- a/docs/content/features/openai-realtime.md
+++ b/docs/content/features/openai-realtime.md
@@ -31,6 +31,8 @@ This configuration links the following components:
Make sure all referenced models (`silero-vad-ggml`, `whisper-large-turbo`, `qwen3-4b`, `tts-1`) are also installed or defined in your LocalAI instance.
+A pipeline stage can name a [failover chain]({{%relref "features/model-failover" %}}); the stage then switches targets without closing the session.
+
### Streaming the pipeline
By default each stage runs to completion before the next begins: the whole utterance is transcribed, the full LLM reply is generated, then it is synthesized. Each stage can instead be streamed incrementally, which lowers the time-to-first-audio of a turn:
diff --git a/docs/content/operations/cloud-proxy.md b/docs/content/operations/cloud-proxy.md
index a258ece14..312fa2327 100644
--- a/docs/content/operations/cloud-proxy.md
+++ b/docs/content/operations/cloud-proxy.md
@@ -29,6 +29,9 @@ egress remains subject to the same redaction rules a local model would apply.
- Use the intelligent router to send small or simple prompts to a local model
and complex ones to Claude or GPT-4o.
+To fall back to a local model when the upstream is down, list the proxy model
+in a [failover chain]({{%relref "features/model-failover" %}}).
+
## How it works
1. Request hits LocalAI on `/v1/chat/completions` (OpenAI-shaped) or
@@ -270,6 +273,113 @@ ACLs, and the cloud-proxy fork all run against the resolved target.
See [Middleware: PII filtering and intelligent routing]({{< relref "middleware.md" >}})
for the full router and PII-filter reference.
+## Proxying to another LocalAI (`localai-proxy`)
+
+`cloud-proxy` forwards chat and Messages requests only. To serve a model from
+another LocalAI instance for every API it has, use `backend: localai-proxy`.
+The backend receives the request from the local pipeline like any other
+backend and sends it to the REST API of the upstream LocalAI. Because it is a
+normal backend, a `localai-proxy` model can be a stage of a realtime pipeline
+or a target of a [failover chain]({{% relref "features/model-failover" %}}).
+
+```yaml
+name: remote-llm
+backend: localai-proxy
+known_usecases: [chat]
+proxy:
+ # Base URL of the upstream LocalAI. Do not add /v1 or an endpoint path:
+ # the backend adds the path for each API.
+ upstream_url: https://argus.lan:8080
+ # The model name on the upstream. When empty, the name of this config.
+ upstream_model: gemma-3-12b
+ # Optional. The upstream API key, from an environment variable
+ # (or api_key_file). Sent as "Authorization: Bearer ".
+ api_key_env: ARGUS_API_KEY
+ # Optional. Time limit for each non-streaming request. Streams have no limit.
+ request_timeout_seconds: 120
+```
+
+A model that does live transcription in a realtime pipeline also names a
+realtime pipeline on the upstream. The backend opens a transcription session
+on the upstream `/v1/realtime` endpoint with that pipeline:
+
+```yaml
+name: remote-stt
+backend: localai-proxy
+known_usecases: [transcript]
+options:
+ - realtime_pipeline:asr-pipeline
+proxy:
+ upstream_url: https://argus.lan:8080
+ upstream_model: parakeet
+```
+
+Set `known_usecases` on every `localai-proxy` model. Failover uses it to match
+targets, and LocalAI cannot guess the usecases of a remote model. For a chat
+model, `known_usecases: [chat]` has one more effect: LocalAI sends the chat
+messages to the upstream `/v1/chat/completions` endpoint, and the upstream
+applies its own chat template, tool parsing and reasoning parsing. Without
+`chat`, or when the config has its own templates, LocalAI renders the prompt
+locally and sends it to `/v1/completions`. `proxy.mode` and `proxy.provider`
+have no effect on this backend.
+
+Supported APIs:
+
+- Text: chat and completions (also streamed), embeddings, rerank, tokenize,
+ detokenize, score.
+- Audio: TTS (also streamed), sound generation, transcription (also streamed),
+ live transcription (with `realtime_pipeline`), diarization, VAD, sound
+ classification, audio transformations.
+- Image, video and 3D: image generation, upscaling, video generation, 3D
+ generation and animation. The backend downloads the files that the upstream
+ generates.
+- Vision: object detection, depth, face verification and analysis, voice
+ verification, analysis and embeddings.
+- Stores: set, get, delete, find.
+
+Methods that have no REST API on the upstream return the gRPC error
+`Unimplemented` ("localai-proxy: has no upstream counterpart"). The
+upstream returning `501 Not Implemented` maps to the same code. Both mean a
+capability gap, not a broken target: audio encoding and decoding,
+audio-to-audio streams, token classification (PII NER), model metadata,
+fine-tuning, quantization and model export fall in this bucket. A failover
+chain skips a target that returns `Unimplemented` and tries the next target,
+but does not mark the target down. This applies to every API, also to the APIs
+that report `Unimplemented` to the client as HTTP `501` (images, video, 3D,
+detection, depth, face and voice).
+
+Errors from the upstream: a 5xx response (other than 501) or a connection
+failure becomes `Unavailable`, and a failover chain marks the target down. A
+4xx response becomes `InvalidArgument`, and LocalAI returns it to the client
+without a retry or a trip — except `429 Too Many Requests`, which becomes
+`ResourceExhausted`: the request itself is fine, the upstream is just out of
+capacity, so a failover chain retries it on the next target and trips the
+rate-limited one, moving traffic off it until it recovers.
+
+Known limits:
+
+- Voice-profile paths pass through unresolved. When LocalAI resolves a TTS
+ voice to a local file (for example a voice clone reference), the backend
+ sends that path to the upstream, where it does not exist. Use voices that
+ the upstream knows by name.
+- Depth exports are not supported. The upstream writes them to its own disk,
+ so a depth request with exports or a destination file returns
+ `Unimplemented`. Depth maps and points without exports work.
+- The REST transcription API has no end-of-utterance (`eou`) flag, so
+ transcriptions through the proxy never set it. Live transcription through
+ `realtime_pipeline` sets `eou` at the end of each utterance.
+- Sound generation from a source audio file is not supported.
+- Chat and completions do not forward grammars, so JSON mode and other
+ grammar-constrained output are not enforced by the upstream. Images, audio
+ and video attached to messages are not forwarded either. The backend logs a
+ warning for each request that loses one of these fields.
+- Streamed TTS cannot detect an upstream synthesis failure that ends the
+ stream cleanly. The client receives the audio produced so far as a complete
+ response, and a failover chain does not retry it. A stream that is cut off
+ is reported as an error.
+- `upstream_url` is the root of the upstream server. If it has a `/v1` path,
+ the backend removes `/v1` and everything after it, and logs a warning.
+
## Limitations
- **Passthrough does no wire-shape translation.** Use `mode: translate` (with
diff --git a/docs/superpowers/plans/2026-08-02-whisper-medusa-backend.md b/docs/superpowers/plans/2026-08-02-whisper-medusa-backend.md
deleted file mode 100644
index 3e35636e9..000000000
--- a/docs/superpowers/plans/2026-08-02-whisper-medusa-backend.md
+++ /dev/null
@@ -1,63 +0,0 @@
-# Whisper-Medusa Backend Implementation Plan
-
-> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
-
-**Goal:** Add a dedicated LocalAI speech-to-text backend for aiola Whisper-Medusa checkpoints.
-
-**Architecture:** A Python gRPC backend owns model loading, audio normalization, and Medusa generation. LocalAI's existing `AudioTranscription` RPC remains unchanged; build, gallery, and documentation surfaces follow the existing Python ASR backend pattern.
-
-**Tech Stack:** Python 3.11, PyTorch, torchaudio, transformers, whisper-medusa, gRPC, YAML, Make.
-
-## Global Constraints
-
-- Accept local model paths and Hugging Face model identifiers through `resolve_model_reference`.
-- Normalize input audio to mono 16 kHz before generation.
-- Default the language to `en` and expose upstream generation regulation options.
-- Document upstream's archived status, 30-second clip limit, and checkpoint language limitations.
-- Build Linux CPU and NVIDIA CUDA 12 images; do not claim unsupported Darwin or ROCm coverage.
-
----
-
-### Task 1: Backend behavior
-
-**Files:**
-- Create: `backend/python/whisper-medusa/backend.py`
-- Create: `backend/python/whisper-medusa/test_unit.py`
-
-**Interfaces:**
-- Consumes: LocalAI `LoadModel` and `AudioTranscription` protobuf requests.
-- Produces: `BackendServicer`, `_parse_options`, and `_prepare_audio`.
-
-- [ ] Write unit tests for option parsing, mono conversion, resampling, load failure, and transcription.
-- [ ] Run `python -m unittest test_unit.py` and confirm it fails because the backend does not exist.
-- [ ] Implement the minimal gRPC backend and rerun the unit tests to green.
-
-### Task 2: Packaging and registration
-
-**Files:**
-- Create: `backend/python/whisper-medusa/{Makefile,install.sh,protogen.sh,run.sh,test.sh,requirements.txt,requirements-cpu.txt,requirements-cublas12.txt}`
-- Modify: `Makefile`
-- Modify: `.github/backend-matrix.yml`
-- Modify: `backend/index.yaml`
-
-**Interfaces:**
-- Consumes: the Python backend Docker build conventions.
-- Produces: `whisper-medusa` install/build targets and CPU/CUDA backend images.
-
-- [ ] Add packaging scripts and pinned upstream dependency.
-- [ ] Register the backend in Make and backend metadata.
-- [ ] Add Linux amd64 CPU and CUDA 12 CI matrix entries.
-- [ ] Validate Make and YAML parsing.
-
-### Task 3: Documentation and verification
-
-**Files:**
-- Create: `docs/content/features/whisper-medusa.md`
-- Modify: `docs/content/features/backends.md`
-
-**Interfaces:**
-- Produces: user-facing model YAML and limitation guidance.
-
-- [ ] Document setup, configuration, options, and upstream constraints.
-- [ ] Run unit tests, syntax compilation, registration checks, and diff review.
-- [ ] Commit with the required `Assisted-by` trailer, push, and open a PR closing issue #3127.
diff --git a/docs/superpowers/specs/2026-08-21-configurable-copy-buffer-design.md b/docs/superpowers/specs/2026-08-21-configurable-copy-buffer-design.md
deleted file mode 100644
index 1085af7fe..000000000
--- a/docs/superpowers/specs/2026-08-21-configurable-copy-buffer-design.md
+++ /dev/null
@@ -1,73 +0,0 @@
-# Configurable copy buffer design
-
-**Date:** 21 August 2026
-**Status:** Approved
-
-## Problem
-
-`pkg/xio.Copy` wraps a source reader so a context can stop a copy between
-reads. It delegates to `io.Copy`, which uses a 32 KiB buffer for the wrapped
-reader and writer types used by model downloads.
-
-Small writes limit model import throughput when the models directory uses an
-SMB volume. The development deployment reads large files from the volume at
-about 104 MiB/s. A model import writes to the same volume at less than 1 MiB/s.
-
-## Design
-
-Keep `xio.Copy` as the context-aware copy entry point. Add variadic functional
-options so existing callers continue to compile without changes.
-
-Add an exported `Option` type and a `WithBufferSize(size int) Option` function.
-`Copy` uses a 1 MiB buffer by default. A caller can override the buffer size
-with `WithBufferSize`.
-
-If a caller supplies a non-positive buffer size, `Copy` uses the 1 MiB default.
-This rule prevents invalid configuration from causing an `io.CopyBuffer`
-panic.
-
-`Copy` allocates one buffer for each active call. It passes that buffer to
-`io.CopyBuffer`. The context-aware reader continues to check cancellation
-before each source read.
-
-The first change does not use `sync.Pool`. A pool adds shared state and retains
-large caller-selected buffers. Measurements do not justify that complexity.
-
-## Compatibility
-
-The existing signature gains only a variadic argument:
-
-```go
-func Copy(ctx context.Context, dst io.Writer, src io.Reader, options ...Option) (int64, error)
-```
-
-All existing calls remain source compatible. Copy results and cancellation
-errors do not change.
-
-The default buffer increases temporary memory use by approximately 992 KiB for
-each concurrent copy compared with the current 32 KiB buffer.
-
-## Tests and measurement
-
-Add a Ginkgo suite for `pkg/xio`. Tests cover these behaviors:
-
-- `Copy` copies the complete source.
-- The default buffer permits reads larger than 32 KiB.
-- `WithBufferSize` changes the maximum requested read size.
-- A non-positive override uses the default buffer.
-- A canceled context stops the copy and returns the context error.
-
-Add a benchmark that runs `Copy` with the default buffer and representative
-overrides. The benchmark records throughput and allocations. It does not make
-timing assertions.
-
-Run the focused `pkg/xio` suite first. Then run the packages that call
-`xio.Copy`: `pkg/downloader` and `pkg/oci`.
-
-## Deployment validation
-
-The code change alone does not alter the running development deployment. After
-CI publishes a development image and Flux deploys it, import a large model to
-the NAS-backed models directory. Compare the progress rate with the previous
-0.7-0.8 MiB/s result.
-
diff --git a/docs/superpowers/specs/2026-08-21-distributed-model-config-revisions-design.md b/docs/superpowers/specs/2026-08-21-distributed-model-config-revisions-design.md
deleted file mode 100644
index 6d1de2e63..000000000
--- a/docs/superpowers/specs/2026-08-21-distributed-model-config-revisions-design.md
+++ /dev/null
@@ -1,357 +0,0 @@
-# Distributed Model Configuration Revisions
-
-## Problem
-
-Editing a model configuration in a distributed LocalAI deployment can leave the
-cluster serving different effective configurations for the same logical model.
-The frontend reloads the edited YAML and asks workers to stop the model, but the
-existing `backend.stop` message is fire-and-forget. The frontend therefore
-removes routing state without knowing whether the worker process stopped.
-
-Separately, the replica reconciler persists `ModelLoadInfo` independently of
-live `NodeModel` rows. This is necessary for restoring `min_replicas` after a
-worker failure, but the persisted options currently have no relationship to a
-specific revision of the model configuration. After an edit, the reconciler can
-restore a replica from options captured before the edit.
-
-The observed result was one replica serving a context near 100K while another
-served the default 8K context and default parallelism. Requests behaved
-differently depending on which replica the router selected. The problem is not
-specific to `context_size`: any load-time model option can be stale.
-
-## Goals
-
-- Make all routable replicas of a logical model belong to the current model
- configuration revision.
-- Prevent the reconciler and late load jobs from restoring options belonging to
- an older revision.
-- Remove a model from routing before attempting distributed cleanup.
-- Confirm that the exact worker process exited before deleting its registry
- row.
-- Recover safely when a worker or NATS is temporarily unreachable.
-- Apply the same lifecycle to raw YAML edits, structured configuration patches,
- renames, disabling, and changes received from peer frontends.
-- Preserve the existing ability to restore `min_replicas` after ordinary
- worker or backend failure when the model configuration has not changed.
-- Expose enough state to diagnose why two replicas have different effective
- options.
-
-## Non-goals
-
-- Requiring identical hardware-derived options on heterogeneous workers.
-- Changing `model.unload`, which remains a memory-release operation.
-- Replacing backend administration operations such as backend upgrade, delete,
- or stop-all.
-- Automatically upgrading workers that do not support the new stop protocol.
-- Making arbitrary out-of-band filesystem edits transactional across multiple
- machines. Such edits are detected when the model configuration loader next
- refreshes the model.
-
-## Configuration identity
-
-Each validated model configuration has a `config_revision`. The revision is a
-SHA-256 digest of a canonical semantic representation of the validated model
-configuration. Formatting, YAML comments, and map ordering do not affect the
-revision. Load-time request overrides and node-specific hardware tuning are not
-part of this digest.
-
-Canonicalization must use the typed, validated configuration rather than raw
-YAML bytes. The canonical representation includes every field that can affect
-model loading or serving. Fields used only to locate the source file or report
-runtime status are excluded. The canonical encoder must produce stable field
-and map ordering and must distinguish absent values where absence has different
-semantics from an explicit zero value.
-
-The revision is carried with the model options from configuration loading into
-the distributed router. It is also persisted in:
-
-- `ModelConfigState`, keyed by logical model name, as the currently accepted
- revision;
-- `ModelLoadInfo`, alongside the serialized `pb.ModelOptions` used for future
- reconciliation;
-- `NodeModel`, identifying the revision used for that live replica.
-
-Each `NodeModel` also records an `effective_options_hash`, computed from the
-fully materialized `pb.ModelOptions` after node-specific hardware defaults and
-file-path staging rewrites. This hash is diagnostic only. Two replicas may have
-different effective hashes and remain compatible when they share the same
-configuration revision.
-
-Rows created by older versions have an empty revision. They remain usable until
-the model's first revision-aware configuration mutation. Once a current
-revision is recorded, empty-revision rows are stale and cannot be routed.
-
-## Registry invariants
-
-The database is the coordination boundary shared by frontend replicas.
-
-1. At most one current configuration revision exists per logical model name.
-2. A `NodeModel` is routable only when it is in the loaded state and its
- `config_revision` equals the current `ModelConfigState` revision.
-3. A `ModelLoadInfo` row is reconcilable only when its revision equals the
- current `ModelConfigState` revision.
-4. A load job may publish `NodeModel` or `ModelLoadInfo` state only when its
- captured revision still equals the current revision.
-5. Advancing the current revision and quarantining prior-revision replica rows
- happen in one database transaction.
-
-The load-info upsert becomes compare-and-set rather than unconditional
-last-write-wins. If the load's revision is no longer current, the upsert returns
-a typed stale-revision error. The load is then abandoned and its worker process
-is stopped through the exact stop protocol. A late load can therefore neither
-be routed nor overwrite current reconciliation options.
-
-Normal worker death does not change `ModelConfigState` or delete matching
-`ModelLoadInfo`; this preserves restart recovery. A configuration mutation
-advances `ModelConfigState` and invalidates older load information.
-
-## Configuration mutation lifecycle
-
-All model configuration mutation entry points use one model administration
-lifecycle service. The structured PATCH endpoint must no longer bypass local
-shutdown behavior.
-
-For an edit that keeps the same logical model name, the service:
-
-1. Validates and persists the new configuration.
-2. Reloads it and computes its semantic revision.
-3. In one transaction, records the new current revision, marks every replica
- from another or empty revision as `unloading`, and removes or supersedes old
- `ModelLoadInfo`.
-4. Broadcasts the revision-aware invalidation to peer frontends.
-5. Starts cleanup for each quarantined replica using exact `model.stop`.
-6. Deletes a replica row only after confirmed process termination or confirmed
- absence of that exact process.
-
-Marking rows `unloading` precedes network calls. A worker that cannot be reached
-therefore cannot continue receiving inference traffic through LocalAI even if
-its old backend process is still alive.
-
-The configuration save is durable even if cleanup is incomplete. The endpoint
-must not report that saving failed after the new file and revision have
-committed. Its response reports that cleanup is pending, and the condition is
-also logged and exposed through the existing model/node lifecycle status
-surfaces. Subsequent retries finish cleanup.
-
-For rename, the old identity is quarantined and stopped under its old name. The
-new identity receives its own current revision. Old load information is not
-copied to the new name. Disable performs the same quarantine and cleanup but
-does not permit fresh loads while disabled. Delete follows the existing file
-deletion lifecycle after exact process cleanup.
-
-Peer invalidation events carry the logical model name, operation, and new
-revision. Applying an event is idempotent. A peer that already observes that
-revision refreshes its in-memory configuration but does not create a second
-cleanup generation.
-
-When the existing configuration watcher detects an out-of-band file change, it
-computes the revision after validation and submits the same lifecycle
-transition. A parse or validation failure leaves the last accepted revision
-current and does not quarantine its replicas. This does not make filesystem
-writes atomic, but it ensures a successfully observed external edit cannot
-silently bypass revision-aware routing.
-
-## Exact worker process stop
-
-A new request/reply NATS operation, `model.stop`, is separate from the existing
-ambiguous `backend.stop` operation.
-
-The request contains:
-
-```text
-model_name
-process_key
-expected_address
-force
-config_revision
-```
-
-`process_key` is the exact supervisor key, including replica index. The
-controller derives it from the registry row rather than asking the worker to
-resolve a bare backend or model name. `expected_address` prevents a stale row
-from stopping an unrelated process after port reuse. `config_revision` is
-included for auditability; process key and expected address are the worker-side
-identity checks because workers do not own the configuration database.
-
-The reply contains:
-
-```text
-matched
-freed
-terminated
-process_key
-address
-error
-```
-
-The worker verifies that both process key and address identify the same
-supervised process. A mismatched address is an error and never stops anything.
-An absent process is a successful idempotent outcome with `matched=false` and
-`terminated=true` because there is no process left to clean up.
-
-For a graceful request, the worker performs bounded gRPC `Free()` and then
-terminates the supervised process. A `Free()` failure is recorded but does not
-prevent termination. A forced request skips `Free()`. The worker replies only
-after the process has exited and its supervisor bookkeeping and port ownership
-have been updated.
-
-The existing operations retain their meanings:
-
-- `model.unload` calls gRPC `Free()` without promising process termination;
-- `backend.stop` remains an administration and compatibility operation whose
- identifier may be a backend name;
-- `model.stop` is the only operation used to confirm configuration-generation
- cleanup for an exact replica.
-
-Sending both `model.unload` and `model.stop` is unnecessary because graceful
-`model.stop` already performs bounded `Free()` before termination.
-
-## Unreachable workers and retry
-
-An `unloading` replica is never routable. Failed `model.stop` attempts retain
-the row with its last error, attempt count, and next retry time. A bounded,
-backoff-based cleanup loop retries exact stops. Retries are idempotent and are
-claimed through the database so multiple frontend replicas do not concurrently
-own the same attempt.
-
-The existing recovery paths remain backstops:
-
-- Worker re-registration clears all `NodeModel` rows for that node because a
- restarted worker has no surviving supervised backend processes.
-- The per-model health monitor removes rows after consecutive unreachable
- backend probes.
-- Node offline handling prevents scheduling onto a worker with stale
- heartbeats.
-
-Cleanup-row removal through any of these paths fires the existing replica
-removal hooks. It does not restore stale `ModelLoadInfo` because only the
-current revision is eligible for reconciliation.
-
-If a worker keeps heartbeating but does not support `model.stop`, the row stays
-quarantined and the error clearly identifies an incompatible worker version.
-The system favors temporary unavailability over silently serving an obsolete
-configuration. Restarting or upgrading that worker lets re-registration or a
-subsequent retry complete cleanup.
-
-## Reconciliation and loading
-
-The reconciler reads the current revision and matching `ModelLoadInfo` in one
-consistent operation. If no matching load information exists, it does not use
-an older blob. It records a diagnostic explaining that the model must first be
-loaded under its current revision.
-
-The next inference request builds options from the current configuration,
-captures its revision, and performs the normal install, staging, and load
-sequence. On success, it transactionally records the replica and current
-`ModelLoadInfo`. The reconciler may then restore additional `min_replicas`
-using that revision.
-
-Every scheduling and routing decision rechecks revision eligibility when it
-claims a replica. A replica selected immediately before a concurrent edit must
-fail the claim after the edit advances the current revision. Existing in-flight
-requests may finish; no new request is assigned to the old replica. Graceful
-cleanup waits for bounded `Free()` behavior and then terminates it.
-
-## API and observability
-
-Model and node lifecycle responses should expose, where replica details are
-already returned:
-
-- current model `config_revision`;
-- replica `config_revision`;
-- `effective_options_hash`;
-- lifecycle state, including `unloading`;
-- pending cleanup error and retry time.
-
-Logs for routing, reconciliation, load completion, stale-load rejection, and
-cleanup include model name, replica index, node ID, and abbreviated revision.
-No serialized model options or request content is added to logs.
-
-The Web UI does not require a new workflow. After saving, it may show that the
-configuration is saved while one or more old replicas are still being cleaned
-up. User-facing distributed-model documentation explains this state and the
-requirement to upgrade workers that lack acknowledged `model.stop` support.
-
-## Rolling upgrades
-
-Database migrations add nullable revision and cleanup columns so old binaries
-can continue reading existing rows. New frontends treat missing revisions as
-legacy state according to the compatibility rule above.
-
-The new NATS subject avoids changing the semantics of `backend.stop` for old
-workers. A new frontend receiving no responder for `model.stop` leaves the
-replica quarantined and reports the compatibility problem. It must not fall
-back to fire-and-forget `backend.stop`, because doing so would recreate the
-original false-success failure.
-
-Deployments should upgrade workers before or together with frontends. Mixed
-frontend versions are tolerated at the database level, but old frontends do
-not enforce revision-aware routing. Documentation must state that strict
-cross-replica consistency is guaranteed only after all frontend replicas run
-the revision-aware version.
-
-## Testing
-
-All Go tests use Ginkgo and Gomega.
-
-### Registry tests
-
-- Advancing a revision and quarantining old replicas is atomic.
-- Only loaded replicas matching the current revision are returned for routing.
-- Empty legacy revisions become stale after a revision-aware mutation.
-- Load-info compare-and-set rejects a late old-revision write.
-- Matching load information survives ordinary replica removal and worker
- failure.
-- Re-registration removes quarantined rows without changing current revision or
- matching load information.
-
-### Router and reconciler tests
-
-- Given one 8K old-revision replica and one 100K current-revision replica, every
- new request routes to the current revision.
-- Changing `parallel` produces the same revision transition behavior as changing
- `context_size`.
-- The reconciler never loads from stale `ModelLoadInfo`.
-- A late durable load job cannot publish a stale replica or overwrite current
- load information.
-- A request racing a configuration edit cannot claim the old generation.
-- Heterogeneous effective option hashes remain routable when their
- configuration revision matches.
-
-### Worker protocol tests
-
-- Exact process key and address stop the intended process and wait for exit.
-- An address mismatch stops nothing.
-- An already-absent process returns idempotent success.
-- Graceful stop attempts bounded `Free()` and still terminates after a failure.
-- Forced stop skips `Free()`.
-- Replica port ownership and quarantine are updated before replying.
-
-### Lifecycle tests
-
-- Raw YAML edit, structured PATCH, rename, disable, and peer application all
- advance or apply the expected revision and quarantine old replicas.
-- A successful stop deletes the matching row.
-- A timeout leaves a non-routable `unloading` row with retry state.
-- Retry eventually deletes the row after the worker recovers.
-- A worker without `model.stop` support produces a visible compatibility error
- and never triggers fire-and-forget fallback.
-- Partial cleanup does not roll back an already persisted configuration edit.
-
-### Live distributed regression
-
-An integration scenario loads a model on two workers, edits context and
-parallel settings, and verifies that no request is routed to an old revision.
-After cleanup and reload, every replica reports the current revision. The test
-also disconnects one worker during the edit, verifies its replica is
-quarantined, reconnects it, and verifies retry or re-registration removes the
-stale row.
-
-## Documentation impact
-
-The implementation updates the distributed model lifecycle documentation under
-`docs/content/` in the same change. It documents revision consistency,
-quarantined cleanup state, rolling-upgrade requirements, and why an edited model
-may wait for its first request before `min_replicas` can be restored.
-
-No configuration key or public inference API changes are introduced.
diff --git a/docs/superpowers/specs/2026-08-21-distributed-staging-operations-design.md b/docs/superpowers/specs/2026-08-21-distributed-staging-operations-design.md
deleted file mode 100644
index 856340e69..000000000
--- a/docs/superpowers/specs/2026-08-21-distributed-staging-operations-design.md
+++ /dev/null
@@ -1,67 +0,0 @@
-# Distributed Staging Operations Design
-
-## Problem
-
-`GET /api/operations` reads file-transfer progress from the frontend replica's
-in-memory `StagingTracker`. Distributed frontends broadcast tracker updates over
-NATS, but those messages are transient. A replica that starts after staging has
-begun, temporarily disconnects, or misses an update can return no staging row.
-When a browser's one-second polls are balanced across replicas, the operation
-therefore appears and disappears.
-
-Distributed cold loads already persist their phase, placement, heartbeat, and
-byte progress in PostgreSQL's `model_load_jobs` table. That row is the durable
-cluster authority and should provide the baseline operations view.
-
-## Design
-
-Add a `NodeRegistry` query that lists active model-load jobs. The operations
-endpoint will use those jobs to build one staging operation per tracking key
-when the job is in the `staging` phase. It will then overlay matching local or
-NATS-mirrored `StagingTracker` data, because the tracker can contain a fresher
-message and filename than the periodically persisted job.
-
-The merge is keyed by the model tracking key. A tracker entry replaces the
-database entry's progress and display details rather than creating a duplicate.
-Tracker-only entries remain visible for compatibility with staging paths that
-do not have a durable load-job row. Database-only entries remain visible on
-every replica, which eliminates flicker.
-
-The database row supplies:
-
-- stable operation identity (`staging:`),
-- model name and staging phase,
-- node name,
-- overall progress calculated by `ModelLoadJob.Progress()`, and
-- byte counters used by the frontend's ETA calculation.
-
-The tracker overlay supplies its message, filename, node name, progress, and
-byte counters when available.
-
-## Failure Handling
-
-If the database query fails, `/api/operations` will log the error and fall back
-to the current tracker-only response. An observability failure must not break
-the entire operations endpoint or hide unrelated gallery operations.
-
-Only live `staging` rows are included. Pending, backend-installing, loading, and
-failed rows are represented by their existing user-facing flows and must not be
-mislabelled as file staging.
-
-## Testing
-
-Add focused Ginkgo coverage for:
-
-1. A database-only staging job appears in the operations payload, reproducing
- the request landing on a replica that missed all NATS broadcasts.
-2. A matching tracker entry overlays the database entry without duplication.
-3. Non-staging load jobs do not appear as staging operations.
-4. A database read failure retains tracker-only staging operations and the
- endpoint still succeeds.
-
-Run the affected Go package tests only; no long build is required.
-
-## Documentation
-
-This corrects consistency of an existing UI operation and introduces no new
-API, option, or user workflow. No user documentation change is required.
diff --git a/docs/superpowers/specs/2026-08-21-scheduling-rule-editing-node-labels-design.md b/docs/superpowers/specs/2026-08-21-scheduling-rule-editing-node-labels-design.md
deleted file mode 100644
index 950aa7645..000000000
--- a/docs/superpowers/specs/2026-08-21-scheduling-rule-editing-node-labels-design.md
+++ /dev/null
@@ -1,125 +0,0 @@
-# Scheduling Rule Editing and Node Label Reference
-
-## Summary
-
-Improve the React scheduling view so cluster operators can edit existing scheduling rules and inspect node labels without moving back and forth to the Nodes page.
-
-The scheduling page will gain a compact, collapsible node-label reference above the rules table. It will also gain an Edit action that opens the existing scheduling form with the selected rule prefilled. The model name will remain locked while editing because it identifies the rule being updated.
-
-## Goals
-
-- Let operators update an existing scheduling rule in place.
-- Make the labels available on each node visible from the scheduling workflow.
-- Keep the label reference usable for clusters with many nodes.
-- Preserve the existing scheduling API and node API contracts.
-- Keep the scheduling rules usable when node-label loading fails.
-
-## Non-goals
-
-- Editing node labels from the scheduling page.
-- Renaming the model associated with an existing scheduling rule.
-- Adding backend endpoints or changing scheduling semantics.
-- Adding a separate scheduling documentation page for this discoverability enhancement.
-
-## User Experience
-
-### Node label reference
-
-A collapsible **Node labels** section appears above the scheduling rules. It loads node data through the existing `nodesApi.list()` client and groups labels by node so operators can tell which selectors match which machines.
-
-The expanded section contains:
-
-- A fuzzy search field that matches node names, label keys, label values, and complete `key=value` text.
-- A summary showing the visible result count and total matching node count.
-- Node groups containing the node name, operational status, and its `key=value` label chips.
-- Five matching nodes initially.
-- A **Show 20 more** action when additional matches exist.
-
-Changing the search query resets the visible limit to five. Clearing the query restores the unfiltered result set. The reference can be collapsed to preserve vertical space.
-
-The matching implementation should be lightweight and local to the page. It should normalize searchable node data and support forgiving, case-insensitive token matching without adding a large dependency solely for this feature.
-
-### Editing a scheduling rule
-
-Each scheduling-rule row gains an **Edit** action beside **Delete**. Selecting Edit opens the existing scheduling form above the table and populates every editable field from the selected configuration:
-
-- Scheduling mode
-- Node selector
-- Minimum and maximum replicas
-- Routing policy
-- Prefix-cache thresholds
-
-The model selector is replaced by, or presented as, a visibly locked model field while editing. This prevents a rename from creating a second rule while leaving the original in place.
-
-Only one add or edit form may be open at a time. Opening Add clears edit state; opening Edit closes any blank Add form. Cancel closes the form and discards its local changes.
-
-Saving continues to use `nodesApi.setScheduling()`. On success, the page closes the form, shows the existing success toast, and refreshes the scheduling rules. On failure, it shows the error toast and keeps the populated form open so the operator does not lose changes.
-
-## Component Design
-
-### Scheduling form
-
-Refactor `SchedulingForm` to accept an optional existing scheduling configuration. Initial form state will be derived from that configuration, including conversion of a serialized `node_selector` when necessary and derivation of the current mode from `spread_all`, replica values, and selector presence.
-
-The form remains responsible for validation and for producing the existing scheduling request shape. The parent remains responsible for API calls, toast notifications, refreshes, and deciding whether the form is adding or editing.
-
-### Node label reference
-
-Add a focused scheduling-page component for label discovery. It receives node data and owns only presentation state:
-
-- Expanded or collapsed
-- Search query
-- Visible result limit
-
-Small pure helpers will normalize a node's searchable text and calculate filtered results. Node fetching remains in the scheduling page so loading and retry behavior stay next to the existing scheduling fetch lifecycle.
-
-### Styling
-
-Add scheduling-specific classes to `core/http/react-ui/src/App.css`. Reuse existing design-system tokens and button, input, badge, stack, and text primitives. Do not add static inline styles.
-
-On wide screens, node groups use a responsive compact grid. On narrow screens, they collapse to one column. Search, collapse, pagination, and row actions remain keyboard accessible and expose explicit accessible names.
-
-## Data Flow
-
-1. The page mounts and independently requests scheduling configurations and nodes.
-2. Scheduling configurations populate the rules table.
-3. Node data populates the label reference; local search and limiting do not trigger network requests.
-4. Selecting Edit copies one rule into form state and locks its model identity.
-5. Saving posts the existing scheduling payload and refreshes the scheduling list.
-6. Node-label retry repeats only the node request and does not disturb scheduling rules or an open scheduling form.
-
-## States and Error Handling
-
-- **Node loading:** Show a compact loading state inside the reference. Do not block the rules table.
-- **No nodes:** Explain that no nodes are available yet.
-- **Node without labels:** Include it in node-name search results and display **No labels**.
-- **No search matches:** Show a clear empty result while preserving the query.
-- **Node fetch failure:** Show an inline error with Retry. Scheduling remains fully usable.
-- **Malformed selector:** Preserve the current defensive rendering behavior and avoid crashing the edit form; treat an unparseable selector as empty while keeping the rule visible.
-- **Save failure:** Preserve all form values and show the existing error toast.
-- **Save success:** Close the form and refresh the rules.
-
-## Verification
-
-Add or extend a focused Playwright scheduling spec to cover:
-
-- Labels grouped under the correct nodes.
-- Search by node name.
-- Search by complete `key=value` text.
-- Five-node initial limit and **Show 20 more** expansion.
-- Empty-cluster, unlabeled-node, no-match, and failed-loading states.
-- Edit opening with the complete rule prefilled.
-- Locked model identity during editing.
-- Updated values sent through the existing scheduling endpoint.
-- Failed saves preserving the open form.
-
-Run the focused Playwright spec, the React inline-style lint, and the production React build. Long repository-wide builds are outside the scope of this frontend-only change.
-
-## Acceptance Criteria
-
-- An operator can edit and save any existing scheduling rule without deleting and recreating it.
-- The model identity cannot be changed while editing.
-- An operator can inspect labels grouped by node without leaving Scheduling.
-- The label reference remains compact with many nodes and supports forgiving search plus progressive expansion.
-- A node API failure does not prevent viewing or editing scheduling rules.
-- The enhancement works at narrow viewport widths and is keyboard accessible.
diff --git a/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md b/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md
deleted file mode 100644
index e84d36193..000000000
--- a/docs/superpowers/specs/2026-09-07-ephemeral-staging-retention-design.md
+++ /dev/null
@@ -1,158 +0,0 @@
-# Request-owned ephemeral staging
-
-## Problem
-
-Distributed requests copy transient inputs below
-`/ephemeral//`. The worker currently removes
-these files only when a periodic age sweep considers them stale. A Reachy Mini
-sending camera and sound data about once per second created more than 21,000
-request directories and filled its Mac worker before the six-hour retention
-window elapsed.
-
-Reducing the retention window is insufficient. A time limit bounds residence
-time, but the retained bytes still scale with request rate and input size. A
-quota sweeper would also have to infer whether an old file is still in use.
-Neither rule prevents concurrent uploads from consuming the worker's last free
-space.
-
-## Goals
-
-- Give every ephemeral input an explicit owner and release it when that request
- finishes, fails, or is cancelled.
-- Keep cleanup transport-independent for HTTP and S3/NATS workers.
-- Reserve capacity before accepting ephemeral bytes so concurrent requests
- cannot consume configured disk headroom.
-- Reject a request cleanly when its input does not fit; never evict an input
- that a running request may still be reading.
-- Recover abandoned files after frontend or worker crashes.
-- Never inspect or remove models, data, configuration, or paths outside the
- worker's ephemeral staging tree.
-
-## Non-goals
-
-- Retaining request inputs as a cache.
-- Evicting persistent model or data files to make an inference request fit.
-- Treating modification timestamps as proof that a request is active.
-
-## Request ownership
-
-The `FileStagingClient` already creates one request ID before staging inputs and
-waits for synchronous and streaming backend calls to finish. It will track each
-ephemeral key before attempting to stage it and defer one request-scoped
-release identified by the request ID. Release runs after the backend call
-returns, including error and cancellation paths, using a short background
-timeout so cancellation of the request does not cancel its cleanup.
-
-Request IDs will use the full UUID rather than the current eight-character
-prefix. The worker enumerates only category directories for that validated
-request ID and removes each entry with exact, symlink-safe deletion.
-
-`FileStager` will expose an idempotent exact-key `ReleaseRemote` operation and
-an optional request-scoped operation. The client uses one fixed-size request
-message for the normal path and retains exact-key calls as a rolling-upgrade
-fallback:
-
-- HTTP sends one authenticated request containing the fixed-size request ID.
- The worker derives and removes that request's exact files, then prunes empty
- request and category directories without following symlinks.
-- S3/NATS sends one request-reply containing the request ID so the selected
- worker evicts the request's local cached files. The frontend then deletes the
- matching objects from its tracked exact-key list.
- Either deletion may already have happened and still counts as success.
-
-If staging fails partway through a request, the deferred release still includes
-the planned key, allowing it to remove a partial file when the transport can
-identify one. Cleanup errors are logged and do not replace the inference result.
-
-HTTP and S3 ingress register request operations before any pre-reservation
-work. Before enumerating files, the capacity guard marks the request released
-and waits for registered operations and admitted writes to finish. Later
-operations, reservations, and cache claims for that request are rejected.
-Markers expire after one hour and are capped at 16,384 entries, but a marker is
-never evicted while its registered operation or cleanup scan is active.
-Concurrent operation and cleanup state have the same hard cap. Disk bytes
-remain independently bounded by capacity admission.
-Cleanup waits within its deadline when all cleanup-pin slots are occupied.
-If that deadline expires, existing entries lose active ownership so recovery
-can reclaim them; registered ingress for the request remains closed until it
-exits.
-
-## Capacity admission
-
-A worker-local ephemeral capacity guard is shared by its HTTP and S3/NATS input
-paths. It accounts for both `/ephemeral`, used by HTTP, and
-`/ephemeral`, used by S3 downloads. Before writing an ephemeral object,
-the transport reserves its declared size. HTTP obtains the size from the upload
-metadata; S3/NATS obtains it from object metadata. Reservations are serialized
-in memory, cover both committed ephemeral bytes and concurrent writes, and are
-returned on release or failed transfer.
-
-Admission succeeds only when both conditions remain true after the reservation:
-
-1. Total ephemeral bytes remain below the configured ephemeral staging limit.
-2. The filesystem retains the configured minimum free-space headroom.
-
-The guard rejects the transfer before inference when either condition fails.
-An input with unknown size is written through a bounded accounting writer that
-reserves fixed-size chunks before writing each chunk and stops before crossing
-the limit. The existing maximum-upload-size check remains the per-file ceiling.
-
-The limit and headroom are worker settings. By default, ephemeral data may use
-the smaller of 10 GiB or 10 percent of filesystem capacity, while the worker
-preserves the larger of 1 GiB or 5 percent as free-space headroom. The worker
-logs the effective values at startup. A zero or negative operator value selects
-the default rather than disabling protection. The guard scans the ephemeral
-tree at startup to account for abandoned committed bytes. Filesystem free-space
-checks are repeated at reservation time because other processes may share the
-volume.
-
-## Crash recovery
-
-The existing periodic cleanup remains as a fallback for ownership messages lost
-when a frontend or worker process dies. It uses a one-hour recovery TTL,
-performs one startup sweep, and repeats every 15 minutes. It skips every key
-held by an active reservation, considers the newest modification time in each
-remaining request tree, and does not follow directory symlinks. It removes only
-request directories below the registered `/ephemeral` and
-`/ephemeral` roots.
-
-The recovery window does not control normal storage growth. Request completion
-and capacity reservations do. A recovery deletion updates the capacity guard's
-accounted bytes.
-
-## Error handling and observability
-
-Admission failures report the requested bytes, current ephemeral usage, limit,
-available bytes, and required headroom. Successful release and recovery update
-usage counters. Read, stat, and remove failures include the affected path and
-allow unrelated cleanup to continue. Missing ephemeral files and directories
-are normal for idempotent release.
-
-## Testing
-
-Regression tests will establish the following behavior:
-
-1. Successful, failed, cancelled, and streaming calls issue one request-scoped
- worker cleanup only after the backend has returned.
-2. Partial staging failures release the planned key without changing the main
- error returned to the caller.
-3. HTTP and S3/NATS release remove local files; S3/NATS also removes the object.
-4. Release rejects persistent keys and path traversal, does not follow
- symlinks, and leaves paths outside `ephemeral` untouched.
-5. Concurrent reservations cannot exceed the byte limit or free-space
- headroom, and failed transfers return their reservations.
-6. Unknown-length writes stop at the capacity boundary.
-7. Startup accounting includes abandoned ephemeral files, and the recovery
- sweep removes only stale, inactive leftovers and updates accounting.
-
-Focused package tests will run with race detection, followed by the relevant
-repository lint and vet checks.
-
-## Rollout
-
-The change requires a new LocalAI worker and frontend build because both sides
-participate in release. The Mac worker starts by accounting for its existing
-backlog and removing recovery-expired files. The deployment check will verify
-available space, admission and release logs, stable ephemeral usage under
-continuous camera and audio traffic, and successful vision, sound detection,
-and transcription requests.
diff --git a/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md b/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md
deleted file mode 100644
index b4175655f..000000000
--- a/docs/superpowers/specs/2026-09-07-exl3-gallery-design.md
+++ /dev/null
@@ -1,99 +0,0 @@
-# EXL3 gallery entries
-
-## Goal
-
-Add four gallery entries that expose the EXL3 configurations validated or
-tracked by `vllm.cpp`. Pin each Hugging Face artifact to the revision recorded
-by its source or benchmark evidence.
-
-## Entries
-
-### Qwen3.8 target
-
-Add `qwen3.8-27b-exl3-vllm-cpp` for
-`Mia-AiLab/Qwen3.8-27B-EXL3-3.5bpw`. This entry serves the target without a
-draft model.
-
-Use revision `19441ac874c4018295da848e250f23511361cda4`. Configure an 8,192-token
-context, 2,048 cache blocks, eight sequences, and 16,384 batched tokens. Disable
-prefix caching to match the measured serving configuration.
-
-### Qwen3.8 with DFlash2
-
-Add `qwen3.8-27b-dflash2-exl3-vllm-cpp`. This entry stages the Qwen3.8 target
-and `Mia-AiLab/Qwen3.8-27B-DFlash2-EXL3-5.0bpw` at revision
-`4f0436269bca761b071f05319e8e04a87cc633f9`.
-
-Configure the `dflash` method with seven speculative tokens. Use the shipped
-paged draft route. Apply the same serving limits as the target-only entry.
-
-Tag this entry with `dflash` because it enables speculative decoding. Declare
-the target-only entry as its variant. LocalAI can then prefer the faster entry
-when the host supports it.
-
-### DeepSeek V4 Flash for Spark
-
-Add `deepseek-v4-flash-spark-exl3-vllm-cpp` for
-`0xSero/deepseek-v4-flash-0731-spark`. Use the current repository revision,
-`ce5ff0f1efb2e184aafc759d281bfae47d3a359c`. State that the `vllm.cpp`
-runtime record used the older revision `22f28d32b9b29b4352eaa380ff8c2c170b2847ab`.
-
-Describe the entry as a Spark and GB10-oriented REAP-K216 checkpoint. State its
-large memory requirement and CUDA requirement. Do not claim a completed speed
-or correctness gate that the source record does not contain.
-
-### DeepSeek V4 Flash 3.0 bpw
-
-Add `deepseek-v4-flash-exl3-3bpw-vllm-cpp` for
-`0xSero/DeepSeek-V4-Flash-0731-EXL3-3.0bpw`. Use the current repository
-revision, `e0bf84ac76a5100e8790c22ad10b70b1e2d06d71`.
-
-Tag and describe this entry as experimental. The model card states that the
-artifact is structurally complete, but end-to-end generation has not passed.
-Keep this entry separate from the Spark entry because the repositories use
-different layouts and have different runtime evidence.
-
-## Artifact staging
-
-Use LocalAI's Hugging Face artifact source for each repository. Stage complete
-model repositories because these safetensors checkpoints need configuration,
-tokenizer, index, and weight files.
-
-Assign the Qwen draft artifact to a companion target. Pass its staged path in
-the `vllm-cpp` speculative configuration. Do not download files through backend
-startup logic.
-
-## User-visible metadata
-
-Use the `vllm-cpp`, `exl3`, `gpu`, and `cuda` tags on all four entries. Add
-architecture, reasoning, tool-calling, and speculative-decoding tags only when
-the configured model supports them.
-
-Descriptions must distinguish measured results from unresolved work. The Qwen
-DFlash2 description can cite the measured configuration and throughput. The
-DeepSeek descriptions must not imply an end-to-end validation that does not
-exist.
-
-## Validation
-
-Run the gallery schema and focused gallery tests. Add a focused test if the
-artifact or variant structure is not already covered.
-
-Validate these properties:
-
-- Every name is unique.
-- Every variant points to an existing entry.
-- Each Hugging Face source has a pinned revision.
-- The DFlash2 entry stages both repositories and passes the draft path.
-- Only the configured DFlash2 entry has the `dflash` tag.
-- YAML parsing and gallery loading succeed.
-
-No model download or GPU benchmark is part of this LocalAI change. The
-`vllm.cpp` evidence supplies the runtime record.
-
-## Out of scope
-
-- Changes to the `vllm-cpp` backend binaries.
-- New EXL3 kernels or model loaders.
-- New benchmark claims.
-- Gallery entries for unselected EXL3 bit widths.
diff --git a/pkg/grpc/interface.go b/pkg/grpc/interface.go
index 3da403552..c4d6d4967 100644
--- a/pkg/grpc/interface.go
+++ b/pkg/grpc/interface.go
@@ -104,6 +104,16 @@ type AIModelRich interface {
PredictStreamRich(*pb.PredictOptions, chan<- *pb.Reply) error
}
+// AIModelRichContext is an optional extension to AIModelRich for backends
+// whose work outlives a plain function call, such as a proxy waiting on a
+// remote server. The gRPC server prefers it and passes the call's context, so
+// a caller that disconnects or gives up stops the work instead of letting it
+// run to the end. The channel contract is the same as PredictStreamRich.
+type AIModelRichContext interface {
+ PredictRichContext(context.Context, *pb.PredictOptions) (*pb.Reply, error)
+ PredictStreamRichContext(context.Context, *pb.PredictOptions, chan<- *pb.Reply) error
+}
+
// ClassifyModel is an optional extension to AIModel for backends that
// implement the TokenClassify RPC (zero-shot NER). The gRPC server
// type-asserts to this interface; backends that do not implement it
@@ -123,3 +133,13 @@ type ClassifyModel interface {
type ScoreModel interface {
Score(context.Context, *pb.ScoreRequest) (*pb.ScoreResponse, error)
}
+
+// RerankModel is an optional extension to AIModel for backends that
+// implement the Rerank RPC (candidate document reranking against a query).
+// The gRPC server type-asserts to this interface; backends that do not
+// implement it fall through to the UnimplementedBackendServer default. This
+// mirrors the ScoreModel pattern: adding a method to AIModel itself would
+// break every backend, so the capability is opt-in.
+type RerankModel interface {
+ Rerank(context.Context, *pb.RerankRequest) (*pb.RerankResult, error)
+}
diff --git a/pkg/grpc/model_identity_modalities_test.go b/pkg/grpc/model_identity_modalities_test.go
index b96db9ed5..a166c77bf 100644
--- a/pkg/grpc/model_identity_modalities_test.go
+++ b/pkg/grpc/model_identity_modalities_test.go
@@ -113,9 +113,12 @@ var _ AIModel = (*modalityBackend)(nil)
// server implements, sending `identity` in each request's ModelIdentity field.
// One call per RPC, so the returned error count is also the RPC count.
//
-// Rerank, Score and TokenClassify are absent on purpose: the generic Go server
-// does not implement them (they fall through to UnimplementedBackendServer), so
-// only the C++ and Python backends can enforce them.
+// Score and TokenClassify are absent on purpose: the generic Go server does
+// not implement them (they fall through to UnimplementedBackendServer), so
+// only the C++ and Python backends can enforce them. Rerank is different: the
+// Go server does serve it (see server_rerank_test.go), but only for backends
+// that opt in via RerankModel — modalityBackend does not, so it is left out
+// here too rather than adding a no-op implementation with nothing to guard.
func callAllModalities(c Backend, identity string) map[string]error {
ctx := context.Background()
errs := map[string]error{}
diff --git a/pkg/grpc/rich_context_test.go b/pkg/grpc/rich_context_test.go
new file mode 100644
index 000000000..b7dee0bdf
--- /dev/null
+++ b/pkg/grpc/rich_context_test.go
@@ -0,0 +1,75 @@
+package grpc
+
+import (
+ "context"
+ "time"
+
+ "github.com/mudler/LocalAI/pkg/grpc/base"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+// ctxBackend reports the context each rich call receives, so a spec can check
+// that cancelling the caller reaches the backend.
+type ctxBackend struct {
+ base.SingleThread
+ seen chan context.Context
+}
+
+func (b *ctxBackend) PredictRich(opts *pb.PredictOptions) (*pb.Reply, error) {
+ return b.PredictRichContext(context.Background(), opts)
+}
+
+func (b *ctxBackend) PredictStreamRich(opts *pb.PredictOptions, out chan<- *pb.Reply) error {
+ return b.PredictStreamRichContext(context.Background(), opts, out)
+}
+
+func (b *ctxBackend) PredictRichContext(ctx context.Context, _ *pb.PredictOptions) (*pb.Reply, error) {
+ b.seen <- ctx
+ <-ctx.Done()
+ return nil, ctx.Err()
+}
+
+func (b *ctxBackend) PredictStreamRichContext(ctx context.Context, _ *pb.PredictOptions, _ chan<- *pb.Reply) error {
+ b.seen <- ctx
+ <-ctx.Done()
+ return ctx.Err()
+}
+
+var _ AIModelRichContext = (*ctxBackend)(nil)
+
+var _ = Describe("AIModelRichContext dispatch", func() {
+ // A backend that only saw context.Background would block forever here,
+ // so bound each call and check the backend returned because of the cancel.
+ expectCancelReaches := func(call func(ctx context.Context) error, seen chan context.Context) {
+ ctx, cancel := context.WithCancel(context.Background())
+ errCh := make(chan error, 1)
+ go func() { errCh <- call(ctx) }()
+
+ var got context.Context
+ Eventually(seen).Should(Receive(&got))
+ cancel()
+ Eventually(got.Done(), 2*time.Second).Should(BeClosed())
+ Eventually(errCh, 2*time.Second).Should(Receive(HaveOccurred()))
+ }
+
+ It("passes the caller's context to PredictStreamRichContext", func() {
+ b := &ctxBackend{seen: make(chan context.Context, 1)}
+ Provide("test://rich-ctx-stream", b)
+ c := NewClient("test://rich-ctx-stream", true, nil, false)
+ expectCancelReaches(func(ctx context.Context) error {
+ return c.PredictStream(ctx, &pb.PredictOptions{}, func(*pb.Reply) {})
+ }, b.seen)
+ })
+
+ It("passes the caller's context to PredictRichContext", func() {
+ b := &ctxBackend{seen: make(chan context.Context, 1)}
+ Provide("test://rich-ctx-predict", b)
+ c := NewClient("test://rich-ctx-predict", true, nil, false)
+ expectCancelReaches(func(ctx context.Context) error {
+ _, err := c.Predict(ctx, &pb.PredictOptions{})
+ return err
+ }, b.seen)
+ })
+})
diff --git a/pkg/grpc/server.go b/pkg/grpc/server.go
index 40eec06fe..4608ae748 100644
--- a/pkg/grpc/server.go
+++ b/pkg/grpc/server.go
@@ -132,6 +132,21 @@ func (s *server) Score(ctx context.Context, in *pb.ScoreRequest) (*pb.ScoreRespo
return sm.Score(ctx, in)
}
+func (s *server) Rerank(ctx context.Context, in *pb.RerankRequest) (*pb.RerankResult, error) {
+ if err := s.checkModelIdentity(in); err != nil {
+ return nil, err
+ }
+ rm, ok := s.llm.(RerankModel)
+ if !ok {
+ return nil, status.Errorf(codes.Unimplemented, "method Rerank not implemented")
+ }
+ if s.llm.Locking() {
+ s.llm.Lock()
+ defer s.llm.Unlock()
+ }
+ return rm.Rerank(ctx, in)
+}
+
func (s *server) LoadModel(ctx context.Context, in *pb.ModelOptions) (*pb.Result, error) {
if s.llm.Locking() {
s.llm.Lock()
@@ -161,6 +176,9 @@ func (s *server) Predict(ctx context.Context, in *pb.PredictOptions) (*pb.Reply,
s.llm.Lock()
defer s.llm.Unlock()
}
+ if rich, ok := s.llm.(AIModelRichContext); ok {
+ return rich.PredictRichContext(ctx, in)
+ }
if rich, ok := s.llm.(AIModelRich); ok {
return rich.PredictRich(in)
}
@@ -565,7 +583,12 @@ func (s *server) PredictStream(in *pb.PredictOptions, stream pb.Backend_PredictS
// Server-side close: PredictStreamRich implementations send into
// the channel and return when finished; closing is the host's
// concern so impls don't have to remember `defer close(...)`.
- err := rich.PredictStreamRich(in, replyChan)
+ var err error
+ if withCtx, ok := s.llm.(AIModelRichContext); ok {
+ err = withCtx.PredictStreamRichContext(stream.Context(), in, replyChan)
+ } else {
+ err = rich.PredictStreamRich(in, replyChan)
+ }
close(replyChan)
<-done
return err
@@ -1083,7 +1106,7 @@ func NewBackendServer(model AIModel) pb.BackendServer {
}
// AuthTokenEnvVar is the environment variable used to configure gRPC bearer token auth.
-const AuthTokenEnvVar = "LOCALAI_GRPC_AUTH_TOKEN"
+const AuthTokenEnvVar = "LOCALAI_GRPC_AUTH_TOKEN" // #nosec G101 -- the name of an environment variable, not a credential
// validateToken extracts the bearer token from gRPC metadata and validates it.
func validateToken(ctx context.Context, expected string) error {
diff --git a/pkg/grpc/server_rerank_test.go b/pkg/grpc/server_rerank_test.go
new file mode 100644
index 000000000..d995b6b12
--- /dev/null
+++ b/pkg/grpc/server_rerank_test.go
@@ -0,0 +1,49 @@
+package grpc
+
+import (
+ "context"
+
+ "github.com/mudler/LocalAI/pkg/grpc/base"
+ pb "github.com/mudler/LocalAI/pkg/grpc/proto"
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "google.golang.org/grpc/codes"
+ grpcstatus "google.golang.org/grpc/status"
+)
+
+// rerankBackend implements RerankModel on top of the minimal AIModel surface,
+// mirroring how a Go backend would opt into Score today.
+type rerankBackend struct {
+ base.SingleThread
+}
+
+func (b *rerankBackend) Rerank(context.Context, *pb.RerankRequest) (*pb.RerankResult, error) {
+ return &pb.RerankResult{
+ Results: []*pb.DocumentResult{{Index: 0, RelevanceScore: 1}},
+ }, nil
+}
+
+var _ AIModel = (*rerankBackend)(nil)
+var _ RerankModel = (*rerankBackend)(nil)
+
+var _ = Describe("Rerank", func() {
+ It("is served when the backend implements RerankModel", func() {
+ Provide("test://rerank-served", &rerankBackend{})
+ c := NewClient("test://rerank-served", true, nil, false)
+
+ res, err := c.Rerank(context.Background(), &pb.RerankRequest{Query: "q", Documents: []string{"a", "b"}})
+ Expect(err).ToNot(HaveOccurred())
+ Expect(res.Results).To(HaveLen(1))
+ })
+
+ It("reports Unimplemented when the backend does not implement RerankModel", func() {
+ Provide("test://rerank-unimplemented", &base.SingleThread{})
+ c := NewClient("test://rerank-unimplemented", true, nil, false)
+
+ _, err := c.Rerank(context.Background(), &pb.RerankRequest{Query: "q"})
+ Expect(err).To(HaveOccurred())
+ st, ok := grpcstatus.FromError(err)
+ Expect(ok).To(BeTrue())
+ Expect(st.Code()).To(Equal(codes.Unimplemented))
+ })
+})
diff --git a/pkg/mcp/localaitools/client.go b/pkg/mcp/localaitools/client.go
index bf683a508..7cba8a8de 100644
--- a/pkg/mcp/localaitools/client.go
+++ b/pkg/mcp/localaitools/client.go
@@ -130,4 +130,15 @@ type LocalAIClient interface {
// ClearRouterCorpus wipes a knn router's corpus — file and live
// index.
ClearRouterCorpus(ctx context.Context, routerModel string) (*RouterCorpusClearResult, error)
+
+ // ---- Failover chains ----
+ // ListFailoverChains reports every configured failover chain, its
+ // currently active target, and the health of each target.
+ ListFailoverChains(ctx context.Context) ([]FailoverChainInfo, error)
+ // PinFailoverTarget forces chain to serve every request from target,
+ // regardless of health, until unpinned.
+ PinFailoverTarget(ctx context.Context, chain, target string) error
+ // UnpinFailoverTarget removes chain's pin so health decides the
+ // active target again.
+ UnpinFailoverTarget(ctx context.Context, chain string) error
}
diff --git a/pkg/mcp/localaitools/coverage_test.go b/pkg/mcp/localaitools/coverage_test.go
index a802226f0..75831769e 100644
--- a/pkg/mcp/localaitools/coverage_test.go
+++ b/pkg/mcp/localaitools/coverage_test.go
@@ -46,27 +46,30 @@ var toolToHTTPRoute = map[string]string{
ToolGetRouterCorpusStats: "GET /api/router/:name/corpus/stats",
ToolListAliases: "GET /api/aliases",
ToolListVoiceProfiles: "GET /api/voice-profiles",
+ ToolListFailoverChains: "GET /api/failover",
// Mutating tools.
- ToolInstallModel: "POST /models/apply",
- ToolImportModelURI: "POST /models/import-uri",
- ToolDeleteModel: "POST /models/delete/:name",
- ToolEditModelConfig: "PATCH /api/models/config-json/:name",
- ToolReloadModels: "POST /models/reload",
- ToolLoadModel: "POST /backend/load",
- ToolInstallBackend: "POST /backends/apply",
- ToolUpgradeBackend: "POST /backends/upgrade/:name",
- ToolToggleModelState: "PUT /models/toggle-state/:name/:action",
- ToolToggleModelPinned: "PUT /models/toggle-pinned/:name/:action",
- ToolSetBranding: "POST /api/settings (instance_name, instance_tagline)",
- ToolSetAlias: "PATCH /api/models/config-json/:name (swap) or POST /models/import (create)",
- ToolSeedRouterCorpus: "POST /api/router/:name/corpus",
- ToolClearRouterCorpus: "DELETE /api/router/:name/corpus",
- ToolCreateVoiceProfile: "POST /api/voice-profiles",
- ToolDeleteVoiceProfile: "DELETE /api/voice-profiles/:id",
- ToolSetNodeVRAMBudget: "PUT /api/nodes/:id/vram-budget",
- ToolSetScheduling: "POST /api/nodes/scheduling",
- ToolDeleteScheduling: "DELETE /api/nodes/scheduling/:model",
+ ToolInstallModel: "POST /models/apply",
+ ToolImportModelURI: "POST /models/import-uri",
+ ToolDeleteModel: "POST /models/delete/:name",
+ ToolEditModelConfig: "PATCH /api/models/config-json/:name",
+ ToolReloadModels: "POST /models/reload",
+ ToolLoadModel: "POST /backend/load",
+ ToolInstallBackend: "POST /backends/apply",
+ ToolUpgradeBackend: "POST /backends/upgrade/:name",
+ ToolToggleModelState: "PUT /models/toggle-state/:name/:action",
+ ToolToggleModelPinned: "PUT /models/toggle-pinned/:name/:action",
+ ToolSetBranding: "POST /api/settings (instance_name, instance_tagline)",
+ ToolSetAlias: "PATCH /api/models/config-json/:name (swap) or POST /models/import (create)",
+ ToolSeedRouterCorpus: "POST /api/router/:name/corpus",
+ ToolClearRouterCorpus: "DELETE /api/router/:name/corpus",
+ ToolCreateVoiceProfile: "POST /api/voice-profiles",
+ ToolDeleteVoiceProfile: "DELETE /api/voice-profiles/:id",
+ ToolSetNodeVRAMBudget: "PUT /api/nodes/:id/vram-budget",
+ ToolSetScheduling: "POST /api/nodes/scheduling",
+ ToolDeleteScheduling: "DELETE /api/nodes/scheduling/:model",
+ ToolPinFailoverTarget: "POST /api/failover/:chain/pin",
+ ToolUnpinFailoverTarget: "DELETE /api/failover/:chain/pin",
}
// allKnownTools is the union of expectedFullCatalog (defined in
diff --git a/pkg/mcp/localaitools/dto.go b/pkg/mcp/localaitools/dto.go
index 24da8aa18..afd19bafd 100644
--- a/pkg/mcp/localaitools/dto.go
+++ b/pkg/mcp/localaitools/dto.go
@@ -413,3 +413,26 @@ type VRAMEstimateRequest struct {
GPULayers int `json:"gpu_layers,omitempty" jsonschema:"Number of layers to offload to GPU. -1 for all."`
KVQuantBits int `json:"kv_quant_bits,omitempty" jsonschema:"KV cache quantization bits (e.g. 4, 8, 16)."`
}
+
+// FailoverTargetInfo is the LLM-facing view of one failover chain target's
+// health. It mirrors failover.TargetStatus but drops ConsecutiveOK and
+// LastProbe — internal probing detail the LLM doesn't need to decide
+// whether to pin or unpin a target.
+type FailoverTargetInfo struct {
+ Model string `json:"model"`
+ Kind string `json:"kind"`
+ Warm bool `json:"warm"`
+ State string `json:"state"`
+ LastError string `json:"last_error,omitempty"`
+}
+
+// FailoverChainInfo is the LLM-facing view of one failover chain: its
+// current state, the target serving it now, an optional pin, and every
+// target's health.
+type FailoverChainInfo struct {
+ Name string `json:"name"`
+ State string `json:"state"`
+ Active string `json:"active"`
+ Pinned string `json:"pinned,omitempty"`
+ Targets []FailoverTargetInfo `json:"targets"`
+}
diff --git a/pkg/mcp/localaitools/fakes_test.go b/pkg/mcp/localaitools/fakes_test.go
index b0697b272..cba6efa57 100644
--- a/pkg/mcp/localaitools/fakes_test.go
+++ b/pkg/mcp/localaitools/fakes_test.go
@@ -59,6 +59,9 @@ type fakeClient struct {
getPIIEvents func(PIIEventsQuery) ([]PIIEvent, error)
getMiddlewareStatus func() (*MiddlewareStatus, error)
getRouterDecisions func(RouterDecisionsQuery) ([]RouterDecision, error)
+ listFailoverChains func() ([]FailoverChainInfo, error)
+ pinFailoverTarget func(string, string) error
+ unpinFailoverTarget func(string) error
}
type fakeCall struct {
@@ -393,3 +396,27 @@ func (f *fakeClient) ClearRouterCorpus(_ context.Context, routerModel string) (*
f.record("ClearRouterCorpus", routerModel)
return &RouterCorpusClearResult{Router: routerModel}, nil
}
+
+func (f *fakeClient) ListFailoverChains(_ context.Context) ([]FailoverChainInfo, error) {
+ f.record("ListFailoverChains", nil)
+ if f.listFailoverChains != nil {
+ return f.listFailoverChains()
+ }
+ return []FailoverChainInfo{}, nil
+}
+
+func (f *fakeClient) PinFailoverTarget(_ context.Context, chain, target string) error {
+ f.record("PinFailoverTarget", []any{chain, target})
+ if f.pinFailoverTarget != nil {
+ return f.pinFailoverTarget(chain, target)
+ }
+ return nil
+}
+
+func (f *fakeClient) UnpinFailoverTarget(_ context.Context, chain string) error {
+ f.record("UnpinFailoverTarget", chain)
+ if f.unpinFailoverTarget != nil {
+ return f.unpinFailoverTarget(chain)
+ }
+ return nil
+}
diff --git a/pkg/mcp/localaitools/httpapi/client.go b/pkg/mcp/localaitools/httpapi/client.go
index 923f35eba..631977c03 100644
--- a/pkg/mcp/localaitools/httpapi/client.go
+++ b/pkg/mcp/localaitools/httpapi/client.go
@@ -829,3 +829,23 @@ func (c *Client) ClearRouterCorpus(ctx context.Context, routerModel string) (*lo
}
return &out, nil
}
+
+// ---- Failover chains ----
+
+func (c *Client) ListFailoverChains(ctx context.Context) ([]localaitools.FailoverChainInfo, error) {
+ var out struct {
+ Chains []localaitools.FailoverChainInfo `json:"chains"`
+ }
+ if err := c.do(ctx, http.MethodGet, routeFailover, nil, &out); err != nil {
+ return nil, err
+ }
+ return out.Chains, nil
+}
+
+func (c *Client) PinFailoverTarget(ctx context.Context, chain, target string) error {
+ return c.do(ctx, http.MethodPost, routeFailover+"/"+url.PathEscape(chain)+"/pin", map[string]string{"target": target}, nil)
+}
+
+func (c *Client) UnpinFailoverTarget(ctx context.Context, chain string) error {
+ return c.do(ctx, http.MethodDelete, routeFailover+"/"+url.PathEscape(chain)+"/pin", nil, nil)
+}
diff --git a/pkg/mcp/localaitools/httpapi/client_test.go b/pkg/mcp/localaitools/httpapi/client_test.go
index c4e596650..fd1cff632 100644
--- a/pkg/mcp/localaitools/httpapi/client_test.go
+++ b/pkg/mcp/localaitools/httpapi/client_test.go
@@ -365,6 +365,68 @@ var _ = Describe("Model aliases", func() {
})
})
+var _ = Describe("Failover chains", func() {
+ Describe("ListFailoverChains", func() {
+ It("issues GET /api/failover and unwraps the chains array", func() {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ Expect(r.Method).To(Equal(http.MethodGet))
+ Expect(r.URL.Path).To(Equal("/api/failover"))
+ _ = json.NewEncoder(w).Encode(map[string]any{
+ "chains": []map[string]any{
+ {
+ "name": "chain",
+ "state": "primary",
+ "active": "a",
+ "pinned": nil,
+ "targets": []map[string]any{
+ {"model": "a", "kind": "local", "warm": false, "state": "healthy"},
+ },
+ },
+ },
+ })
+ }))
+ DeferCleanup(srv.Close)
+
+ out, err := New(srv.URL, "").ListFailoverChains(context.Background())
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out).To(HaveLen(1))
+ Expect(out[0].Name).To(Equal("chain"))
+ Expect(out[0].Active).To(Equal("a"))
+ Expect(out[0].Pinned).To(BeEmpty())
+ Expect(out[0].Targets).To(ConsistOf(localaitools.FailoverTargetInfo{Model: "a", Kind: "local", State: "healthy"}))
+ })
+ })
+
+ Describe("PinFailoverTarget", func() {
+ It("issues POST /api/failover/:chain/pin with the target in the body", func() {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ Expect(r.Method).To(Equal(http.MethodPost))
+ Expect(r.URL.Path).To(Equal("/api/failover/chain/pin"))
+ var body map[string]string
+ Expect(json.NewDecoder(r.Body).Decode(&body)).To(Succeed())
+ Expect(body).To(HaveKeyWithValue("target", "b"))
+ w.WriteHeader(http.StatusOK)
+ }))
+ DeferCleanup(srv.Close)
+
+ Expect(New(srv.URL, "").PinFailoverTarget(context.Background(), "chain", "b")).To(Succeed())
+ })
+ })
+
+ Describe("UnpinFailoverTarget", func() {
+ It("issues DELETE /api/failover/:chain/pin", func() {
+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ Expect(r.Method).To(Equal(http.MethodDelete))
+ Expect(r.URL.Path).To(Equal("/api/failover/chain/pin"))
+ w.WriteHeader(http.StatusOK)
+ }))
+ DeferCleanup(srv.Close)
+
+ Expect(New(srv.URL, "").UnpinFailoverTarget(context.Background(), "chain")).To(Succeed())
+ })
+ })
+})
+
var _ = Describe("ErrHTTPNotFound", func() {
Context("on a clean 404 status", func() {
var (
diff --git a/pkg/mcp/localaitools/httpapi/routes.go b/pkg/mcp/localaitools/httpapi/routes.go
index fca25efad..229e6d0ff 100644
--- a/pkg/mcp/localaitools/httpapi/routes.go
+++ b/pkg/mcp/localaitools/httpapi/routes.go
@@ -34,6 +34,7 @@ const (
routeMiddleware = "/api/middleware/status"
routeRouterDecisions = "/api/router/decisions"
routeVoiceProfiles = "/api/voice-profiles"
+ routeFailover = "/api/failover"
)
func routeJobStatus(jobID string) string {
diff --git a/pkg/mcp/localaitools/inproc/client.go b/pkg/mcp/localaitools/inproc/client.go
index 207956c56..ff80b6cac 100644
--- a/pkg/mcp/localaitools/inproc/client.go
+++ b/pkg/mcp/localaitools/inproc/client.go
@@ -21,6 +21,7 @@ import (
"github.com/mudler/LocalAI/core/gallery/importers"
"github.com/mudler/LocalAI/core/http/auth"
"github.com/mudler/LocalAI/core/schema"
+ "github.com/mudler/LocalAI/core/services/failover"
"github.com/mudler/LocalAI/core/services/galleryop"
"github.com/mudler/LocalAI/core/services/modeladmin"
"github.com/mudler/LocalAI/core/services/nodes"
@@ -83,6 +84,12 @@ type Client struct {
RouterEmbedderFingerprint func(modelName string) (string, error)
RouterVectorStore func(storeName string) backend.VectorStore
+ // Failover backs list_failover_chains / pin_failover_target /
+ // unpin_failover_target. nil makes the tools report "failover is not
+ // running" — the same as a deployment with no failover chains
+ // configured.
+ Failover *failover.Manager
+
modelAdmin *modeladmin.ConfigService
}
@@ -1001,7 +1008,7 @@ func (c *Client) GetMiddlewareStatus(ctx context.Context) (*localaitools.Middlew
EnabledGlobally: c.PIIEvents != nil,
Models: []localaitools.MiddlewarePIIModel{},
}
- piiSection.DefaultEnabledForBackends = []string{"cloud-proxy"}
+ piiSection.DefaultEnabledForBackends = []string{model.CloudProxyBackend}
if c.ConfigLoader != nil {
for _, cfg := range c.ConfigLoader.GetAllModelsConfigs() {
cfg := cfg
@@ -1010,7 +1017,7 @@ func (c *Client) GetMiddlewareStatus(ctx context.Context) (*localaitools.Middlew
Backend: cfg.Backend,
Enabled: cfg.PIIIsEnabled(),
Explicit: cfg.PII.Enabled != nil,
- DefaultForBackend: cfg.Backend == "cloud-proxy",
+ DefaultForBackend: cfg.Backend == model.CloudProxyBackend,
Detectors: cfg.PIIDetectors(),
})
}
@@ -1103,3 +1110,39 @@ func (c *Client) ClearRouterCorpus(ctx context.Context, routerModel string) (*lo
}
return &localaitools.RouterCorpusClearResult{Router: cfg.Name, Cleared: cleared}, nil
}
+
+// ---- Failover chains ----
+
+func (c *Client) ListFailoverChains(_ context.Context) ([]localaitools.FailoverChainInfo, error) {
+ out := []localaitools.FailoverChainInfo{}
+ if c.Failover == nil {
+ return out, nil
+ }
+ for _, ch := range c.Failover.Status() {
+ info := localaitools.FailoverChainInfo{Name: ch.Name, State: string(ch.State), Active: ch.Active}
+ if ch.Pinned != nil {
+ info.Pinned = *ch.Pinned
+ }
+ for _, t := range ch.Targets {
+ info.Targets = append(info.Targets, localaitools.FailoverTargetInfo{
+ Model: t.Model, Kind: string(t.Kind), Warm: t.Warm, State: string(t.State), LastError: t.LastError,
+ })
+ }
+ out = append(out, info)
+ }
+ return out, nil
+}
+
+func (c *Client) PinFailoverTarget(_ context.Context, chain, target string) error {
+ if c.Failover == nil {
+ return errors.New("failover is not running")
+ }
+ return c.Failover.Pin(chain, target)
+}
+
+func (c *Client) UnpinFailoverTarget(_ context.Context, chain string) error {
+ if c.Failover == nil {
+ return errors.New("failover is not running")
+ }
+ return c.Failover.Unpin(chain)
+}
diff --git a/pkg/mcp/localaitools/inproc/client_test.go b/pkg/mcp/localaitools/inproc/client_test.go
index 38a05a8a7..0a9e6a5b7 100644
--- a/pkg/mcp/localaitools/inproc/client_test.go
+++ b/pkg/mcp/localaitools/inproc/client_test.go
@@ -13,6 +13,7 @@ import (
"github.com/mudler/LocalAI/core/config"
"github.com/mudler/LocalAI/core/gallery"
+ "github.com/mudler/LocalAI/core/services/failover"
"github.com/mudler/LocalAI/core/services/galleryop"
"github.com/mudler/LocalAI/core/services/nodes"
localaitools "github.com/mudler/LocalAI/pkg/mcp/localaitools"
@@ -129,6 +130,95 @@ var _ = Describe("inproc.Client model aliases", func() {
})
})
+// fakeFailoverSource is a minimal failover.ConfigSource over an in-memory
+// map, so these specs don't need a real ModelConfigLoader + on-disk YAML.
+type fakeFailoverSource struct {
+ cfgs map[string]config.ModelConfig
+}
+
+func (s *fakeFailoverSource) GetModelConfig(name string) (config.ModelConfig, bool) {
+ c, ok := s.cfgs[name]
+ return c, ok
+}
+
+func (s *fakeFailoverSource) GetAllModelsConfigs() []config.ModelConfig {
+ out := make([]config.ModelConfig, 0, len(s.cfgs))
+ for _, c := range s.cfgs {
+ out = append(out, c)
+ }
+ return out
+}
+
+var _ = Describe("inproc.Client failover chains", func() {
+ var (
+ ctx context.Context
+ c *Client
+ fm *failover.Manager
+ )
+
+ BeforeEach(func() {
+ ctx = context.Background()
+ src := &fakeFailoverSource{cfgs: map[string]config.ModelConfig{
+ "a": {Name: "a", Backend: "llama-cpp"},
+ "b": {Name: "b", Backend: "llama-cpp"},
+ "chain": {Name: "chain", Failover: &config.FailoverConfig{
+ Targets: []config.FailoverTarget{{Model: "a"}, {Model: "b"}},
+ }},
+ }}
+ fm = failover.New(src)
+ c = &Client{Failover: fm}
+ })
+
+ It("ListFailoverChains reports the chain, its active target, and target health", func() {
+ out, err := c.ListFailoverChains(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out).To(HaveLen(1))
+ Expect(out[0].Name).To(Equal("chain"))
+ Expect(out[0].Active).To(Equal("a"))
+ Expect(out[0].Pinned).To(BeEmpty())
+ Expect(out[0].Targets).To(HaveLen(2))
+ Expect(out[0].Targets[0].Model).To(Equal("a"))
+ Expect(out[0].Targets[0].Kind).To(Equal("local"))
+ Expect(out[0].Targets[0].State).To(Equal("healthy"))
+ })
+
+ It("returns an empty slice, not an error, when no failover manager is wired", func() {
+ c = &Client{}
+ out, err := c.ListFailoverChains(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out).To(BeEmpty())
+ })
+
+ It("PinFailoverTarget pins the chain and ListFailoverChains reflects it", func() {
+ Expect(c.PinFailoverTarget(ctx, "chain", "b")).To(Succeed())
+
+ out, err := c.ListFailoverChains(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out[0].Pinned).To(Equal("b"))
+ })
+
+ It("PinFailoverTarget errors when the failover manager is unavailable", func() {
+ c = &Client{}
+ err := c.PinFailoverTarget(ctx, "chain", "b")
+ Expect(err).To(HaveOccurred())
+ })
+
+ It("UnpinFailoverTarget clears a pin", func() {
+ Expect(c.PinFailoverTarget(ctx, "chain", "b")).To(Succeed())
+ Expect(c.UnpinFailoverTarget(ctx, "chain")).To(Succeed())
+
+ out, err := c.ListFailoverChains(ctx)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(out[0].Pinned).To(BeEmpty())
+ })
+
+ It("UnpinFailoverTarget errors when the failover manager is unavailable", func() {
+ c = &Client{}
+ err := c.UnpinFailoverTarget(ctx, "chain")
+ Expect(err).To(HaveOccurred())
+ })
+})
+
var _ = Describe("inproc.Client model scheduling", func() {
var (
ctx context.Context
diff --git a/pkg/mcp/localaitools/prompts/10_safety.md b/pkg/mcp/localaitools/prompts/10_safety.md
index 15671fdc4..5f9c5f840 100644
--- a/pkg/mcp/localaitools/prompts/10_safety.md
+++ b/pkg/mcp/localaitools/prompts/10_safety.md
@@ -2,7 +2,7 @@
These rules are non-negotiable. The user trusts you to operate their server without unintended changes.
-1. **Confirm before mutating.** Before calling any of these tools — `install_model`, `import_model_uri`, `delete_model`, `install_backend`, `upgrade_backend`, `edit_model_config`, `reload_models`, `load_model`, `toggle_model_state`, `toggle_model_pinned`, `set_branding`, `set_alias`, `seed_router_corpus`, `clear_router_corpus`, `create_voice_profile`, `delete_voice_profile`, `set_node_vram_budget`, `set_scheduling`, `delete_scheduling` — first state in plain language what you are about to do (which tool, which target, which arguments) and wait for the user's explicit confirmation in the next turn. "Yes", "do it", "go ahead", "proceed" all count as confirmation. Anything else does not.
+1. **Confirm before mutating.** Before calling any of these tools — `install_model`, `import_model_uri`, `delete_model`, `install_backend`, `upgrade_backend`, `edit_model_config`, `reload_models`, `load_model`, `toggle_model_state`, `toggle_model_pinned`, `set_branding`, `set_alias`, `seed_router_corpus`, `clear_router_corpus`, `create_voice_profile`, `delete_voice_profile`, `set_node_vram_budget`, `set_scheduling`, `delete_scheduling`, `pin_failover_target`, `unpin_failover_target` — first state in plain language what you are about to do (which tool, which target, which arguments) and wait for the user's explicit confirmation in the next turn. "Yes", "do it", "go ahead", "proceed" all count as confirmation. Anything else does not.
2. **Disambiguate before mutating.** If the user's request is ambiguous (several gallery candidates match, the model name has multiple installed versions, the backend has variants), present the candidates as a numbered list and ask the user to pick before calling any mutating tool.
diff --git a/pkg/mcp/localaitools/prompts/20_tools.md b/pkg/mcp/localaitools/prompts/20_tools.md
index 2a2830417..88404801e 100644
--- a/pkg/mcp/localaitools/prompts/20_tools.md
+++ b/pkg/mcp/localaitools/prompts/20_tools.md
@@ -24,6 +24,7 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the
- `get_router_decisions` — Inspect recent router decisions and classifier signals.
- `get_router_corpus_stats` — Inspect a KNN router corpus by count and label only; exemplar texts are never returned.
- `list_aliases` — List configured model aliases and their targets.
+- `list_failover_chains` — List failover chains, their active target and target health.
## Mutating (require user confirmation per safety rule 1)
@@ -46,3 +47,5 @@ The MCP `tools/list` endpoint also exposes the full input schema for each of the
- `set_node_vram_budget` — Set or clear a federated node's VRAM budget override.
- `set_scheduling` — Create or update a distributed per-model scheduling config.
- `delete_scheduling` — Remove a distributed per-model scheduling config.
+- `pin_failover_target` — Force a failover chain to one target.
+- `unpin_failover_target` — Remove a failover pin.
diff --git a/pkg/mcp/localaitools/server.go b/pkg/mcp/localaitools/server.go
index 8a6c64b86..f47332db5 100644
--- a/pkg/mcp/localaitools/server.go
+++ b/pkg/mcp/localaitools/server.go
@@ -54,6 +54,7 @@ func NewServer(client LocalAIClient, opts Options) *mcp.Server {
registerUsageTools(srv, client, opts)
registerPIITools(srv, client, opts)
registerMiddlewareTools(srv, client, opts)
+ registerFailoverTools(srv, client, opts)
return srv
}
diff --git a/pkg/mcp/localaitools/server_test.go b/pkg/mcp/localaitools/server_test.go
index c18a2301c..6fe104b9e 100644
--- a/pkg/mcp/localaitools/server_test.go
+++ b/pkg/mcp/localaitools/server_test.go
@@ -92,6 +92,7 @@ var expectedReadOnlyCatalog = sortedStrings(
ToolListVoiceProfiles,
ToolSystemInfo,
ToolVRAMEstimate,
+ ToolListFailoverChains,
)
// expectedFullCatalog derives from the read-only catalog plus the canonical
@@ -155,6 +156,9 @@ var _ = Describe("Tool dispatch", func() {
{ToolListAliases, struct{}{}, "ListAliases"},
{ToolCreateVoiceProfile, CreateVoiceProfileRequest{Name: "Narrator", Transcript: "Reference words", AudioBase64: "UklGRg==", ConsentConfirmed: true}, "CreateVoiceProfile"},
{ToolDeleteVoiceProfile, DeleteVoiceProfileRequest{ID: "00000000-0000-0000-0000-000000000001"}, "DeleteVoiceProfile"},
+ {ToolListFailoverChains, map[string]any{}, "ListFailoverChains"},
+ {ToolPinFailoverTarget, map[string]any{"chain": "c", "target": "b"}, "PinFailoverTarget"},
+ {ToolUnpinFailoverTarget, map[string]any{"chain": "c"}, "UnpinFailoverTarget"},
}
for _, c := range cases {
diff --git a/pkg/mcp/localaitools/tools.go b/pkg/mcp/localaitools/tools.go
index cfc7f0bfd..e2c9b32d3 100644
--- a/pkg/mcp/localaitools/tools.go
+++ b/pkg/mcp/localaitools/tools.go
@@ -49,10 +49,19 @@ const (
ToolSetNodeVRAMBudget = "set_node_vram_budget"
ToolSetScheduling = "set_scheduling"
ToolDeleteScheduling = "delete_scheduling"
+ // ToolPinFailoverTarget and ToolUnpinFailoverTarget live here (rather
+ // than grouped with ToolListFailoverChains below) so mutatingToolNames
+ // stays a contiguous scan of this block.
+ ToolPinFailoverTarget = "pin_failover_target"
+ ToolUnpinFailoverTarget = "unpin_failover_target"
// ToolListAliases is read-only but lives here so the alias tools stay
// grouped; the catalog tests assert its read-only placement.
ToolListAliases = "list_aliases"
+
+ // ToolListFailoverChains is read-only but lives here so the failover
+ // tools stay grouped; the catalog tests assert its read-only placement.
+ ToolListFailoverChains = "list_failover_chains"
)
// DefaultServerName is the MCP Implementation.Name surfaced when
@@ -83,4 +92,6 @@ var mutatingToolNames = []string{
ToolSetNodeVRAMBudget,
ToolSetScheduling,
ToolDeleteScheduling,
+ ToolPinFailoverTarget,
+ ToolUnpinFailoverTarget,
}
diff --git a/pkg/mcp/localaitools/tools_failover.go b/pkg/mcp/localaitools/tools_failover.go
new file mode 100644
index 000000000..ca3368896
--- /dev/null
+++ b/pkg/mcp/localaitools/tools_failover.go
@@ -0,0 +1,53 @@
+package localaitools
+
+import (
+ "context"
+
+ "github.com/modelcontextprotocol/go-sdk/mcp"
+)
+
+// registerFailoverTools wires the conversational failover-chain tools.
+// list_failover_chains reports the health of every chain, pin_failover_target
+// forces a chain to one target, and unpin_failover_target hands control back
+// to health-based selection.
+func registerFailoverTools(s *mcp.Server, client LocalAIClient, opts Options) {
+ mcp.AddTool(s, &mcp.Tool{
+ Name: ToolListFailoverChains,
+ Description: "List model failover chains, the target serving each one now, and the health of every target.",
+ }, func(ctx context.Context, _ *mcp.CallToolRequest, _ struct{}) (*mcp.CallToolResult, any, error) {
+ chains, err := client.ListFailoverChains(ctx)
+ if err != nil {
+ return errorResult(err), nil, nil
+ }
+ return jsonResult(chains), nil, nil
+ })
+
+ if opts.DisableMutating {
+ return
+ }
+
+ mcp.AddTool(s, &mcp.Tool{
+ Name: ToolPinFailoverTarget,
+ Description: "Force a failover chain to serve every request from one target, regardless of health, until it is unpinned. Requires user confirmation per safety rule 1.",
+ }, func(ctx context.Context, _ *mcp.CallToolRequest, args struct {
+ Chain string `json:"chain" jsonschema:"failover chain name"`
+ Target string `json:"target" jsonschema:"target model to pin"`
+ }) (*mcp.CallToolResult, any, error) {
+ if err := client.PinFailoverTarget(ctx, args.Chain, args.Target); err != nil {
+ return errorResult(err), nil, nil
+ }
+ return jsonResult(map[string]string{"chain": args.Chain, "pinned": args.Target}), nil, nil
+ })
+
+ mcp.AddTool(s, &mcp.Tool{
+ Name: ToolUnpinFailoverTarget,
+ Description: "Remove the pin from a failover chain so health decides the target again. Requires user confirmation per safety rule 1.",
+ }, func(ctx context.Context, _ *mcp.CallToolRequest, args struct {
+ Chain string `json:"chain" jsonschema:"failover chain name"`
+ }) (*mcp.CallToolResult, any, error) {
+ if err := client.UnpinFailoverTarget(ctx, args.Chain); err != nil {
+ return errorResult(err), nil, nil
+ }
+ return jsonResult(map[string]string{"chain": args.Chain, "pinned": ""}), nil, nil
+ })
+}
diff --git a/pkg/model/initializers.go b/pkg/model/initializers.go
index cbfb7840c..2ae9242e3 100644
--- a/pkg/model/initializers.go
+++ b/pkg/model/initializers.go
@@ -51,6 +51,12 @@ const (
TransformersBackend = "transformers"
LocalStoreBackend = "local-store"
ValkeyStoreBackend = "valkey-store"
+
+ // Proxy backends serve a model by forwarding to another server instead
+ // of loading weights. Core special-cases both (credentials, failover
+ // kind, PII defaults), so every check goes through these names.
+ CloudProxyBackend = "cloud-proxy"
+ LocalAIProxyBackend = "localai-proxy"
)
// starts the grpcModelProcess for the backend, and returns a grpc client
@@ -473,7 +479,9 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
// Wrap remote models so connection errors during inference trigger eviction
if m.Process() == nil {
client = newConnectionEvictingClient(client, o.modelID, func() {
- ml.ShutdownModel(o.modelID)
+ if err := ml.ShutdownModel(o.modelID); err != nil {
+ xlog.Debug("evicting a model after its connection failed", "model", o.modelID, "error", err)
+ }
})
}
return client, nil
@@ -497,7 +505,9 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
// Wrap remote models so connection errors during inference trigger eviction
if m := ml.CheckIsLoaded(o.modelID); m != nil && m.Process() == nil {
client = newConnectionEvictingClient(client, o.modelID, func() {
- ml.ShutdownModel(o.modelID)
+ if err := ml.ShutdownModel(o.modelID); err != nil {
+ xlog.Debug("evicting a model after its connection failed", "model", o.modelID, "error", err)
+ }
})
}
return client, nil
@@ -538,7 +548,9 @@ func (ml *ModelLoader) Load(opts ...Option) (grpc.Backend, error) {
// Wrap remote models so connection errors during inference trigger eviction
if m := ml.CheckIsLoaded(o.modelID); m != nil && m.Process() == nil {
model = newConnectionEvictingClient(model, o.modelID, func() {
- ml.ShutdownModel(o.modelID)
+ if err := ml.ShutdownModel(o.modelID); err != nil {
+ xlog.Debug("evicting a model after its connection failed", "model", o.modelID, "error", err)
+ }
})
}
return model, nil
diff --git a/swagger/docs.go b/swagger/docs.go
index 0950fdb6c..c7cd3c8f3 100644
--- a/swagger/docs.go
+++ b/swagger/docs.go
@@ -881,6 +881,160 @@ const docTemplate = `{
}
}
},
+ "/api/failover": {
+ "get": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "List failover chains and the health of their targets",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/localai.FailoverChainsResponse"
+ }
+ }
+ }
+ }
+ },
+ "/api/failover/events": {
+ "get": {
+ "description": "The first event is \"snapshot\" with the full state, then \"chain.switched\" and \"target.state\" events.",
+ "produces": [
+ "text/event-stream"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Stream failover events (server-sent events)",
+ "responses": {
+ "200": {
+ "description": "OK"
+ }
+ }
+ }
+ },
+ "/api/failover/{chain}": {
+ "get": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Get one failover chain",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "/api/failover/{chain}/pin": {
+ "post": {
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Pin a failover chain to one target",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "Target to pin",
+ "name": "request",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/localai.FailoverPinRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "400": {
+ "description": "Bad Request",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ },
+ "delete": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Remove the pin from a failover chain",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ }
+ },
"/api/instructions": {
"get": {
"description": "Returns a compact list of instruction areas with descriptions and URLs for detailed guides",
@@ -4242,6 +4396,97 @@ const docTemplate = `{
}
}
},
+ "failover.ChainState": {
+ "type": "string",
+ "enum": [
+ "primary",
+ "fallback",
+ "degraded"
+ ],
+ "x-enum-varnames": [
+ "ChainPrimary",
+ "ChainFallback",
+ "ChainDegraded"
+ ]
+ },
+ "failover.ChainStatus": {
+ "type": "object",
+ "properties": {
+ "active": {
+ "type": "string"
+ },
+ "active_since": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "pinned": {
+ "type": "string"
+ },
+ "state": {
+ "$ref": "#/definitions/failover.ChainState"
+ },
+ "targets": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/failover.TargetStatus"
+ }
+ }
+ }
+ },
+ "failover.Kind": {
+ "type": "string",
+ "enum": [
+ "local",
+ "remote"
+ ],
+ "x-enum-varnames": [
+ "KindLocal",
+ "KindRemote"
+ ]
+ },
+ "failover.TargetState": {
+ "type": "string",
+ "enum": [
+ "healthy",
+ "down",
+ "recovering",
+ "missing"
+ ],
+ "x-enum-varnames": [
+ "StateHealthy",
+ "StateDown",
+ "StateRecovering",
+ "StateMissing"
+ ]
+ },
+ "failover.TargetStatus": {
+ "type": "object",
+ "properties": {
+ "consecutive_ok": {
+ "type": "integer"
+ },
+ "kind": {
+ "$ref": "#/definitions/failover.Kind"
+ },
+ "last_error": {
+ "type": "string"
+ },
+ "last_probe": {
+ "type": "string"
+ },
+ "model": {
+ "type": "string"
+ },
+ "state": {
+ "$ref": "#/definitions/failover.TargetState"
+ },
+ "warm": {
+ "type": "boolean"
+ }
+ }
+ },
"functions.Function": {
"type": "object",
"properties": {
@@ -4652,6 +4897,25 @@ const docTemplate = `{
}
}
},
+ "localai.FailoverChainsResponse": {
+ "type": "object",
+ "properties": {
+ "chains": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ }
+ }
+ },
+ "localai.FailoverPinRequest": {
+ "type": "object",
+ "properties": {
+ "target": {
+ "type": "string"
+ }
+ }
+ },
"localai.GalleryBackend": {
"type": "object",
"properties": {
diff --git a/swagger/swagger.json b/swagger/swagger.json
index 06d6c2c8d..3f374d87c 100644
--- a/swagger/swagger.json
+++ b/swagger/swagger.json
@@ -878,6 +878,160 @@
}
}
},
+ "/api/failover": {
+ "get": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "List failover chains and the health of their targets",
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/localai.FailoverChainsResponse"
+ }
+ }
+ }
+ }
+ },
+ "/api/failover/events": {
+ "get": {
+ "description": "The first event is \"snapshot\" with the full state, then \"chain.switched\" and \"target.state\" events.",
+ "produces": [
+ "text/event-stream"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Stream failover events (server-sent events)",
+ "responses": {
+ "200": {
+ "description": "OK"
+ }
+ }
+ }
+ },
+ "/api/failover/{chain}": {
+ "get": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Get one failover chain",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ }
+ },
+ "/api/failover/{chain}/pin": {
+ "post": {
+ "consumes": [
+ "application/json"
+ ],
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Pin a failover chain to one target",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ },
+ {
+ "description": "Target to pin",
+ "name": "request",
+ "in": "body",
+ "required": true,
+ "schema": {
+ "$ref": "#/definitions/localai.FailoverPinRequest"
+ }
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "400": {
+ "description": "Bad Request",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ },
+ "delete": {
+ "produces": [
+ "application/json"
+ ],
+ "tags": [
+ "failover"
+ ],
+ "summary": "Remove the pin from a failover chain",
+ "parameters": [
+ {
+ "type": "string",
+ "description": "Chain name",
+ "name": "chain",
+ "in": "path",
+ "required": true
+ }
+ ],
+ "responses": {
+ "200": {
+ "description": "OK",
+ "schema": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ },
+ "404": {
+ "description": "Not Found",
+ "schema": {
+ "$ref": "#/definitions/schema.ErrorResponse"
+ }
+ }
+ }
+ }
+ },
"/api/instructions": {
"get": {
"description": "Returns a compact list of instruction areas with descriptions and URLs for detailed guides",
@@ -4239,6 +4393,97 @@
}
}
},
+ "failover.ChainState": {
+ "type": "string",
+ "enum": [
+ "primary",
+ "fallback",
+ "degraded"
+ ],
+ "x-enum-varnames": [
+ "ChainPrimary",
+ "ChainFallback",
+ "ChainDegraded"
+ ]
+ },
+ "failover.ChainStatus": {
+ "type": "object",
+ "properties": {
+ "active": {
+ "type": "string"
+ },
+ "active_since": {
+ "type": "string"
+ },
+ "name": {
+ "type": "string"
+ },
+ "pinned": {
+ "type": "string"
+ },
+ "state": {
+ "$ref": "#/definitions/failover.ChainState"
+ },
+ "targets": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/failover.TargetStatus"
+ }
+ }
+ }
+ },
+ "failover.Kind": {
+ "type": "string",
+ "enum": [
+ "local",
+ "remote"
+ ],
+ "x-enum-varnames": [
+ "KindLocal",
+ "KindRemote"
+ ]
+ },
+ "failover.TargetState": {
+ "type": "string",
+ "enum": [
+ "healthy",
+ "down",
+ "recovering",
+ "missing"
+ ],
+ "x-enum-varnames": [
+ "StateHealthy",
+ "StateDown",
+ "StateRecovering",
+ "StateMissing"
+ ]
+ },
+ "failover.TargetStatus": {
+ "type": "object",
+ "properties": {
+ "consecutive_ok": {
+ "type": "integer"
+ },
+ "kind": {
+ "$ref": "#/definitions/failover.Kind"
+ },
+ "last_error": {
+ "type": "string"
+ },
+ "last_probe": {
+ "type": "string"
+ },
+ "model": {
+ "type": "string"
+ },
+ "state": {
+ "$ref": "#/definitions/failover.TargetState"
+ },
+ "warm": {
+ "type": "boolean"
+ }
+ }
+ },
"functions.Function": {
"type": "object",
"properties": {
@@ -4649,6 +4894,25 @@
}
}
},
+ "localai.FailoverChainsResponse": {
+ "type": "object",
+ "properties": {
+ "chains": {
+ "type": "array",
+ "items": {
+ "$ref": "#/definitions/failover.ChainStatus"
+ }
+ }
+ }
+ },
+ "localai.FailoverPinRequest": {
+ "type": "object",
+ "properties": {
+ "target": {
+ "type": "string"
+ }
+ }
+ },
"localai.GalleryBackend": {
"type": "object",
"properties": {
diff --git a/swagger/swagger.yaml b/swagger/swagger.yaml
index 1337afa9a..7eeb216fd 100644
--- a/swagger/swagger.yaml
+++ b/swagger/swagger.yaml
@@ -53,6 +53,70 @@ definitions:
name:
type: string
type: object
+ failover.ChainState:
+ enum:
+ - primary
+ - fallback
+ - degraded
+ type: string
+ x-enum-varnames:
+ - ChainPrimary
+ - ChainFallback
+ - ChainDegraded
+ failover.ChainStatus:
+ properties:
+ active:
+ type: string
+ active_since:
+ type: string
+ name:
+ type: string
+ pinned:
+ type: string
+ state:
+ $ref: '#/definitions/failover.ChainState'
+ targets:
+ items:
+ $ref: '#/definitions/failover.TargetStatus'
+ type: array
+ type: object
+ failover.Kind:
+ enum:
+ - local
+ - remote
+ type: string
+ x-enum-varnames:
+ - KindLocal
+ - KindRemote
+ failover.TargetState:
+ enum:
+ - healthy
+ - down
+ - recovering
+ - missing
+ type: string
+ x-enum-varnames:
+ - StateHealthy
+ - StateDown
+ - StateRecovering
+ - StateMissing
+ failover.TargetStatus:
+ properties:
+ consecutive_ok:
+ type: integer
+ kind:
+ $ref: '#/definitions/failover.Kind'
+ last_error:
+ type: string
+ last_probe:
+ type: string
+ model:
+ type: string
+ state:
+ $ref: '#/definitions/failover.TargetState'
+ warm:
+ type: boolean
+ type: object
functions.Function:
properties:
description:
@@ -344,6 +408,18 @@ definitions:
logo_url:
type: string
type: object
+ localai.FailoverChainsResponse:
+ properties:
+ chains:
+ items:
+ $ref: '#/definitions/failover.ChainStatus'
+ type: array
+ type: object
+ localai.FailoverPinRequest:
+ properties:
+ target:
+ type: string
+ type: object
localai.GalleryBackend:
properties:
force:
@@ -3866,6 +3942,107 @@ paths:
summary: Upload a branding asset
tags:
- branding
+ /api/failover:
+ get:
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/localai.FailoverChainsResponse'
+ summary: List failover chains and the health of their targets
+ tags:
+ - failover
+ /api/failover/{chain}:
+ get:
+ parameters:
+ - description: Chain name
+ in: path
+ name: chain
+ required: true
+ type: string
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/failover.ChainStatus'
+ "404":
+ description: Not Found
+ schema:
+ $ref: '#/definitions/schema.ErrorResponse'
+ summary: Get one failover chain
+ tags:
+ - failover
+ /api/failover/{chain}/pin:
+ delete:
+ parameters:
+ - description: Chain name
+ in: path
+ name: chain
+ required: true
+ type: string
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/failover.ChainStatus'
+ "404":
+ description: Not Found
+ schema:
+ $ref: '#/definitions/schema.ErrorResponse'
+ summary: Remove the pin from a failover chain
+ tags:
+ - failover
+ post:
+ consumes:
+ - application/json
+ parameters:
+ - description: Chain name
+ in: path
+ name: chain
+ required: true
+ type: string
+ - description: Target to pin
+ in: body
+ name: request
+ required: true
+ schema:
+ $ref: '#/definitions/localai.FailoverPinRequest'
+ produces:
+ - application/json
+ responses:
+ "200":
+ description: OK
+ schema:
+ $ref: '#/definitions/failover.ChainStatus'
+ "400":
+ description: Bad Request
+ schema:
+ $ref: '#/definitions/schema.ErrorResponse'
+ "404":
+ description: Not Found
+ schema:
+ $ref: '#/definitions/schema.ErrorResponse'
+ summary: Pin a failover chain to one target
+ tags:
+ - failover
+ /api/failover/events:
+ get:
+ description: The first event is "snapshot" with the full state, then "chain.switched"
+ and "target.state" events.
+ produces:
+ - text/event-stream
+ responses:
+ "200":
+ description: OK
+ summary: Stream failover events (server-sent events)
+ tags:
+ - failover
/api/instructions:
get:
description: Returns a compact list of instruction areas with descriptions and
diff --git a/tests/e2e/cloud_proxy_helpers_test.go b/tests/e2e/cloud_proxy_helpers_test.go
index 819d9aa08..f98049b4c 100644
--- a/tests/e2e/cloud_proxy_helpers_test.go
+++ b/tests/e2e/cloud_proxy_helpers_test.go
@@ -5,6 +5,7 @@ import (
"io"
"net/http"
"net/http/httptest"
+ "slices"
"strings"
"sync"
"sync/atomic"
@@ -46,6 +47,9 @@ type fakeOpenAIUpstreamServer struct {
mu sync.Mutex
script func(req []byte) (status int, body string, contentType string)
+ // models is what GET /v1/models lists: failover liveness probes check
+ // that the upstream still serves the target's model.
+ models []string
}
func newFakeOpenAIUpstream() *fakeOpenAIUpstreamServer {
@@ -59,6 +63,20 @@ func newFakeOpenAIUpstream() *fakeOpenAIUpstreamServer {
}
func (f *fakeOpenAIUpstreamServer) serve(w http.ResponseWriter, r *http.Request) {
+ // Answered before recording: periodic probes must not overwrite the
+ // request a spec is about to assert on.
+ if r.Method == http.MethodGet && r.URL.Path == "/v1/models" {
+ f.mu.Lock()
+ ids := slices.Clone(f.models)
+ f.mu.Unlock()
+ data := make([]map[string]string, 0, len(ids))
+ for _, id := range ids {
+ data = append(data, map[string]string{"id": id})
+ }
+ w.Header().Set("Content-Type", "application/json")
+ _ = json.NewEncoder(w).Encode(map[string]any{"data": data})
+ return
+ }
atomic.AddInt32(&f.recorder.RequestHits, 1)
body, _ := io.ReadAll(r.Body)
f.recorder.mu.Lock()
@@ -80,6 +98,12 @@ func (f *fakeOpenAIUpstreamServer) serve(w http.ResponseWriter, r *http.Request)
func (f *fakeOpenAIUpstreamServer) URL() string { return f.srv.URL }
func (f *fakeOpenAIUpstreamServer) Close() { f.srv.Close() }
+func (f *fakeOpenAIUpstreamServer) SetModels(ids ...string) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.models = ids
+}
+
func (f *fakeOpenAIUpstreamServer) SetScript(script func(req []byte) (status int, body string, contentType string)) {
f.mu.Lock()
defer f.mu.Unlock()
diff --git a/tests/e2e/e2e_failover_test.go b/tests/e2e/e2e_failover_test.go
new file mode 100644
index 000000000..22a4226f6
--- /dev/null
+++ b/tests/e2e/e2e_failover_test.go
@@ -0,0 +1,174 @@
+package e2e_test
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+)
+
+var _ = Describe("Failover chains", Label("failover"), func() {
+ expectServedByMock := func(resp *http.Response) {
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ Expect(resp.StatusCode).To(BeNumerically("<", 300), "%s\nheaders: %v", body, resp.Header)
+ Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("mock-model"))
+ Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback"))
+ }
+
+ // The entry name is the chain suffix: chain- is written by the suite.
+ DescribeTable("retries every endpoint family on the next target",
+ func(path string, body func(model string) map[string]any) {
+ expectServedByMock(postJSONTo(path, body("chain-"+CurrentSpecReport().LeafNodeText)))
+ },
+ Entry("chat", "/chat/completions", func(m string) map[string]any {
+ return map[string]any{"model": m, "messages": []map[string]string{{"role": "user", "content": "hi"}}}
+ }),
+ Entry("completion", "/completions", func(m string) map[string]any {
+ return map[string]any{"model": m, "prompt": "hi"}
+ }),
+ Entry("embeddings", "/embeddings", func(m string) map[string]any {
+ return map[string]any{"model": m, "input": "hi"}
+ }),
+ Entry("tts", "/audio/speech", func(m string) map[string]any {
+ return map[string]any{"model": m, "input": "hi", "voice": "default"}
+ }),
+ Entry("image", "/images/generations", func(m string) map[string]any {
+ return map[string]any{"model": m, "prompt": "a cat", "size": "256x256"}
+ }),
+ Entry("rerank", "/rerank", func(m string) map[string]any {
+ return map[string]any{"model": m, "query": "q", "documents": []string{"a", "b"}}
+ }),
+ Entry("vad", "/vad", func(m string) map[string]any {
+ return map[string]any{"model": m, "audio": []float32{0, 0, 0, 0}}
+ }),
+ )
+
+ It("retries transcription with the multipart body", func() {
+ var body bytes.Buffer
+ mw := multipart.NewWriter(&body)
+ Expect(mw.WriteField("model", "chain-transcription")).To(Succeed())
+ fw, err := mw.CreateFormFile("file", "a.wav")
+ Expect(err).ToNot(HaveOccurred())
+ // 200 ms of 16 kHz mono silence.
+ _, err = fw.Write(wavFromPCM(make([]byte, 6400), 16000))
+ Expect(err).ToNot(HaveOccurred())
+ Expect(mw.Close()).To(Succeed())
+ resp, err := http.Post(apiURL+"/audio/transcriptions", mw.FormDataContentType(), &body)
+ Expect(err).ToNot(HaveOccurred())
+ expectServedByMock(resp)
+ })
+
+ Describe("remote targets", Ordered, func() {
+ var up1, up2 *fakeOpenAIUpstreamServer
+
+ chatReply := func([]byte) (int, string, string) {
+ return 200, `{"id":"x","object":"chat.completion","choices":[{"index":0,"message":{"role":"assistant","content":"hi"},"finish_reason":"stop"}]}`, "application/json"
+ }
+
+ BeforeAll(func() {
+ if cloudProxyPath == "" {
+ Skip("cloud-proxy backend binary not built (make build-cloud-proxy-backend)")
+ }
+ up1, up2 = newFakeOpenAIUpstream(), newFakeOpenAIUpstream()
+ DeferCleanup(up1.Close)
+ DeferCleanup(up2.Close)
+ up1.SetModels("up-1")
+ up2.SetModels("up-2")
+ registerFailoverRemoteModels(up1.URL(), up2.URL())
+ })
+
+ It("fails over when the primary upstream errors and fails back when it recovers", func() {
+ up1.SetScript(func([]byte) (int, string, string) {
+ return 503, `{"error":"no healthy nodes"}`, "application/json"
+ })
+ up2.SetScript(chatReply)
+
+ resp := postJSONTo("/chat/completions", map[string]any{"model": "chain-remote", "messages": []map[string]string{{"role": "user", "content": "hi"}}})
+ defer func() { _ = resp.Body.Close() }()
+ body, _ := io.ReadAll(resp.Body)
+ Expect(resp.StatusCode).To(Equal(200), string(body))
+ Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-2"))
+ Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback"))
+ Expect(chainActive("chain-remote")).To(Equal("up-2"))
+ // The targets set no upstream_model: each upstream must get its
+ // target's name (what the liveness probe checks), not the chain
+ // name the client sent. up-2 is healthy, so only this request
+ // posted to it.
+ Expect(upstreamBodyModel(up2)).To(Equal("up-2"))
+
+ up1.SetScript(chatReply)
+ Eventually(func() string { return chainActive("chain-remote") }, 30*time.Second, 500*time.Millisecond).
+ Should(Equal("up-1"))
+
+ resp2 := postJSONTo("/chat/completions", map[string]any{"model": "chain-remote", "messages": []map[string]string{{"role": "user", "content": "hi"}}})
+ defer func() { _ = resp2.Body.Close() }()
+ Expect(resp2.StatusCode).To(Equal(200))
+ Expect(resp2.Header.Get("X-LocalAI-Served-Model")).To(Equal("up-1"))
+ Expect(resp2.Header.Get("X-LocalAI-Failover")).To(BeEmpty())
+ Expect(upstreamBodyModel(up1)).To(Equal("up-1"))
+ })
+ })
+})
+
+// upstreamBodyModel returns the "model" field of the last request body the
+// fake upstream recorded.
+func upstreamBodyModel(up *fakeOpenAIUpstreamServer) string {
+ _, _, _, body := up.recorder.snapshot()
+ var req struct {
+ Model string `json:"model"`
+ }
+ Expect(json.Unmarshal(body, &req)).To(Succeed(), string(body))
+ return req.Model
+}
+
+// chainActive returns the active target of a chain as the REST status reports
+// it, or "" when the status cannot be read.
+func chainActive(chain string) string {
+ r, err := http.Get(anthropicBaseURL + "/api/failover/" + chain)
+ if err != nil {
+ return ""
+ }
+ defer func() { _ = r.Body.Close() }()
+ var st struct {
+ Active string `json:"active"`
+ }
+ _ = json.NewDecoder(r.Body).Decode(&st)
+ return st.Active
+}
+
+// registerFailoverRemoteModels registers two cloud-proxy passthrough models
+// (up-1, up-2) and a chain over them. The upstream URLs exist only at runtime,
+// so the YAMLs are written after startup and the loader re-reads the models
+// directory; the failover manager picks the chain up on its next tick.
+func registerFailoverRemoteModels(url1, url2 string) {
+ proxyModel := func(name, upstream string) map[string]any {
+ return map[string]any{
+ "name": name,
+ "backend": "cloud-proxy",
+ "parameters": map[string]any{"model": name + ".bin"},
+ "proxy": map[string]any{
+ "mode": "passthrough",
+ "provider": "openai",
+ "upstream_url": upstream + "/v1/chat/completions",
+ "api_key_env": "CLOUD_PROXY_E2E_OPENAI_KEY",
+ },
+ }
+ }
+ chain := map[string]any{
+ "name": "chain-remote",
+ "failover": map[string]any{
+ "targets": []map[string]any{{"model": "up-1"}, {"model": "up-2"}},
+ "probe": map[string]any{"interval": "1s"},
+ "recovery": map[string]any{"probes": 2, "min_dwell": "2s"},
+ },
+ }
+ registerModelConfigs(proxyModel("up-1", url1), proxyModel("up-2", url2), chain)
+ Eventually(func() string { return chainActive("chain-remote") }, 10*time.Second, 200*time.Millisecond).
+ Should(Equal("up-1"))
+}
diff --git a/tests/e2e/e2e_localai_proxy_test.go b/tests/e2e/e2e_localai_proxy_test.go
new file mode 100644
index 000000000..fc86f612e
--- /dev/null
+++ b/tests/e2e/e2e_localai_proxy_test.go
@@ -0,0 +1,215 @@
+package e2e_test
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "mime/multipart"
+ "net/http"
+ "net/http/httptest"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "path/filepath"
+ "sync/atomic"
+ "time"
+
+ . "github.com/onsi/ginkgo/v2"
+ . "github.com/onsi/gomega"
+ "gopkg.in/yaml.v3"
+)
+
+// The localai-proxy specs point the backend at this same test server: a
+// request to an lp-* model leaves LocalAI through the localai-proxy process,
+// comes back in over REST and is answered by the mock model it names, so the
+// whole round trip (core -> gRPC -> REST upstream -> gRPC reply) is real.
+var _ = Describe("localai-proxy backend", Label("failover"), Ordered, func() {
+ BeforeAll(func() {
+ if localAIProxyPath == "" {
+ Skip("localai-proxy backend binary not built (make build-localai-proxy-backend)")
+ }
+ registerModelConfigs(
+ localAIProxyModel("lp-chat", anthropicBaseURL, "mock-model", "chat"),
+ localAIProxyModel("lp-embeddings", anthropicBaseURL, "mock-model", "embeddings"),
+ localAIProxyModel("lp-tts", anthropicBaseURL, "mock-model", "tts"),
+ localAIProxyModel("lp-transcription", anthropicBaseURL, "mock-model", "transcript"),
+ // The upstream serves this target from a model whose load always
+ // fails, so every request through it errors with a 5xx.
+ localAIProxyModel("lp-broken", anthropicBaseURL, "fail-chat", "chat"),
+ map[string]any{
+ "name": "chain-lp",
+ "failover": map[string]any{
+ "targets": []map[string]any{{"model": "lp-broken"}, {"model": "mock-model"}},
+ },
+ },
+ )
+ Eventually(func() string { return chainActive("chain-lp") }, 10*time.Second, 200*time.Millisecond).
+ Should(Equal("lp-broken"))
+ })
+
+ // sameBody posts the same request to the proxied model and to the model
+ // the upstream serves it from, and returns both bodies: the proxy answered
+ // with the upstream's answer when they match.
+ sameBody := func(path string, body func(model string) map[string]any, proxied, upstream string) (string, string) {
+ get := func(model string) string {
+ resp := postJSONTo(path, body(model))
+ defer func() { _ = resp.Body.Close() }()
+ b, err := io.ReadAll(resp.Body)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(resp.StatusCode).To(Equal(http.StatusOK), "%s: %s", model, b)
+ return string(b)
+ }
+ return get(proxied), get(upstream)
+ }
+
+ It("answers chat with the upstream model's reply", func() {
+ chat := func(m string) map[string]any {
+ return map[string]any{"model": m, "messages": []map[string]string{{"role": "user", "content": "hi"}}}
+ }
+ got, want := sameBody("/chat/completions", chat, "lp-chat", "mock-model")
+ Expect(chatContent(got)).ToNot(BeEmpty())
+ Expect(chatContent(got)).To(Equal(chatContent(want)))
+ })
+
+ It("answers embeddings with the upstream model's vector", func() {
+ embed := func(m string) map[string]any { return map[string]any{"model": m, "input": "hello"} }
+ got, want := sameBody("/embeddings", embed, "lp-embeddings", "mock-model")
+ Expect(embeddingVector(got)).ToNot(BeEmpty())
+ Expect(embeddingVector(got)).To(Equal(embeddingVector(want)))
+ })
+
+ It("answers TTS with the upstream model's audio", func() {
+ speech := func(m string) map[string]any { return map[string]any{"model": m, "input": "hello", "voice": "default"} }
+ got, want := sameBody("/audio/speech", speech, "lp-tts", "mock-model")
+ Expect(got).To(HavePrefix("RIFF"))
+ Expect(got).To(Equal(want))
+ })
+
+ It("answers transcription with the upstream model's text", func() {
+ transcribe := func(model string) string {
+ var body bytes.Buffer
+ mw := multipart.NewWriter(&body)
+ Expect(mw.WriteField("model", model)).To(Succeed())
+ fw, err := mw.CreateFormFile("file", "a.wav")
+ Expect(err).ToNot(HaveOccurred())
+ _, err = fw.Write(wavFromPCM(make([]byte, 6400), 16000))
+ Expect(err).ToNot(HaveOccurred())
+ Expect(mw.Close()).To(Succeed())
+ resp, err := http.Post(apiURL+"/audio/transcriptions", mw.FormDataContentType(), &body)
+ Expect(err).ToNot(HaveOccurred())
+ defer func() { _ = resp.Body.Close() }()
+ b, _ := io.ReadAll(resp.Body)
+ Expect(resp.StatusCode).To(Equal(http.StatusOK), "%s: %s", model, b)
+ var out struct {
+ Text string `json:"text"`
+ }
+ Expect(json.Unmarshal(b, &out)).To(Succeed(), string(b))
+ return out.Text
+ }
+ got := transcribe("lp-transcription")
+ Expect(got).To(HavePrefix("transcribed:"))
+ Expect(got).To(Equal(transcribe("mock-model")))
+ })
+
+ It("fails over from a proxy target whose upstream model fails to the local target", func() {
+ resp := postJSONTo("/chat/completions", map[string]any{
+ "model": "chain-lp",
+ "messages": []map[string]string{{"role": "user", "content": "hi"}},
+ })
+ defer func() { _ = resp.Body.Close() }()
+ b, _ := io.ReadAll(resp.Body)
+ Expect(resp.StatusCode).To(Equal(http.StatusOK), string(b))
+ Expect(resp.Header.Get("X-LocalAI-Served-Model")).To(Equal("mock-model"))
+ Expect(resp.Header.Get("X-LocalAI-Failover")).To(Equal("fallback"))
+ Expect(chainActive("chain-lp")).To(Equal("mock-model"))
+ })
+})
+
+// postJSONTo posts body as JSON to an /v1 path of the test server.
+func postJSONTo(path string, body map[string]any) *http.Response {
+ b, err := json.Marshal(body)
+ ExpectWithOffset(1, err).ToNot(HaveOccurred())
+ resp, err := http.Post(apiURL+path, "application/json", bytes.NewReader(b))
+ ExpectWithOffset(1, err).ToNot(HaveOccurred())
+ return resp
+}
+
+func chatContent(body string) string {
+ var out struct {
+ Choices []struct {
+ Message struct {
+ Content string `json:"content"`
+ } `json:"message"`
+ } `json:"choices"`
+ }
+ ExpectWithOffset(1, json.Unmarshal([]byte(body), &out)).To(Succeed(), body)
+ ExpectWithOffset(1, out.Choices).ToNot(BeEmpty(), body)
+ return out.Choices[0].Message.Content
+}
+
+func embeddingVector(body string) []float32 {
+ var out struct {
+ Data []struct {
+ Embedding []float32 `json:"embedding"`
+ } `json:"data"`
+ }
+ ExpectWithOffset(1, json.Unmarshal([]byte(body), &out)).To(Succeed(), body)
+ ExpectWithOffset(1, out.Data).ToNot(BeEmpty(), body)
+ return out.Data[0].Embedding
+}
+
+// localAIProxyModel is a localai-proxy config serving upstreamModel from the
+// LocalAI at baseURL. known_usecases is what failover matches targets on, and
+// "chat" also makes the proxy send structured messages upstream.
+func localAIProxyModel(name, baseURL, upstreamModel string, usecases ...string) map[string]any {
+ return map[string]any{
+ "name": name,
+ "backend": "localai-proxy",
+ "known_usecases": usecases,
+ "parameters": map[string]any{"model": name + ".bin"},
+ "proxy": map[string]any{
+ "upstream_url": baseURL,
+ "upstream_model": upstreamModel,
+ },
+ }
+}
+
+// registerModelConfigs writes model YAMLs after startup and has the loader
+// re-read the models directory, for configs that embed runtime URLs. The
+// failover manager picks new chains up on its next tick.
+func registerModelConfigs(cfgs ...map[string]any) {
+ for _, cfg := range cfgs {
+ data, err := yaml.Marshal(cfg)
+ ExpectWithOffset(1, err).ToNot(HaveOccurred())
+ ExpectWithOffset(1, os.WriteFile(filepath.Join(modelsPath, cfg["name"].(string)+".yaml"), data, 0644)).To(Succeed())
+ }
+ ExpectWithOffset(1, localAIApp.ModelConfigLoader().LoadModelConfigsFromPath(modelsPath)).To(Succeed())
+}
+
+// upstreamGate is a reverse proxy in front of the test server that a spec
+// can take down: while down it answers 503, as an upstream LocalAI with no
+// healthy backend would, so a localai-proxy target behind it starts failing
+// without restarting its backend process.
+type upstreamGate struct {
+ srv *httptest.Server
+ down atomic.Bool
+}
+
+func newUpstreamGate(target string) *upstreamGate {
+ u, err := url.Parse(target)
+ ExpectWithOffset(1, err).ToNot(HaveOccurred())
+ g := &upstreamGate{}
+ rp := httputil.NewSingleHostReverseProxy(u)
+ g.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if g.down.Load() {
+ http.Error(w, `{"error":{"message":"no healthy backend"}}`, http.StatusServiceUnavailable)
+ return
+ }
+ rp.ServeHTTP(w, r)
+ }))
+ return g
+}
+
+func (g *upstreamGate) URL() string { return g.srv.URL }
+func (g *upstreamGate) Close() { g.srv.Close() }
+func (g *upstreamGate) SetDown(down bool) { g.down.Store(down) }
diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go
index 31d8d0d9c..6cdf84f2b 100644
--- a/tests/e2e/e2e_suite_test.go
+++ b/tests/e2e/e2e_suite_test.go
@@ -39,6 +39,7 @@ var (
apiURL string
mockBackendPath string
cloudProxyPath string
+ localAIProxyPath string
mcpServerURL string
mcpServerShutdown func()
localAIApp *localaiapp.Application
@@ -118,6 +119,29 @@ var _ = BeforeSuite(func() {
Expect(err).ToNot(HaveOccurred())
Expect(os.WriteFile(configPath, configYAML, 0644)).To(Succeed())
+ // Failover chains, one per endpoint family: target 0 is a mock model whose
+ // load always fails (the mock rejects models named fail-load*), so every
+ // request exercises the retry onto mock-model.
+ for _, family := range []string{"chat", "completion", "embeddings", "transcription", "tts", "image", "rerank", "vad"} {
+ for _, cfg := range []map[string]any{
+ {
+ "name": "fail-" + family,
+ "backend": "mock-backend",
+ "parameters": map[string]any{"model": "fail-load-" + family},
+ },
+ {
+ "name": "chain-" + family,
+ "failover": map[string]any{
+ "targets": []map[string]any{{"model": "fail-" + family}, {"model": "mock-model"}},
+ },
+ },
+ } {
+ data, err := yaml.Marshal(cfg)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(os.WriteFile(filepath.Join(modelsPath, cfg["name"].(string)+".yaml"), data, 0644)).To(Succeed())
+ }
+ }
+
// Create model config for autoparser tests (NoGrammar so tool calls
// are driven entirely by the backend's ChatDeltas, not grammar enforcement)
autoparserConfig := map[string]any{
@@ -250,6 +274,44 @@ var _ = BeforeSuite(func() {
Expect(err).ToNot(HaveOccurred())
Expect(os.WriteFile(filepath.Join(modelsPath, "realtime-pipeline.yaml"), pipelineData, 0644)).To(Succeed())
+ // Realtime pipelines whose LLM is a failover chain: target 0 always fails
+ // to load, target 1 is mock-llm. rt-failover skips the warm-up so the
+ // switch happens on the first turn, mid-session; rt-failover-warm keeps it
+ // so the switch happens while the session starts. Each has its own chain
+ // because chain state is shared across sessions.
+ for _, rt := range []struct {
+ name, suffix string
+ disableWarmup bool
+ }{{"rt-failover", "rt", true}, {"rt-failover-warm", "rt-warm", false}} {
+ for _, cfg := range []map[string]any{
+ {
+ "name": "fail-" + rt.suffix,
+ "backend": "mock-backend",
+ "parameters": map[string]any{"model": "fail-load-" + rt.suffix},
+ },
+ {
+ "name": "chain-" + rt.suffix,
+ "failover": map[string]any{
+ "targets": []map[string]any{{"model": "fail-" + rt.suffix}, {"model": "mock-llm"}},
+ },
+ },
+ {
+ "name": rt.name,
+ "pipeline": map[string]any{
+ "vad": "mock-vad",
+ "transcription": "mock-stt",
+ "llm": "chain-" + rt.suffix,
+ "tts": "mock-tts",
+ "disable_warmup": rt.disableWarmup,
+ },
+ },
+ } {
+ data, err := yaml.Marshal(cfg)
+ Expect(err).ToNot(HaveOccurred())
+ Expect(os.WriteFile(filepath.Join(modelsPath, cfg["name"].(string)+".yaml"), data, 0644)).To(Succeed())
+ }
+ }
+
// Classifier-mode pipeline (LocalAI extension): responses are
// prefill-scored against the option list via the mock backend's
// ROUTE_HINT-driven Score instead of being generated. Threshold 0.6:
@@ -585,6 +647,23 @@ var _ = BeforeSuite(func() {
}
}
+ // localai-proxy backend: its models point back at this server, whose URL
+ // exists only once it listens, so the specs register them at runtime.
+ // Like cloud-proxy, a missing binary makes those specs Skip.
+ for _, p := range []string{
+ filepath.Join("..", "e2e", "mock-backend", "localai-proxy"),
+ filepath.Join("tests", "e2e", "mock-backend", "localai-proxy"),
+ filepath.Join("..", "..", "tests", "e2e", "mock-backend", "localai-proxy"),
+ } {
+ if _, err := os.Stat(p); err == nil {
+ localAIProxyPath = p
+ break
+ }
+ }
+ if localAIProxyPath != "" {
+ Expect(os.Chmod(localAIProxyPath, 0755)).To(Succeed())
+ }
+
// Live PII NER tier. When PII_NER_MODEL_GGUF points at a downloaded
// privacy-filter GGUF, register two detector models that drive the real
// gRPC TokenClassify path on the privacy-filter backend (discovered via
@@ -633,6 +712,11 @@ var _ = BeforeSuite(func() {
config.WithSystemState(systemState),
config.WithDebug(true),
config.WithGeneratedContentDir(generatedDir),
+ // Mirrors the CLI boundary (core/cli/run.go): the failover prober
+ // resolves api_key_env upstream credentials through this lookup.
+ // Without it, remote failover targets configured with api_key_env
+ // (e.g. the cloud-proxy chain-remote spec) never pass liveness.
+ config.WithProxyAPIKeyEnvLookup(os.Getenv),
)
Expect(err).ToNot(HaveOccurred())
@@ -642,6 +726,9 @@ var _ = BeforeSuite(func() {
if cloudProxyPath != "" {
localAIApp.ModelLoader().SetExternalBackend("cloud-proxy", cloudProxyPath)
}
+ if localAIProxyPath != "" {
+ localAIApp.ModelLoader().SetExternalBackend("localai-proxy", localAIProxyPath)
+ }
// Create HTTP app
app, err = httpapi.API(localAIApp)
diff --git a/tests/e2e/mock-backend/.gitignore b/tests/e2e/mock-backend/.gitignore
index 32923bce6..7df9e3a51 100644
--- a/tests/e2e/mock-backend/.gitignore
+++ b/tests/e2e/mock-backend/.gitignore
@@ -1 +1,2 @@
-mock-backend
\ No newline at end of file
+mock-backend
+cloud-proxy
diff --git a/tests/e2e/mock-backend/main.go b/tests/e2e/mock-backend/main.go
index 36520cb55..f6b0ecf08 100644
--- a/tests/e2e/mock-backend/main.go
+++ b/tests/e2e/mock-backend/main.go
@@ -93,6 +93,10 @@ func (m *MockBackend) LoadModel(ctx context.Context, in *pb.ModelOptions) (*pb.R
"draft_model", in.DraftModel,
"mmproj", in.MMProj)
recordLoadParams(in)
+ // Lets e2e specs build a failover target whose backend cannot load.
+ if strings.HasPrefix(in.Model, "fail-load") {
+ return &pb.Result{Message: "mock: load failure", Success: false}, nil
+ }
return &pb.Result{
Message: "Model loaded successfully (mocked)",
Success: true,
@@ -644,6 +648,7 @@ func (m *MockBackend) AudioTranscription(ctx context.Context, in *pb.TranscriptR
rms := 0.0
if dst != "" {
+ // #nosec G304 -- test-only mock backend reading the path core just staged
if data, err := os.ReadFile(dst); err == nil {
if len(data) >= 44 {
wavSR = int(binary.LittleEndian.Uint32(data[24:28]))
@@ -1014,7 +1019,7 @@ func (m *MockBackend) ModelMetadata(ctx context.Context, in *pb.ModelOptions) (*
// survive resampling (DC is sample-rate independent). Near-zero DC maps to a
// neutral vector equidistant from both. Returns nil for unreadable audio.
func voiceEmbedFromWAV(path string) []float32 {
- data, err := os.ReadFile(path)
+ data, err := os.ReadFile(path) // #nosec G304 -- test-only mock backend reading the path core just staged
if err != nil || len(data) < 44 {
return nil
}
diff --git a/tests/e2e/realtime_ws_test.go b/tests/e2e/realtime_ws_test.go
index 6aeb3fcd4..90089fca7 100644
--- a/tests/e2e/realtime_ws_test.go
+++ b/tests/e2e/realtime_ws_test.go
@@ -192,6 +192,169 @@ var _ = Describe("Realtime WebSocket API", Label("Realtime"), func() {
})
})
+ Context("Failover chain stage", Label("failover"), func() {
+ // userTurn adds a user text item, asks for a response and reads until
+ // response.done. It returns the user item id, the response.done event
+ // and the localai.model.failover events seen on the way.
+ userTurn := func(conn *websocket.Conn, text string) (string, map[string]any, []map[string]any) {
+ sendClientEvent(conn, map[string]any{
+ "type": "conversation.item.create",
+ "item": map[string]any{
+ "type": "message",
+ "role": "user",
+ "content": []map[string]any{{"type": "input_text", "text": text}},
+ },
+ })
+ added := drainUntil(conn, "conversation.item.added", 10*time.Second)
+ item, _ := added["item"].(map[string]any)
+ userID, _ := item["id"].(string)
+ ExpectWithOffset(1, userID).ToNot(BeEmpty())
+
+ sendClientEvent(conn, map[string]any{"type": "response.create"})
+ var failovers []map[string]any
+ deadline := time.Now().Add(60 * time.Second)
+ for time.Now().Before(deadline) {
+ evt := readServerEvent(conn, time.Until(deadline))
+ switch evt["type"] {
+ case "localai.model.failover":
+ failovers = append(failovers, evt)
+ case "error":
+ Fail(fmt.Sprintf("unexpected error event: %v", evt))
+ case "response.done":
+ return userID, evt, failovers
+ }
+ }
+ Fail("timed out waiting for response.done")
+ return "", nil, nil
+ }
+
+ retrieveItem := func(conn *websocket.Conn, id string) map[string]any {
+ sendClientEvent(conn, map[string]any{"type": "conversation.item.retrieve", "item_id": id})
+ evt := drainUntil(conn, "conversation.item.retrieved", 10*time.Second)
+ item, _ := evt["item"].(map[string]any)
+ return item
+ }
+
+ It("switches the LLM mid-session and keeps the conversation", func() {
+ conn := connectWS("rt-failover")
+ defer func() { _ = conn.Close() }()
+
+ Expect(readServerEvent(conn, 30*time.Second)["type"]).To(Equal("session.created"))
+ initial := drainUntil(conn, "localai.model.failover", 10*time.Second)
+ Expect(initial).To(HaveKeyWithValue("stage", "llm"))
+ Expect(initial).To(HaveKeyWithValue("chain", "chain-rt"))
+ Expect(initial).To(HaveKeyWithValue("reason", "initial"))
+ Expect(initial).To(HaveKeyWithValue("to", "fail-rt"))
+
+ sendClientEvent(conn, disableVADEvent())
+ drainUntil(conn, "session.updated", 10*time.Second)
+
+ firstID, done, failovers := userTurn(conn, "Hello, how are you?")
+ Expect(failovers).To(ContainElement(And(
+ HaveKeyWithValue("stage", "llm"),
+ HaveKeyWithValue("from", "fail-rt"),
+ HaveKeyWithValue("to", "mock-llm"),
+ HaveKeyWithValue("reason", "trip"),
+ )))
+ resp, _ := done["response"].(map[string]any)
+ Expect(resp).To(HaveKeyWithValue("status", "completed"))
+ output, _ := resp["output"].([]any)
+ Expect(output).ToNot(BeEmpty())
+ firstReply, _ := output[0].(map[string]any)
+ firstReplyID, _ := firstReply["id"].(string)
+ Expect(firstReplyID).ToNot(BeEmpty())
+
+ _, done, _ = userTurn(conn, "And now?")
+ resp, _ = done["response"].(map[string]any)
+ Expect(resp).To(HaveKeyWithValue("status", "completed"))
+
+ // The switch kept the session: the first turn is still in it.
+ Expect(retrieveItem(conn, firstID)).To(HaveKeyWithValue("id", firstID))
+ Expect(retrieveItem(conn, firstReplyID)).To(HaveKeyWithValue("id", firstReplyID))
+ })
+
+ It("serves the LLM stage from a remote LocalAI and switches to the local target when it fails", func() {
+ if localAIProxyPath == "" {
+ Skip("localai-proxy backend binary not built (make build-localai-proxy-backend)")
+ }
+ // The remote target reaches this server through a gate the spec
+ // can take down, as if the remote LocalAI lost its backends.
+ gate := newUpstreamGate(anthropicBaseURL)
+ DeferCleanup(gate.Close)
+ registerModelConfigs(
+ localAIProxyModel("lp-rt-llm", gate.URL(), "mock-llm", "chat"),
+ map[string]any{
+ "name": "chain-rt-lp",
+ "failover": map[string]any{
+ "targets": []map[string]any{{"model": "lp-rt-llm"}, {"model": "mock-llm"}},
+ },
+ },
+ map[string]any{
+ "name": "rt-lp",
+ "pipeline": map[string]any{
+ "vad": "mock-vad",
+ "transcription": "mock-stt",
+ "llm": "chain-rt-lp",
+ "tts": "mock-tts",
+ "disable_warmup": true,
+ },
+ },
+ )
+ Eventually(func() string { return chainActive("chain-rt-lp") }, 10*time.Second, 200*time.Millisecond).
+ Should(Equal("lp-rt-llm"))
+
+ conn := connectWS("rt-lp")
+ defer func() { _ = conn.Close() }()
+
+ Expect(readServerEvent(conn, 30*time.Second)["type"]).To(Equal("session.created"))
+ initial := drainUntil(conn, "localai.model.failover", 10*time.Second)
+ Expect(initial).To(HaveKeyWithValue("stage", "llm"))
+ Expect(initial).To(HaveKeyWithValue("chain", "chain-rt-lp"))
+ Expect(initial).To(HaveKeyWithValue("reason", "initial"))
+ Expect(initial).To(HaveKeyWithValue("to", "lp-rt-llm"))
+
+ sendClientEvent(conn, disableVADEvent())
+ drainUntil(conn, "session.updated", 10*time.Second)
+
+ // The first turn goes through the proxy to the remote mock-llm.
+ _, done, failovers := userTurn(conn, "Hello, how are you?")
+ Expect(failovers).To(BeEmpty())
+ resp, _ := done["response"].(map[string]any)
+ Expect(resp).To(HaveKeyWithValue("status", "completed"))
+ Expect(chainActive("chain-rt-lp")).To(Equal("lp-rt-llm"))
+
+ gate.SetDown(true)
+ _, done, failovers = userTurn(conn, "And now?")
+ Expect(failovers).To(ContainElement(And(
+ HaveKeyWithValue("stage", "llm"),
+ HaveKeyWithValue("from", "lp-rt-llm"),
+ HaveKeyWithValue("to", "mock-llm"),
+ HaveKeyWithValue("reason", "trip"),
+ )))
+ resp, _ = done["response"].(map[string]any)
+ Expect(resp).To(HaveKeyWithValue("status", "completed"))
+ Expect(chainActive("chain-rt-lp")).To(Equal("mock-llm"))
+ })
+
+ It("starts the session on the next target when the active one fails to warm up", func() {
+ conn := connectWS("rt-failover-warm")
+ defer func() { _ = conn.Close() }()
+
+ Expect(readServerEvent(conn, 30*time.Second)["type"]).To(Equal("session.created"))
+ initial := drainUntil(conn, "localai.model.failover", 10*time.Second)
+ Expect(initial).To(HaveKeyWithValue("chain", "chain-rt-warm"))
+ Expect(initial).To(HaveKeyWithValue("reason", "initial"))
+ Expect(initial).To(HaveKeyWithValue("to", "mock-llm"))
+
+ sendClientEvent(conn, disableVADEvent())
+ drainUntil(conn, "session.updated", 10*time.Second)
+
+ _, done, _ := userTurn(conn, "Hello?")
+ resp, _ := done["response"].(map[string]any)
+ Expect(resp).To(HaveKeyWithValue("status", "completed"))
+ })
+ })
+
Context("Manual audio commit", func() {
It("should produce a response with audio when audio is committed", func() {
conn := connectWS(pipelineModel())