diff --git a/core/application/startup.go b/core/application/startup.go index a03f17bd2..241ea8b22 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -235,7 +235,12 @@ func New(opts ...config.AppOption) (*Application, error) { // In distributed mode, uses PostgreSQL advisory lock so only one frontend // instance runs periodic checks (avoids duplicate upgrades across replicas). if len(options.BackendGalleries) > 0 { - uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB()) + // Pass a lazy getter for the backend manager so the checker always + // uses the active one — DistributedBackendManager is swapped in above + // and asks workers for their installed backends, which is what + // upgrade detection needs in distributed mode. + bmFn := func() galleryop.BackendManager { return application.GalleryService().BackendManager() } + uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB(), bmFn) application.upgradeChecker = uc go uc.Run(options.Context) } diff --git a/core/application/upgrade_checker.go b/core/application/upgrade_checker.go index 94fb3f6c7..3b2d94544 100644 --- a/core/application/upgrade_checker.go +++ b/core/application/upgrade_checker.go @@ -8,6 +8,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/core/gallery" "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/core/services/galleryop" "github.com/mudler/LocalAI/pkg/model" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" @@ -26,6 +27,12 @@ type UpgradeChecker struct { galleries []config.Gallery systemState *system.SystemState db *gorm.DB // non-nil in distributed mode + // backendManagerFn lazily returns the current backend manager (may be + // swapped from Local to Distributed after startup). Pulled through each + // check so the UpgradeChecker uses whichever is active. In distributed + // mode this ensures CheckUpgrades asks workers instead of the (empty) + // frontend filesystem — fixing the bug where upgrades never surfaced. + backendManagerFn func() galleryop.BackendManager checkInterval time.Duration stop chan struct{} @@ -40,18 +47,22 @@ type UpgradeChecker struct { // NewUpgradeChecker creates a new UpgradeChecker service. // Pass db=nil for standalone mode, or a *gorm.DB for distributed mode // (uses advisory locks so only one instance runs periodic checks). -func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB) *UpgradeChecker { +// backendManagerFn is optional; when set, CheckUpgrades is routed through +// the active backend manager — required in distributed mode so the check +// aggregates from workers rather than the empty frontend filesystem. +func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB, backendManagerFn func() galleryop.BackendManager) *UpgradeChecker { return &UpgradeChecker{ - appConfig: appConfig, - modelLoader: ml, - galleries: appConfig.BackendGalleries, - systemState: appConfig.SystemState, - db: db, - checkInterval: 6 * time.Hour, - stop: make(chan struct{}), - done: make(chan struct{}), - triggerCh: make(chan struct{}, 1), - lastUpgrades: make(map[string]gallery.UpgradeInfo), + appConfig: appConfig, + modelLoader: ml, + galleries: appConfig.BackendGalleries, + systemState: appConfig.SystemState, + db: db, + backendManagerFn: backendManagerFn, + checkInterval: 6 * time.Hour, + stop: make(chan struct{}), + done: make(chan struct{}), + triggerCh: make(chan struct{}, 1), + lastUpgrades: make(map[string]gallery.UpgradeInfo), } } @@ -64,13 +75,16 @@ func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoade func (uc *UpgradeChecker) Run(ctx context.Context) { defer close(uc.done) - // Initial delay: don't slow down startup + // Initial delay: don't slow down startup. Short enough that operators + // don't stare at an empty upgrade banner for long; long enough that + // workers have registered and reported their installed backends. + initialDelay := 10 * time.Second select { case <-ctx.Done(): return case <-uc.stop: return - case <-time.After(30 * time.Second): + case <-time.After(initialDelay): } // First check always runs locally (to warm the cache on this instance) @@ -144,7 +158,18 @@ func (uc *UpgradeChecker) GetAvailableUpgrades() map[string]gallery.UpgradeInfo } func (uc *UpgradeChecker) runCheck(ctx context.Context) { - upgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState) + var ( + upgrades map[string]gallery.UpgradeInfo + err error + ) + if uc.backendManagerFn != nil { + if bm := uc.backendManagerFn(); bm != nil { + upgrades, err = bm.CheckUpgrades(ctx) + } + } + if upgrades == nil && err == nil { + upgrades, err = gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState) + } uc.mu.Lock() uc.lastCheckTime = time.Now() diff --git a/core/http/react-ui/src/App.css b/core/http/react-ui/src/App.css index c9e945d16..435b1c05b 100644 --- a/core/http/react-ui/src/App.css +++ b/core/http/react-ui/src/App.css @@ -1529,6 +1529,123 @@ select.input { background: var(--color-warning-light); color: var(--color-warning); } +.badge-accent { + background: var(--color-accent-light); + color: var(--color-accent); +} + +/* Horizontal row of badges used inside table cells — consistent spacing so + cells line up regardless of how many badges are present. */ +.badge-row { + display: inline-flex; + flex-wrap: wrap; + gap: 4px; + align-items: center; +} + +/* Vertically stacked cell content (e.g. version + update chip + drift chip). + Keeps rows readable at scale without inline style={{...}} everywhere. */ +.cell-stack { + display: flex; + flex-direction: column; + gap: 4px; + align-items: flex-start; +} + +.cell-mono { + font-family: 'JetBrains Mono', ui-monospace, monospace; + font-size: var(--text-xs); + color: var(--color-text-primary); +} + +.cell-muted { + color: var(--color-text-muted); + font-size: var(--text-xs); +} + +.cell-subtle { + color: var(--color-text-muted); + font-size: var(--text-xs); + font-weight: 400; + margin-left: 8px; +} + +.cell-name { + display: inline-flex; + align-items: center; + gap: var(--spacing-xs); + font-weight: 500; +} +.cell-name > i { + color: var(--color-accent); + font-size: var(--text-xs); +} + +.row-actions { + display: flex; + gap: var(--spacing-xs); + justify-content: flex-end; + align-items: center; +} + +/* Softer delete button for dense tables — the destructive confirm dialog + already owns the "are you sure" affordance, so the button itself doesn't + need to scream. Keeps the delete red readable without dominating rows. */ +.btn.btn-danger-ghost { + background: transparent; + color: var(--color-error); + border-color: transparent; +} +.btn.btn-danger-ghost:hover:not(:disabled) { + background: var(--color-error-light); + color: var(--color-error); + border-color: var(--color-error-light); +} + +/* Small count pill used inside tabs ("(3) ↑ 2") so update counts are + glanceable without extra rows of UI. */ +.tab-pill { + display: inline-flex; + align-items: center; + gap: 3px; + margin-left: 6px; + padding: 1px 6px; + border-radius: var(--radius-full); + font-size: var(--text-xs); + font-weight: 600; + line-height: 1.4; +} +.tab-pill--warning { + background: var(--color-warning-light); + color: var(--color-warning); +} + +/* Upgrade banner — the yellow strip operators see when updates are available. + Mirrors the gallery so both pages speak the same visual language. */ +.upgrade-banner { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--spacing-md); + padding: var(--spacing-sm) var(--spacing-md); + margin-bottom: var(--spacing-md); + background: var(--color-warning-light); + border: 1px solid var(--color-warning); + border-radius: var(--radius-md); + color: var(--color-warning); +} +.upgrade-banner__text { + display: inline-flex; + align-items: center; + gap: var(--spacing-sm); + font-weight: 500; + font-size: var(--text-sm); +} +.upgrade-banner__actions { + display: inline-flex; + gap: var(--spacing-xs); + align-items: center; +} /* Tabs */ .tabs { diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index bd6bda938..7b3b36d65 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -11,6 +11,22 @@ const TABS = [ { key: 'backends', label: 'Backends', icon: 'fa-server' }, ] +// formatInstalledAt renders an installed_at timestamp as a short relative/abs +// string suitable for dense tables. Returns the raw value if parsing fails so +// we never display "Invalid Date". +function formatInstalledAt(value) { + if (!value) return '—' + const d = new Date(value) + if (isNaN(d.getTime())) return value + const now = Date.now() + const diffMin = Math.floor((now - d.getTime()) / 60000) + if (diffMin < 1) return 'just now' + if (diffMin < 60) return `${diffMin}m ago` + if (diffMin < 60 * 24) return `${Math.floor(diffMin / 60)}h ago` + if (diffMin < 60 * 24 * 30) return `${Math.floor(diffMin / (60 * 24))}d ago` + return d.toISOString().slice(0, 10) +} + export default function Manage() { const { addToast } = useOutletContext() const navigate = useNavigate() @@ -196,6 +212,29 @@ export default function Manage() { } } + const [upgradingAll, setUpgradingAll] = useState(false) + const [showOnlyUpgradable, setShowOnlyUpgradable] = useState(false) + const handleUpgradeAll = async () => { + const names = Object.keys(upgrades) + if (names.length === 0) return + setUpgradingAll(true) + try { + // Serial upgrade — matches the gallery's Upgrade All behavior. + // Each backend upgrade is itself a cluster-wide fan-out, so parallel + // calls would multiply load on every worker. + for (const name of names) { + try { + await backendsApi.upgrade(name) + } catch (err) { + addToast(`Upgrade failed for ${name}: ${err.message}`, 'error') + } + } + addToast(`Upgrade started for ${names.length} backend${names.length === 1 ? '' : 's'}`, 'info') + } finally { + setUpgradingAll(false) + } + } + const handleDeleteBackend = (name) => { setConfirmDialog({ title: 'Delete Backend', @@ -227,18 +266,26 @@ export default function Manage() { {/* Tabs */}
- {TABS.map(t => ( - - ))} + {TABS.map(t => { + const upgradeCount = t.key === 'backends' ? Object.keys(upgrades).length : 0 + return ( + + ) + })}
{/* Models Tab */} @@ -399,6 +446,36 @@ export default function Manage() { {/* Backends Tab */} {activeTab === 'backends' && (
+ {/* Upgrade banner — mirrors the gallery so operators can't miss updates */} + {!backendsLoading && Object.keys(upgrades).length > 0 && ( +
+
+ + + {Object.keys(upgrades).length} backend{Object.keys(upgrades).length === 1 ? ' has' : 's have'} updates available + +
+
+ + +
+
+ )} + {backendsLoading ? (
Loading backends... @@ -419,109 +496,177 @@ export default function Manage() {
- ) : ( + ) : (() => { + const visibleBackends = showOnlyUpgradable + ? backends.filter(b => upgrades[b.Name]) + : backends + if (visibleBackends.length === 0) { + return ( +
+ +

No backends match the current filter.

+ +
+ ) + } + return (
- + + {distributedMode && } + - {backends.map((backend, i) => ( + {visibleBackends.map((backend, i) => { + const upgradeInfo = upgrades[backend.Name] + const hasDrift = upgradeInfo?.node_drift?.length > 0 + const nodes = backend.Nodes || backend.nodes || [] + return ( + {distributedMode && ( + + )} + - ))} + ) + })}
Name TypeMetadataVersionNodesInstalled Actions
-
- - {backend.Name} +
+ + {backend.Name} + {backend.Metadata?.alias && ( + alias: {backend.Metadata.alias} + )} + {backend.Metadata?.meta_backend_for && ( + for: {backend.Metadata.meta_backend_for} + )}
-
+
{backend.IsSystem ? ( - - System + + System ) : ( - - User + + User )} {backend.IsMeta && ( - - Meta + + Meta )}
-
- {backend.Metadata?.alias && ( - - - Alias: {backend.Metadata.alias} +
+ {backend.Metadata?.version ? ( + v{backend.Metadata.version} + ) : ( + — + )} + {upgradeInfo && ( + + + {upgradeInfo.available_version ? ` v${upgradeInfo.available_version}` : ' Update available'} )} - {backend.Metadata?.meta_backend_for && ( - - - For: {backend.Metadata.meta_backend_for} + {hasDrift && ( + `${d.node_name}${d.version ? ' v' + d.version : ''}`).join(', ')}`} + > + + {' '}Drift: {upgradeInfo.node_drift.length} node{upgradeInfo.node_drift.length === 1 ? '' : 's'} )} - {backend.Metadata?.version && ( - - - Version: v{backend.Metadata.version} - {upgrades[backend.Name] && ( - - → v{upgrades[backend.Name].available_version} +
+
+ {nodes.length === 0 ? ( + — + ) : nodes.length <= 3 ? ( +
+ {nodes.map(n => ( + + {n.node_name || n.NodeName} - )} - - )} - {backend.Metadata?.installed_at && ( - - - {backend.Metadata.installed_at} - - )} - {!backend.Metadata?.alias && !backend.Metadata?.meta_backend_for && !backend.Metadata?.installed_at && '—'} -
+ ))} + + ) : (() => { + const total = nodes.length + const offline = nodes.filter(n => { + const s = n.node_status || n.NodeStatus + return s !== 'healthy' && s !== 'draining' + }).length + return ( + 0 ? 'badge-warning' : 'badge-info'}`} + title={nodes.map(n => `${n.node_name || n.NodeName} (${n.node_status || n.NodeStatus})`).join('\n')} + > + on {total} nodes{offline > 0 ? ` · ${offline} offline` : ''} + + ) + })()} + 		+ + {backend.Metadata?.installed_at ? formatInstalledAt(backend.Metadata.installed_at) : '—'} + -
- {!backend.IsSystem ? ( +
+ {backend.IsSystem ? ( + + Protected + + ) : ( <> + {upgradeInfo ? ( + + ) : ( + + )} - - ) : ( - — )}
- )} + ) + })()} )} diff --git a/core/services/galleryop/service.go b/core/services/galleryop/service.go index fef638425..3d77d11d6 100644 --- a/core/services/galleryop/service.go +++ b/core/services/galleryop/service.go @@ -57,6 +57,16 @@ func (g *GalleryService) SetBackendManager(b BackendManager) { g.backendManager = b } +// BackendManager returns the current backend manager. Callers like the +// periodic upgrade checker need this so they run CheckUpgrades through the +// distributed implementation (which asks workers) instead of the frontend's +// local filesystem — the latter is always empty in distributed deployments. +func (g *GalleryService) BackendManager() BackendManager { + g.Lock() + defer g.Unlock() + return g.backendManager +} + // SetNATSClient sets the NATS client for distributed progress publishing. func (g *GalleryService) SetNATSClient(nc messaging.Publisher) { g.Lock()