diff --git a/core/application/application.go b/core/application/application.go index accba0330..9be613ab7 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -37,6 +37,9 @@ type Application struct { // Distributed mode services (nil when not in distributed mode) distributed *DistributedServices + + // Upgrade checker (background service for detecting backend upgrades) + upgradeChecker *UpgradeChecker } func newApplication(appConfig *config.ApplicationConfig) *Application { @@ -79,6 +82,19 @@ func (a *Application) AgentJobService() *agentpool.AgentJobService { return a.agentJobService } +func (a *Application) UpgradeChecker() *UpgradeChecker { + return a.upgradeChecker +} + +// distributedDB returns the PostgreSQL database for distributed coordination, +// or nil in standalone mode. +func (a *Application) distributedDB() *gorm.DB { + if a.distributed != nil { + return a.authDB + } + return nil +} + func (a *Application) AgentPoolService() *agentpool.AgentPoolService { return a.agentPoolService.Load() } diff --git a/core/application/config_file_watcher.go b/core/application/config_file_watcher.go index 8eb26355d..7d174bede 100644 --- a/core/application/config_file_watcher.go +++ b/core/application/config_file_watcher.go @@ -335,6 +335,9 @@ func readRuntimeSettingsJson(startupAppConfig config.ApplicationConfig) fileHand if settings.AutoloadBackendGalleries != nil && !envAutoloadBackendGalleries { appConfig.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries } + if settings.AutoUpgradeBackends != nil { + appConfig.AutoUpgradeBackends = *settings.AutoUpgradeBackends + } if settings.ApiKeys != nil { // API keys from env vars (startup) should be kept, runtime settings keys replace all runtime keys // If runtime_settings.json specifies ApiKeys (even if empty), it replaces all runtime keys diff --git a/core/application/startup.go b/core/application/startup.go index 728c3c972..a03f17bd2 100644 --- a/core/application/startup.go +++ b/core/application/startup.go @@ -231,6 +231,15 @@ func New(opts ...config.AppOption) (*Application, error) { xlog.Error("error registering external backends", "error", err) } + // Start background upgrade checker for backends. + // In distributed mode, uses PostgreSQL advisory lock so only one frontend + // instance runs periodic checks (avoids duplicate upgrades across replicas). + if len(options.BackendGalleries) > 0 { + uc := NewUpgradeChecker(options, application.ModelLoader(), application.distributedDB()) + application.upgradeChecker = uc + go uc.Run(options.Context) + } + if options.ConfigFile != "" { if err := application.ModelConfigLoader().LoadMultipleModelConfigsSingleFile(options.ConfigFile, configLoaderOpts...); err != nil { xlog.Error("error loading config file", "error", err) diff --git a/core/application/upgrade_checker.go b/core/application/upgrade_checker.go new file mode 100644 index 000000000..94fb3f6c7 --- /dev/null +++ b/core/application/upgrade_checker.go @@ -0,0 +1,198 @@ +package application + +import ( + "context" + "sync" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/core/services/advisorylock" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/xlog" + "gorm.io/gorm" +) + +// UpgradeChecker periodically checks for backend upgrades and optionally +// auto-upgrades them. It caches the last check results for API queries. +// +// In standalone mode it runs a simple ticker loop. +// In distributed mode it uses a PostgreSQL advisory lock so that only one +// frontend instance performs periodic checks and auto-upgrades at a time. +type UpgradeChecker struct { + appConfig *config.ApplicationConfig + modelLoader *model.ModelLoader + galleries []config.Gallery + systemState *system.SystemState + db *gorm.DB // non-nil in distributed mode + + checkInterval time.Duration + stop chan struct{} + done chan struct{} + triggerCh chan struct{} + + mu sync.RWMutex + lastUpgrades map[string]gallery.UpgradeInfo + lastCheckTime time.Time +} + +// NewUpgradeChecker creates a new UpgradeChecker service. +// Pass db=nil for standalone mode, or a *gorm.DB for distributed mode +// (uses advisory locks so only one instance runs periodic checks). +func NewUpgradeChecker(appConfig *config.ApplicationConfig, ml *model.ModelLoader, db *gorm.DB) *UpgradeChecker { + return &UpgradeChecker{ + appConfig: appConfig, + modelLoader: ml, + galleries: appConfig.BackendGalleries, + systemState: appConfig.SystemState, + db: db, + checkInterval: 6 * time.Hour, + stop: make(chan struct{}), + done: make(chan struct{}), + triggerCh: make(chan struct{}, 1), + lastUpgrades: make(map[string]gallery.UpgradeInfo), + } +} + +// Run starts the upgrade checker loop. It waits 30 seconds after startup, +// performs an initial check, then re-checks every 6 hours. +// +// In distributed mode, periodic checks are guarded by a PostgreSQL advisory +// lock so only one frontend instance runs them. On-demand triggers (TriggerCheck) +// and the initial check always run locally for fast API response cache warming. +func (uc *UpgradeChecker) Run(ctx context.Context) { + defer close(uc.done) + + // Initial delay: don't slow down startup + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-time.After(30 * time.Second): + } + + // First check always runs locally (to warm the cache on this instance) + uc.runCheck(ctx) + + if uc.db != nil { + // Distributed mode: use advisory lock for periodic checks. + // RunLeaderLoop ticks every checkInterval; only the lock holder executes. + go advisorylock.RunLeaderLoop(ctx, uc.db, advisorylock.KeyBackendUpgradeCheck, uc.checkInterval, func() { + uc.runCheck(ctx) + }) + + // Still listen for on-demand triggers (from API / settings change) + // and stop signal — these run on every instance. + for { + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-uc.triggerCh: + uc.runCheck(ctx) + } + } + } else { + // Standalone mode: simple ticker loop + ticker := time.NewTicker(uc.checkInterval) + defer ticker.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-uc.stop: + return + case <-ticker.C: + uc.runCheck(ctx) + case <-uc.triggerCh: + uc.runCheck(ctx) + } + } + } +} + +// Shutdown stops the upgrade checker loop. +func (uc *UpgradeChecker) Shutdown() { + close(uc.stop) + <-uc.done +} + +// TriggerCheck forces an immediate upgrade check on this instance. +func (uc *UpgradeChecker) TriggerCheck() { + select { + case uc.triggerCh <- struct{}{}: + default: + // Already triggered, skip + } +} + +// GetAvailableUpgrades returns the cached upgrade check results. +func (uc *UpgradeChecker) GetAvailableUpgrades() map[string]gallery.UpgradeInfo { + uc.mu.RLock() + defer uc.mu.RUnlock() + + // Return a copy to avoid races + result := make(map[string]gallery.UpgradeInfo, len(uc.lastUpgrades)) + for k, v := range uc.lastUpgrades { + result[k] = v + } + return result +} + +func (uc *UpgradeChecker) runCheck(ctx context.Context) { + upgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState) + + uc.mu.Lock() + uc.lastCheckTime = time.Now() + if err != nil { + xlog.Debug("Backend upgrade check failed", "error", err) + uc.mu.Unlock() + return + } + uc.lastUpgrades = upgrades + uc.mu.Unlock() + + if len(upgrades) == 0 { + xlog.Debug("All backends up to date") + return + } + + // Log available upgrades + for name, info := range upgrades { + if info.AvailableVersion != "" { + xlog.Info("Backend upgrade available", + "backend", name, + "installed", info.InstalledVersion, + "available", info.AvailableVersion) + } else { + xlog.Info("Backend upgrade available (new build)", + "backend", name) + } + } + + // Auto-upgrade if enabled + if uc.appConfig.AutoUpgradeBackends { + for name, info := range upgrades { + xlog.Info("Auto-upgrading backend", "backend", name, + "from", info.InstalledVersion, "to", info.AvailableVersion) + if err := gallery.UpgradeBackend(ctx, uc.systemState, uc.modelLoader, + uc.galleries, name, nil); err != nil { + xlog.Error("Failed to auto-upgrade backend", + "backend", name, "error", err) + } else { + xlog.Info("Backend upgraded successfully", "backend", name, + "version", info.AvailableVersion) + } + } + // Re-check to update cache after upgrades + if freshUpgrades, err := gallery.CheckBackendUpgrades(ctx, uc.galleries, uc.systemState); err == nil { + uc.mu.Lock() + uc.lastUpgrades = freshUpgrades + uc.mu.Unlock() + } + } +} diff --git a/core/cli/backends.go b/core/cli/backends.go index 23f6b3ff1..6269a9b0e 100644 --- a/core/cli/backends.go +++ b/core/cli/backends.go @@ -40,10 +40,17 @@ type BackendsUninstall struct { BackendsCMDFlags `embed:""` } +type BackendsUpgrade struct { + BackendArgs []string `arg:"" optional:"" name:"backends" help:"Backend names to upgrade (empty = upgrade all)"` + + BackendsCMDFlags `embed:""` +} + type BackendsCMD struct { List BackendsList `cmd:"" help:"List the backends available in your galleries" default:"withargs"` Install BackendsInstall `cmd:"" help:"Install a backend from the gallery"` Uninstall BackendsUninstall `cmd:"" help:"Uninstall a backend"` + Upgrade BackendsUpgrade `cmd:"" help:"Upgrade backends to latest versions"` } func (bl *BackendsList) Run(ctx *cliContext.Context) error { @@ -64,11 +71,27 @@ func (bl *BackendsList) Run(ctx *cliContext.Context) error { if err != nil { return err } + + // Check for upgrades + upgrades, _ := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState) + for _, backend := range backends { + versionStr := "" + if backend.Version != "" { + versionStr = " v" + backend.Version + } if backend.Installed { - fmt.Printf(" * %s@%s (installed)\n", backend.Gallery.Name, backend.Name) + if info, ok := upgrades[backend.Name]; ok { + upgradeStr := info.AvailableVersion + if upgradeStr == "" { + upgradeStr = "new build" + } + fmt.Printf(" * %s@%s%s (installed, upgrade available: %s)\n", backend.Gallery.Name, backend.Name, versionStr, upgradeStr) + } else { + fmt.Printf(" * %s@%s%s (installed)\n", backend.Gallery.Name, backend.Name, versionStr) + } } else { - fmt.Printf(" - %s@%s\n", backend.Gallery.Name, backend.Name) + fmt.Printf(" - %s@%s%s\n", backend.Gallery.Name, backend.Name, versionStr) } } return nil @@ -111,6 +134,79 @@ func (bi *BackendsInstall) Run(ctx *cliContext.Context) error { return nil } +func (bu *BackendsUpgrade) Run(ctx *cliContext.Context) error { + var galleries []config.Gallery + if err := json.Unmarshal([]byte(bu.BackendGalleries), &galleries); err != nil { + xlog.Error("unable to load galleries", "error", err) + } + + systemState, err := system.GetSystemState( + system.WithBackendSystemPath(bu.BackendsSystemPath), + system.WithBackendPath(bu.BackendsPath), + ) + if err != nil { + return err + } + + upgrades, err := gallery.CheckBackendUpgrades(context.Background(), galleries, systemState) + if err != nil { + return fmt.Errorf("failed to check for upgrades: %w", err) + } + + if len(upgrades) == 0 { + fmt.Println("All backends are up to date.") + return nil + } + + // Filter to specified backends if args given + toUpgrade := upgrades + if len(bu.BackendArgs) > 0 { + toUpgrade = make(map[string]gallery.UpgradeInfo) + for _, name := range bu.BackendArgs { + if info, ok := upgrades[name]; ok { + toUpgrade[name] = info + } else { + fmt.Printf("Backend %s: no upgrade available\n", name) + } + } + } + + if len(toUpgrade) == 0 { + fmt.Println("No upgrades to apply.") + return nil + } + + modelLoader := model.NewModelLoader(systemState) + for name, info := range toUpgrade { + versionStr := "" + if info.AvailableVersion != "" { + versionStr = " to v" + info.AvailableVersion + } + fmt.Printf("Upgrading %s%s...\n", name, versionStr) + + progressBar := progressbar.NewOptions( + 1000, + progressbar.OptionSetDescription(fmt.Sprintf("downloading %s", name)), + progressbar.OptionShowBytes(false), + progressbar.OptionClearOnFinish(), + ) + progressCallback := func(fileName string, current string, total string, percentage float64) { + v := int(percentage * 10) + if err := progressBar.Set(v); err != nil { + xlog.Error("error updating progress bar", "error", err) + } + } + + if err := gallery.UpgradeBackend(context.Background(), systemState, modelLoader, galleries, name, progressCallback); err != nil { + fmt.Printf("Failed to upgrade %s: %v\n", name, err) + } else { + fmt.Printf("Backend %s upgraded successfully\n", name) + } + } + + return nil +} + func (bu *BackendsUninstall) Run(ctx *cliContext.Context) error { for _, backendName := range bu.BackendArgs { xlog.Info("uninstalling backend", "backend", backendName) diff --git a/core/cli/run.go b/core/cli/run.go index ce31a6a2d..fe3f7c212 100644 --- a/core/cli/run.go +++ b/core/cli/run.go @@ -47,6 +47,7 @@ type RunCMD struct { BackendImagesReleaseTag string `env:"LOCALAI_BACKEND_IMAGES_RELEASE_TAG,BACKEND_IMAGES_RELEASE_TAG" help:"Fallback release tag for backend images" group:"backends" default:"latest"` BackendImagesBranchTag string `env:"LOCALAI_BACKEND_IMAGES_BRANCH_TAG,BACKEND_IMAGES_BRANCH_TAG" help:"Fallback branch tag for backend images" group:"backends" default:"master"` BackendDevSuffix string `env:"LOCALAI_BACKEND_DEV_SUFFIX,BACKEND_DEV_SUFFIX" help:"Development suffix for backend images" group:"backends" default:"development"` + AutoUpgradeBackends bool `env:"LOCALAI_AUTO_UPGRADE_BACKENDS,AUTO_UPGRADE_BACKENDS" help:"Automatically upgrade backends when new versions are detected" group:"backends" default:"false"` PreloadModels string `env:"LOCALAI_PRELOAD_MODELS,PRELOAD_MODELS" help:"A List of models to apply in JSON at start" group:"models"` Models []string `env:"LOCALAI_MODELS,MODELS" help:"A List of model configuration URLs to load" group:"models"` PreloadModelsConfig string `env:"LOCALAI_PRELOAD_MODELS_CONFIG,PRELOAD_MODELS_CONFIG" help:"A List of models to apply at startup. Path to a YAML config file" group:"models"` @@ -490,6 +491,10 @@ func (r *RunCMD) Run(ctx *cliContext.Context) error { opts = append(opts, config.EnableBackendGalleriesAutoload) } + if r.AutoUpgradeBackends { + opts = append(opts, config.WithAutoUpgradeBackends(r.AutoUpgradeBackends)) + } + if r.PreloadBackendOnly { _, err := application.New(opts...) return err diff --git a/core/config/application_config.go b/core/config/application_config.go index b4d0fd1bf..e22ad4d0c 100644 --- a/core/config/application_config.go +++ b/core/config/application_config.go @@ -57,6 +57,7 @@ type ApplicationConfig struct { ExternalGRPCBackends map[string]string AutoloadGalleries, AutoloadBackendGalleries bool + AutoUpgradeBackends bool SingleBackend bool // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends int // Maximum number of active backends (0 = unlimited, 1 = single backend mode) @@ -390,6 +391,10 @@ var EnableBackendGalleriesAutoload = func(o *ApplicationConfig) { o.AutoloadBackendGalleries = true } +func WithAutoUpgradeBackends(v bool) AppOption { + return func(o *ApplicationConfig) { o.AutoUpgradeBackends = v } +} + var EnableFederated = func(o *ApplicationConfig) { o.Federated = true } @@ -862,6 +867,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { backendGalleries := o.BackendGalleries autoloadGalleries := o.AutoloadGalleries autoloadBackendGalleries := o.AutoloadBackendGalleries + autoUpgradeBackends := o.AutoUpgradeBackends apiKeys := o.ApiKeys agentJobRetentionDays := o.AgentJobRetentionDays @@ -935,6 +941,7 @@ func (o *ApplicationConfig) ToRuntimeSettings() RuntimeSettings { BackendGalleries: &backendGalleries, AutoloadGalleries: &autoloadGalleries, AutoloadBackendGalleries: &autoloadBackendGalleries, + AutoUpgradeBackends: &autoUpgradeBackends, ApiKeys: &apiKeys, AgentJobRetentionDays: &agentJobRetentionDays, OpenResponsesStoreTTL: &openResponsesStoreTTL, @@ -1083,6 +1090,9 @@ func (o *ApplicationConfig) ApplyRuntimeSettings(settings *RuntimeSettings) (req if settings.AutoloadBackendGalleries != nil { o.AutoloadBackendGalleries = *settings.AutoloadBackendGalleries } + if settings.AutoUpgradeBackends != nil { + o.AutoUpgradeBackends = *settings.AutoUpgradeBackends + } if settings.AgentJobRetentionDays != nil { o.AgentJobRetentionDays = *settings.AgentJobRetentionDays } diff --git a/core/config/application_config_test.go b/core/config/application_config_test.go index c5559ce53..fffdeaf0b 100644 --- a/core/config/application_config_test.go +++ b/core/config/application_config_test.go @@ -119,6 +119,13 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(*rs.AgentJobRetentionDays).To(Equal(30)) }) + It("should include auto_upgrade_backends", func() { + appConfig := &ApplicationConfig{AutoUpgradeBackends: true} + rs := appConfig.ToRuntimeSettings() + Expect(rs.AutoUpgradeBackends).ToNot(BeNil()) + Expect(*rs.AutoUpgradeBackends).To(BeTrue()) + }) + It("should use default timeouts when not set", func() { appConfig := &ApplicationConfig{} @@ -426,6 +433,14 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(appConfig.AutoloadBackendGalleries).To(BeTrue()) }) + It("should apply auto_upgrade_backends setting", func() { + appConfig := &ApplicationConfig{} + v := true + rs := &RuntimeSettings{AutoUpgradeBackends: &v} + appConfig.ApplyRuntimeSettings(rs) + Expect(appConfig.AutoUpgradeBackends).To(BeTrue()) + }) + It("should apply agent settings", func() { appConfig := &ApplicationConfig{} @@ -465,6 +480,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Federated: true, AutoloadGalleries: true, AutoloadBackendGalleries: false, + AutoUpgradeBackends: true, AgentJobRetentionDays: 60, } @@ -496,6 +512,7 @@ var _ = Describe("ApplicationConfig RuntimeSettings Conversion", func() { Expect(target.Federated).To(Equal(original.Federated)) Expect(target.AutoloadGalleries).To(Equal(original.AutoloadGalleries)) Expect(target.AutoloadBackendGalleries).To(Equal(original.AutoloadBackendGalleries)) + Expect(target.AutoUpgradeBackends).To(Equal(original.AutoUpgradeBackends)) Expect(target.AgentJobRetentionDays).To(Equal(original.AgentJobRetentionDays)) }) diff --git a/core/config/runtime_settings.go b/core/config/runtime_settings.go index 6a4117f06..21a112b4d 100644 --- a/core/config/runtime_settings.go +++ b/core/config/runtime_settings.go @@ -20,6 +20,7 @@ type RuntimeSettings struct { // Backend management SingleBackend *bool `json:"single_backend,omitempty"` // Deprecated: use MaxActiveBackends = 1 instead MaxActiveBackends *int `json:"max_active_backends,omitempty"` // Maximum number of active backends (0 = unlimited, 1 = single backend mode) + AutoUpgradeBackends *bool `json:"auto_upgrade_backends,omitempty"` // Automatically upgrade backends when new versions are detected // Memory Reclaimer settings (works with GPU if available, otherwise RAM) MemoryReclaimerEnabled *bool `json:"memory_reclaimer_enabled,omitempty"` // Enable memory threshold monitoring MemoryReclaimerThreshold *float64 `json:"memory_reclaimer_threshold,omitempty"` // Threshold 0.0-1.0 (e.g., 0.95 = 95%) diff --git a/core/gallery/backend_types.go b/core/gallery/backend_types.go index 02ac9b275..ef5904f99 100644 --- a/core/gallery/backend_types.go +++ b/core/gallery/backend_types.go @@ -20,12 +20,19 @@ type BackendMetadata struct { GalleryURL string `json:"gallery_url,omitempty"` // InstalledAt is the timestamp when the backend was installed InstalledAt string `json:"installed_at,omitempty"` + // Version is the version of the backend at install time + Version string `json:"version,omitempty"` + // URI is the original URI used to install the backend + URI string `json:"uri,omitempty"` + // Digest is the OCI image digest at install time (for upgrade detection) + Digest string `json:"digest,omitempty"` } type GalleryBackend struct { Metadata `json:",inline" yaml:",inline"` Alias string `json:"alias,omitempty" yaml:"alias,omitempty"` URI string `json:"uri,omitempty" yaml:"uri,omitempty"` + Version string `json:"version,omitempty" yaml:"version,omitempty"` Mirrors []string `json:"mirrors,omitempty" yaml:"mirrors,omitempty"` CapabilitiesMap map[string]string `json:"capabilities,omitempty" yaml:"capabilities,omitempty"` } diff --git a/core/gallery/backends.go b/core/gallery/backends.go index b48aaf8a4..c2622c272 100644 --- a/core/gallery/backends.go +++ b/core/gallery/backends.go @@ -15,6 +15,7 @@ import ( "github.com/mudler/LocalAI/core/config" "github.com/mudler/LocalAI/pkg/downloader" "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" "github.com/mudler/LocalAI/pkg/system" "github.com/mudler/xlog" cp "github.com/otiai10/copy" @@ -158,6 +159,7 @@ func InstallBackendFromGallery(ctx context.Context, galleries []config.Gallery, Name: name, GalleryURL: backend.Gallery.URL, InstalledAt: time.Now().Format(time.RFC3339), + Version: bestBackend.Version, } if err := writeBackendMetadata(metaBackendPath, metaMetadata); err != nil { @@ -279,6 +281,18 @@ func InstallBackend(ctx context.Context, systemState *system.SystemState, modelL Name: name, GalleryURL: config.Gallery.URL, InstalledAt: time.Now().Format(time.RFC3339), + Version: config.Version, + URI: string(uri), + } + + // Record the OCI digest for upgrade detection (non-fatal on failure) + if uri.LooksLikeOCI() { + digest, digestErr := oci.GetImageDigest(string(uri), "", nil, nil) + if digestErr != nil { + xlog.Warn("Failed to get OCI image digest for backend", "uri", string(uri), "error", digestErr) + } else { + metadata.Digest = digest + } } if config.Alias != "" { @@ -373,11 +387,13 @@ func DeleteBackendFromSystem(systemState *system.SystemState, name string) error } type SystemBackend struct { - Name string - RunFile string - IsMeta bool - IsSystem bool - Metadata *BackendMetadata + Name string + RunFile string + IsMeta bool + IsSystem bool + Metadata *BackendMetadata + UpgradeAvailable bool `json:"upgrade_available,omitempty"` + AvailableVersion string `json:"available_version,omitempty"` } type SystemBackends map[string]SystemBackend diff --git a/core/gallery/backends_version_test.go b/core/gallery/backends_version_test.go new file mode 100644 index 000000000..8e97604d4 --- /dev/null +++ b/core/gallery/backends_version_test.go @@ -0,0 +1,118 @@ +package gallery_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + + "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("Backend versioning", func() { + var tempDir string + var systemState *system.SystemState + var modelLoader *model.ModelLoader + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "gallery-version-*") + Expect(err).NotTo(HaveOccurred()) + + systemState, err = system.GetSystemState( + system.WithBackendPath(tempDir), + ) + Expect(err).NotTo(HaveOccurred()) + modelLoader = model.NewModelLoader(systemState) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + It("records version in metadata when installing a backend with a version", func() { + // Create a fake backend source directory with a run.sh + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend" + backend.URI = srcDir + backend.Version = "1.2.3" + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + // Read the metadata file and check version + metadataPath := filepath.Join(tempDir, "test-backend", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + Expect(metadata["version"]).To(Equal("1.2.3")) + }) + + It("records URI in metadata", func() { + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend-uri" + backend.URI = srcDir + backend.Version = "2.0.0" + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-backend-uri", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + Expect(metadata["uri"]).To(Equal(srcDir)) + }) + + It("omits version key when version is empty", func() { + srcDir, err := os.MkdirTemp("", "gallery-version-src-*") + Expect(err).NotTo(HaveOccurred()) + defer os.RemoveAll(srcDir) + err = os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho ok"), 0755) + Expect(err).NotTo(HaveOccurred()) + + backend := &gallery.GalleryBackend{} + backend.Name = "test-backend-noversion" + backend.URI = srcDir + // Version intentionally left empty + + err = gallery.InstallBackend(context.Background(), systemState, modelLoader, backend, nil) + Expect(err).NotTo(HaveOccurred()) + + metadataPath := filepath.Join(tempDir, "test-backend-noversion", "metadata.json") + data, err := os.ReadFile(metadataPath) + Expect(err).NotTo(HaveOccurred()) + + var metadata map[string]any + err = json.Unmarshal(data, &metadata) + Expect(err).NotTo(HaveOccurred()) + + // omitempty should exclude the version key entirely + _, hasVersion := metadata["version"] + Expect(hasVersion).To(BeFalse()) + }) +}) diff --git a/core/gallery/upgrade.go b/core/gallery/upgrade.go new file mode 100644 index 000000000..aec75852c --- /dev/null +++ b/core/gallery/upgrade.go @@ -0,0 +1,223 @@ +package gallery + +import ( + "context" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/pkg/downloader" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/oci" + "github.com/mudler/LocalAI/pkg/system" + "github.com/mudler/xlog" + cp "github.com/otiai10/copy" +) + +// UpgradeInfo holds details about an available backend upgrade. +type UpgradeInfo struct { + BackendName string `json:"backend_name"` + InstalledVersion string `json:"installed_version"` + AvailableVersion string `json:"available_version"` + InstalledDigest string `json:"installed_digest,omitempty"` + AvailableDigest string `json:"available_digest,omitempty"` +} + +// CheckBackendUpgrades compares installed backends against gallery entries +// and returns a map of backend names to UpgradeInfo for those that have +// newer versions or different OCI digests available. +func CheckBackendUpgrades(ctx context.Context, galleries []config.Gallery, systemState *system.SystemState) (map[string]UpgradeInfo, error) { + galleryBackends, err := AvailableBackends(galleries, systemState) + if err != nil { + return nil, fmt.Errorf("failed to list available backends: %w", err) + } + + installedBackends, err := ListSystemBackends(systemState) + if err != nil { + return nil, fmt.Errorf("failed to list installed backends: %w", err) + } + + result := make(map[string]UpgradeInfo) + + for _, installed := range installedBackends { + // Skip system backends — they are managed outside the gallery + if installed.IsSystem { + continue + } + if installed.Metadata == nil { + continue + } + + // Find matching gallery entry by metadata name + galleryEntry := FindGalleryElement(galleryBackends, installed.Metadata.Name) + if galleryEntry == nil { + continue + } + + installedVersion := installed.Metadata.Version + galleryVersion := galleryEntry.Version + + // If both sides have versions, compare them + if galleryVersion != "" && installedVersion != "" { + if galleryVersion != installedVersion { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledVersion: installedVersion, + AvailableVersion: galleryVersion, + } + } + // Versions match — no upgrade needed + continue + } + + // If either version is empty, fall back to OCI digest comparison + if installed.Metadata.Digest != "" && downloader.URI(galleryEntry.URI).LooksLikeOCI() { + remoteDigest, err := oci.GetImageDigest(galleryEntry.URI, "", nil, nil) + if err != nil { + xlog.Warn("Failed to get remote OCI digest for upgrade check", "backend", installed.Metadata.Name, "error", err) + continue + } + if remoteDigest != installed.Metadata.Digest { + result[installed.Metadata.Name] = UpgradeInfo{ + BackendName: installed.Metadata.Name, + InstalledDigest: installed.Metadata.Digest, + AvailableDigest: remoteDigest, + } + } + } + // No version info and no digest to compare — skip + } + + return result, nil +} + +// UpgradeBackend upgrades a single backend to the latest gallery version using +// an atomic swap with backup-based rollback on failure. +func UpgradeBackend(ctx context.Context, systemState *system.SystemState, modelLoader *model.ModelLoader, galleries []config.Gallery, backendName string, downloadStatus func(string, string, string, float64)) error { + // Look up the installed backend + installedBackends, err := ListSystemBackends(systemState) + if err != nil { + return fmt.Errorf("failed to list installed backends: %w", err) + } + + installed, ok := installedBackends.Get(backendName) + if !ok { + return fmt.Errorf("backend %q: %w", backendName, ErrBackendNotFound) + } + + if installed.IsSystem { + return fmt.Errorf("system backend %q cannot be upgraded via gallery", backendName) + } + + // If this is a meta backend, recursively upgrade the concrete backend it points to + if installed.Metadata != nil && installed.Metadata.MetaBackendFor != "" { + xlog.Info("Meta backend detected, upgrading concrete backend", "meta", backendName, "concrete", installed.Metadata.MetaBackendFor) + return UpgradeBackend(ctx, systemState, modelLoader, galleries, installed.Metadata.MetaBackendFor, downloadStatus) + } + + // Find the gallery entry + galleryBackends, err := AvailableBackends(galleries, systemState) + if err != nil { + return fmt.Errorf("failed to list available backends: %w", err) + } + + galleryEntry := FindGalleryElement(galleryBackends, backendName) + if galleryEntry == nil { + return fmt.Errorf("no gallery entry found for backend %q", backendName) + } + + backendPath := filepath.Join(systemState.Backend.BackendsPath, backendName) + tmpPath := backendPath + ".upgrade-tmp" + backupPath := backendPath + ".backup" + + // Clean up any stale tmp/backup dirs from prior attempts + os.RemoveAll(tmpPath) + os.RemoveAll(backupPath) + + // Step 1: Download the new backend into the tmp directory + if err := os.MkdirAll(tmpPath, 0750); err != nil { + return fmt.Errorf("failed to create upgrade tmp dir: %w", err) + } + + uri := downloader.URI(galleryEntry.URI) + if uri.LooksLikeDir() { + if err := cp.Copy(string(uri), tmpPath); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to copy backend from directory: %w", err) + } + } else { + if err := uri.DownloadFileWithContext(ctx, tmpPath, "", 1, 1, downloadStatus); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to download backend: %w", err) + } + } + + // Step 2: Validate — check that run.sh exists in the new content + newRunFile := filepath.Join(tmpPath, runFile) + if _, err := os.Stat(newRunFile); os.IsNotExist(err) { + os.RemoveAll(tmpPath) + return fmt.Errorf("upgrade validation failed: run.sh not found in new backend") + } + + // Step 3: Atomic swap — rename current to backup, then tmp to current + if err := os.Rename(backendPath, backupPath); err != nil { + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to move current backend to backup: %w", err) + } + + if err := os.Rename(tmpPath, backendPath); err != nil { + // Restore backup on failure + xlog.Error("Failed to move new backend into place, restoring backup", "error", err) + if restoreErr := os.Rename(backupPath, backendPath); restoreErr != nil { + xlog.Error("Failed to restore backup", "error", restoreErr) + } + os.RemoveAll(tmpPath) + return fmt.Errorf("failed to move new backend into place: %w", err) + } + + // Step 4: Write updated metadata, preserving alias from old metadata + var oldAlias string + if installed.Metadata != nil { + oldAlias = installed.Metadata.Alias + } + + newMetadata := &BackendMetadata{ + Name: backendName, + Version: galleryEntry.Version, + URI: galleryEntry.URI, + InstalledAt: time.Now().Format(time.RFC3339), + Alias: oldAlias, + } + + if galleryEntry.Gallery.URL != "" { + newMetadata.GalleryURL = galleryEntry.Gallery.URL + } + + // Record OCI digest if applicable (non-fatal on failure) + if uri.LooksLikeOCI() { + digest, digestErr := oci.GetImageDigest(galleryEntry.URI, "", nil, nil) + if digestErr != nil { + xlog.Warn("Failed to get OCI image digest after upgrade", "uri", galleryEntry.URI, "error", digestErr) + } else { + newMetadata.Digest = digest + } + } + + if err := writeBackendMetadata(backendPath, newMetadata); err != nil { + // Metadata write failure is not worth rolling back the entire upgrade + xlog.Error("Failed to write metadata after upgrade", "error", err) + } + + // Step 5: Re-register backends so the model loader picks up any changes + if err := RegisterBackends(systemState, modelLoader); err != nil { + xlog.Warn("Failed to re-register backends after upgrade", "error", err) + } + + // Step 6: Remove backup + os.RemoveAll(backupPath) + + xlog.Info("Backend upgraded successfully", "backend", backendName, "version", galleryEntry.Version) + return nil +} diff --git a/core/gallery/upgrade_test.go b/core/gallery/upgrade_test.go new file mode 100644 index 000000000..f65b4276b --- /dev/null +++ b/core/gallery/upgrade_test.go @@ -0,0 +1,219 @@ +package gallery_test + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/mudler/LocalAI/core/config" + . "github.com/mudler/LocalAI/core/gallery" + "github.com/mudler/LocalAI/pkg/model" + "github.com/mudler/LocalAI/pkg/system" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" +) + +var _ = Describe("Upgrade Detection and Execution", func() { + var ( + tempDir string + backendsPath string + galleryPath string + systemState *system.SystemState + galleries []config.Gallery + ) + + // installBackendWithVersion creates a fake installed backend directory with + // the given name, version, and optional run.sh content. + installBackendWithVersion := func(name, version string, runContent ...string) { + dir := filepath.Join(backendsPath, name) + Expect(os.MkdirAll(dir, 0750)).To(Succeed()) + + content := "#!/bin/sh\necho ok" + if len(runContent) > 0 { + content = runContent[0] + } + Expect(os.WriteFile(filepath.Join(dir, "run.sh"), []byte(content), 0755)).To(Succeed()) + + metadata := BackendMetadata{ + Name: name, + Version: version, + InstalledAt: time.Now().Format(time.RFC3339), + } + data, err := json.MarshalIndent(metadata, "", " ") + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(filepath.Join(dir, "metadata.json"), data, 0644)).To(Succeed()) + } + + // writeGalleryYAML writes a gallery YAML file with the given backends. + writeGalleryYAML := func(backends []GalleryBackend) { + data, err := yaml.Marshal(backends) + Expect(err).NotTo(HaveOccurred()) + Expect(os.WriteFile(galleryPath, data, 0644)).To(Succeed()) + } + + BeforeEach(func() { + var err error + tempDir, err = os.MkdirTemp("", "upgrade-test-*") + Expect(err).NotTo(HaveOccurred()) + + backendsPath = tempDir + + galleryPath = filepath.Join(tempDir, "gallery.yaml") + + // Write a default empty gallery + writeGalleryYAML([]GalleryBackend{}) + + galleries = []config.Gallery{ + { + Name: "test-gallery", + URL: "file://" + galleryPath, + }, + } + + systemState, err = system.GetSystemState( + system.WithBackendPath(backendsPath), + ) + Expect(err).NotTo(HaveOccurred()) + }) + + AfterEach(func() { + os.RemoveAll(tempDir) + }) + + Describe("CheckBackendUpgrades", func() { + It("should detect upgrade when gallery version differs from installed version", func() { + // Install a backend at v1.0.0 + installBackendWithVersion("my-backend", "1.0.0") + + // Gallery advertises v2.0.0 + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + Version: "2.0.0", + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(HaveKey("my-backend")) + Expect(upgrades["my-backend"].InstalledVersion).To(Equal("1.0.0")) + Expect(upgrades["my-backend"].AvailableVersion).To(Equal("2.0.0")) + }) + + It("should NOT flag upgrade when versions match", func() { + installBackendWithVersion("my-backend", "2.0.0") + + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + Version: "2.0.0", + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + + It("should skip backends without version info and without OCI digest", func() { + // Install without version + installBackendWithVersion("my-backend", "") + + // Gallery also without version + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: filepath.Join(tempDir, "some-source"), + }, + }) + + upgrades, err := CheckBackendUpgrades(context.Background(), galleries, systemState) + Expect(err).NotTo(HaveOccurred()) + Expect(upgrades).To(BeEmpty()) + }) + }) + + Describe("UpgradeBackend", func() { + It("should replace backend directory and update metadata", func() { + // Install v1 + installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1") + + // Create a source directory with v2 content + srcDir := filepath.Join(tempDir, "v2-source") + Expect(os.MkdirAll(srcDir, 0750)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(srcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755)).To(Succeed()) + + // Gallery points to the v2 source dir + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: srcDir, + Version: "2.0.0", + }, + }) + + ml := model.NewModelLoader(systemState) + err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil) + Expect(err).NotTo(HaveOccurred()) + + // Verify run.sh was updated + content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal("#!/bin/sh\necho v2")) + + // Verify metadata was updated + metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + var meta BackendMetadata + Expect(json.Unmarshal(metaData, &meta)).To(Succeed()) + Expect(meta.Version).To(Equal("2.0.0")) + Expect(meta.Name).To(Equal("my-backend")) + }) + + It("should restore backup on failure", func() { + // Install v1 + installBackendWithVersion("my-backend", "1.0.0", "#!/bin/sh\necho v1") + + // Gallery points to a nonexistent path (no run.sh will be found) + nonExistentDir := filepath.Join(tempDir, "does-not-exist") + writeGalleryYAML([]GalleryBackend{ + { + Metadata: Metadata{ + Name: "my-backend", + }, + URI: nonExistentDir, + Version: "2.0.0", + }, + }) + + ml := model.NewModelLoader(systemState) + err := UpgradeBackend(context.Background(), systemState, ml, galleries, "my-backend", nil) + Expect(err).To(HaveOccurred()) + + // Verify v1 is still intact + content, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(content)).To(Equal("#!/bin/sh\necho v1")) + + // Verify metadata still says v1 + metaData, err := os.ReadFile(filepath.Join(backendsPath, "my-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + var meta BackendMetadata + Expect(json.Unmarshal(metaData, &meta)).To(Succeed()) + Expect(meta.Version).To(Equal("1.0.0")) + }) + }) +}) diff --git a/core/http/endpoints/localai/backend.go b/core/http/endpoints/localai/backend.go index a67ba4da0..5cd2ba060 100644 --- a/core/http/endpoints/localai/backend.go +++ b/core/http/endpoints/localai/backend.go @@ -15,23 +15,31 @@ import ( "github.com/mudler/xlog" ) +// UpgradeInfoProvider is an interface for querying cached backend upgrade information. +type UpgradeInfoProvider interface { + GetAvailableUpgrades() map[string]gallery.UpgradeInfo + TriggerCheck() +} + type BackendEndpointService struct { galleries []config.Gallery backendPath string backendSystemPath string backendApplier *galleryop.GalleryService + upgradeChecker UpgradeInfoProvider } type GalleryBackend struct { ID string `json:"id"` } -func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService) BackendEndpointService { +func CreateBackendEndpointService(galleries []config.Gallery, systemState *system.SystemState, backendApplier *galleryop.GalleryService, upgradeChecker UpgradeInfoProvider) BackendEndpointService { return BackendEndpointService{ galleries: galleries, backendPath: systemState.Backend.BackendsPath, backendSystemPath: systemState.Backend.BackendsSystemPath, backendApplier: backendApplier, + upgradeChecker: upgradeChecker, } } @@ -146,6 +154,62 @@ func (mgs *BackendEndpointService) ListBackendGalleriesEndpoint() echo.HandlerFu } } +// GetUpgradesEndpoint returns the cached backend upgrade information +// @Summary Get available backend upgrades +// @Tags backends +// @Success 200 {object} map[string]gallery.UpgradeInfo "Response" +// @Router /backends/upgrades [get] +func (mgs *BackendEndpointService) GetUpgradesEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + if mgs.upgradeChecker == nil { + return c.JSON(200, map[string]gallery.UpgradeInfo{}) + } + return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades()) + } +} + +// CheckUpgradesEndpoint forces an immediate upgrade check +// @Summary Force backend upgrade check +// @Tags backends +// @Success 200 {object} map[string]gallery.UpgradeInfo "Response" +// @Router /backends/upgrades/check [post] +func (mgs *BackendEndpointService) CheckUpgradesEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + if mgs.upgradeChecker == nil { + return c.JSON(200, map[string]gallery.UpgradeInfo{}) + } + mgs.upgradeChecker.TriggerCheck() + // Return current cached results (the triggered check runs async) + return c.JSON(200, mgs.upgradeChecker.GetAvailableUpgrades()) + } +} + +// UpgradeBackendEndpoint triggers an upgrade for a specific backend +// @Summary Upgrade a backend +// @Tags backends +// @Param name path string true "Backend name" +// @Success 200 {object} schema.BackendResponse "Response" +// @Router /backends/upgrade/{name} [post] +func (mgs *BackendEndpointService) UpgradeBackendEndpoint() echo.HandlerFunc { + return func(c echo.Context) error { + backendName := c.Param("name") + + uuid, err := uuid.NewUUID() + if err != nil { + return err + } + + mgs.backendApplier.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{ + ID: uuid.String(), + GalleryElementName: backendName, + Galleries: mgs.galleries, + Upgrade: true, + } + + return c.JSON(200, schema.BackendResponse{ID: uuid.String(), StatusURL: fmt.Sprintf("%sbackends/jobs/%s", middleware.BaseURL(c), uuid.String())}) + } +} + // ListAvailableBackendsEndpoint list the available backends in the galleries configured in LocalAI // @Summary List all available Backends // @Tags backends diff --git a/core/http/react-ui/src/pages/Backends.jsx b/core/http/react-ui/src/pages/Backends.jsx index 4e73a9669..3cbb71872 100644 --- a/core/http/react-ui/src/pages/Backends.jsx +++ b/core/http/react-ui/src/pages/Backends.jsx @@ -26,6 +26,7 @@ export default function Backends() { const [expandedRow, setExpandedRow] = useState(null) const [confirmDialog, setConfirmDialog] = useState(null) const [allBackends, setAllBackends] = useState([]) + const [upgrades, setUpgrades] = useState({}) const fetchBackends = useCallback(async () => { try { @@ -52,6 +53,13 @@ export default function Backends() { if (!loading) fetchBackends() }, [operations.length]) + // Fetch available upgrades + useEffect(() => { + backendsApi.checkUpgrades() + .then(data => setUpgrades(data || {})) + .catch(() => {}) + }, [operations.length]) + // Client-side filtering by tag const filteredBackends = filter ? allBackends.filter(b => { @@ -114,6 +122,15 @@ export default function Backends() { }) } + const handleUpgrade = async (id) => { + try { + await backendsApi.upgrade(id) + addToast(`Upgrading ${id}...`, 'info') + } catch (err) { + addToast(`Upgrade failed: ${err.message}`, 'error') + } + } + const handleManualInstall = async (e) => { e.preventDefault() if (!manualUri.trim()) { addToast('Please enter a URI', 'warning'); return } @@ -179,6 +196,14 @@ export default function Backends() {
Installed
+ {Object.keys(upgrades).length > 0 && ( +
+
+ {Object.keys(upgrades).length} +
+
Updates
+
+ )} Docs @@ -300,6 +325,11 @@ export default function Backends() { {/* Name */} {b.name || b.id} + {b.version && ( + + v{b.version} + + )} {/* Description */} @@ -346,9 +376,17 @@ export default function Backends() { ) : b.installed ? ( - - Installed - +
+ + Installed + + {upgrades[b.name] && ( + + + {upgrades[b.name].available_version ? `v${upgrades[b.name].available_version}` : 'Update'} + + )} +
) : ( Not Installed @@ -361,9 +399,15 @@ export default function Backends() {
e.stopPropagation()}> {b.installed ? ( <> - + {upgrades[b.name] ? ( + + ) : ( + + )} diff --git a/core/http/react-ui/src/pages/Manage.jsx b/core/http/react-ui/src/pages/Manage.jsx index 6ac429ff4..3d24ebea8 100644 --- a/core/http/react-ui/src/pages/Manage.jsx +++ b/core/http/react-ui/src/pages/Manage.jsx @@ -22,6 +22,7 @@ export default function Manage() { const [backendsLoading, setBackendsLoading] = useState(true) const [reloading, setReloading] = useState(false) const [reinstallingBackends, setReinstallingBackends] = useState(new Set()) + const [upgrades, setUpgrades] = useState({}) const [confirmDialog, setConfirmDialog] = useState(null) const [distributedMode, setDistributedMode] = useState(false) const [togglingModels, setTogglingModels] = useState(new Set()) @@ -62,6 +63,15 @@ export default function Manage() { nodesApi.list().then(() => setDistributedMode(true)).catch(() => {}) }, [fetchLoadedModels, fetchBackends]) + // Fetch available backend upgrades + useEffect(() => { + if (activeTab === 'backends') { + backendsApi.checkUpgrades() + .then(data => setUpgrades(data || {})) + .catch(() => {}) + } + }, [activeTab]) + const handleStopModel = (modelName) => { setConfirmDialog({ title: 'Stop Model', @@ -169,6 +179,22 @@ export default function Manage() { } } + const handleUpgradeBackend = async (name) => { + try { + setReinstallingBackends(prev => new Set(prev).add(name)) + await backendsApi.upgrade(name) + addToast(`Upgrading ${name}...`, 'info') + } catch (err) { + addToast(`Failed to upgrade: ${err.message}`, 'error') + } finally { + setReinstallingBackends(prev => { + const next = new Set(prev) + next.delete(name) + return next + }) + } + } + const handleDeleteBackend = (name) => { setConfirmDialog({ title: 'Delete Backend', @@ -471,6 +497,17 @@ export default function Manage() { For: {backend.Metadata.meta_backend_for} )} + {backend.Metadata?.version && ( + + + Version: v{backend.Metadata.version} + {upgrades[backend.Name] && ( + + → v{upgrades[backend.Name].available_version} + + )} + + )} {backend.Metadata?.installed_at && ( @@ -485,12 +522,12 @@ export default function Manage() { {!backend.IsSystem ? ( <>
diff --git a/core/http/react-ui/src/utils/api.js b/core/http/react-ui/src/utils/api.js index b9d3983d2..1edac8cbd 100644 --- a/core/http/react-ui/src/utils/api.js +++ b/core/http/react-ui/src/utils/api.js @@ -120,6 +120,9 @@ export const backendsApi = { installExternal: (body) => postJSON(API_CONFIG.endpoints.installExternalBackend, body), getJob: (uid) => fetchJSON(API_CONFIG.endpoints.backendJob(uid)), deleteInstalled: (name) => postJSON(API_CONFIG.endpoints.deleteInstalledBackend(name), {}), + checkUpgrades: () => fetchJSON(API_CONFIG.endpoints.backendsUpgrades), + forceCheckUpgrades: () => postJSON(API_CONFIG.endpoints.backendsUpgradesCheck, {}), + upgrade: (name) => postJSON(API_CONFIG.endpoints.upgradeBackend(name), {}), } // Chat API (non-streaming) diff --git a/core/http/react-ui/src/utils/config.js b/core/http/react-ui/src/utils/config.js index 70fb7f57a..66767eb3f 100644 --- a/core/http/react-ui/src/utils/config.js +++ b/core/http/react-ui/src/utils/config.js @@ -23,6 +23,9 @@ export const API_CONFIG = { installExternalBackend: '/api/backends/install-external', backendJob: (uid) => `/api/backends/job/${uid}`, deleteInstalledBackend: (name) => `/api/backends/system/delete/${name}`, + backendsUpgrades: '/api/backends/upgrades', + backendsUpgradesCheck: '/api/backends/upgrades/check', + upgradeBackend: (name) => `/api/backends/upgrade/${name}`, // Resources resources: '/api/resources', diff --git a/core/http/routes/localai.go b/core/http/routes/localai.go index 48cad2fb0..d5e41a9c5 100644 --- a/core/http/routes/localai.go +++ b/core/http/routes/localai.go @@ -59,13 +59,17 @@ func RegisterLocalAIRoutes(router *echo.Echo, backendGalleryEndpointService := localai.CreateBackendEndpointService( appConfig.BackendGalleries, appConfig.SystemState, - galleryService) + galleryService, + app.UpgradeChecker()) router.POST("/backends/apply", backendGalleryEndpointService.ApplyBackendEndpoint(), adminMiddleware) router.POST("/backends/delete/:name", backendGalleryEndpointService.DeleteBackendEndpoint(), adminMiddleware) router.GET("/backends", backendGalleryEndpointService.ListBackendsEndpoint(), adminMiddleware) router.GET("/backends/available", backendGalleryEndpointService.ListAvailableBackendsEndpoint(appConfig.SystemState), adminMiddleware) router.GET("/backends/galleries", backendGalleryEndpointService.ListBackendGalleriesEndpoint(), adminMiddleware) router.GET("/backends/jobs/:uuid", backendGalleryEndpointService.GetOpStatusEndpoint(), adminMiddleware) + router.GET("/backends/upgrades", backendGalleryEndpointService.GetUpgradesEndpoint(), adminMiddleware) + router.POST("/backends/upgrades/check", backendGalleryEndpointService.CheckUpgradesEndpoint(), adminMiddleware) + router.POST("/backends/upgrade/:name", backendGalleryEndpointService.UpgradeBackendEndpoint(), adminMiddleware) // Custom model import endpoint router.POST("/models/import", localai.ImportModelEndpoint(cl, appConfig), adminMiddleware) diff --git a/core/http/routes/ui_api.go b/core/http/routes/ui_api.go index 570e96e1e..a34d22401 100644 --- a/core/http/routes/ui_api.go +++ b/core/http/routes/ui_api.go @@ -929,6 +929,7 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model "tags": b.Tags, "gallery": b.Gallery.Name, "installed": b.Installed, + "version": b.Version, "processing": currentlyProcessing, "jobID": jobID, "isDeletion": isDeletionOp, @@ -1194,6 +1195,49 @@ func RegisterUIAPIRoutes(app *echo.Echo, cl *config.ModelConfigLoader, ml *model }) }, adminMiddleware) + // Backend upgrade APIs + app.GET("/api/backends/upgrades", func(c echo.Context) error { + if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil { + return c.JSON(200, map[string]any{}) + } + return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades()) + }, adminMiddleware) + + app.POST("/api/backends/upgrades/check", func(c echo.Context) error { + if applicationInstance == nil || applicationInstance.UpgradeChecker() == nil { + return c.JSON(200, map[string]any{}) + } + applicationInstance.UpgradeChecker().TriggerCheck() + return c.JSON(200, applicationInstance.UpgradeChecker().GetAvailableUpgrades()) + }, adminMiddleware) + + app.POST("/api/backends/upgrade/:name", func(c echo.Context) error { + backendName := c.Param("name") + backendName, err := url.QueryUnescape(backendName) + if err != nil { + return c.JSON(http.StatusBadRequest, map[string]any{ + "error": "invalid backend name", + }) + } + + uid, err := uuid.NewUUID() + if err != nil { + return c.JSON(http.StatusInternalServerError, map[string]any{"error": err.Error()}) + } + + galleryService.BackendGalleryChannel <- galleryop.ManagementOp[gallery.GalleryBackend, any]{ + ID: uid.String(), + GalleryElementName: backendName, + Galleries: appConfig.BackendGalleries, + Upgrade: true, + } + + return c.JSON(200, map[string]any{ + "uuid": uid.String(), + "statusUrl": fmt.Sprintf("/api/backends/job/%s", uid.String()), + }) + }, adminMiddleware) + // P2P APIs app.GET("/api/p2p/workers", func(c echo.Context) error { llamaNodes := p2p.GetAvailableNodes(p2p.NetworkID(appConfig.P2PNetworkID, p2p.LlamaCPPWorkerID)) diff --git a/core/http/routes/ui_api_backends_test.go b/core/http/routes/ui_api_backends_test.go index 4e4e5014d..4c46d387d 100644 --- a/core/http/routes/ui_api_backends_test.go +++ b/core/http/routes/ui_api_backends_test.go @@ -20,6 +20,7 @@ import ( "github.com/mudler/LocalAI/pkg/system" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" + "gopkg.in/yaml.v3" ) func TestRoutes(t *testing.T) { @@ -176,6 +177,161 @@ var _ = Describe("Backend API Routes", func() { Expect(response["processed"]).To(Equal(false)) }) }) + + Describe("Backend upgrade API", func() { + var ( + galleryFile string + upgradeApp *echo.Echo + upgradeGallerySvc *galleryop.GalleryService + ) + + BeforeEach(func() { + // Place gallery file inside backends dir so it passes trusted root checks + galleryFile = filepath.Join(systemState.Backend.BackendsPath, "test-gallery.yaml") + + // Create a fake "v1" backend on disk (simulates a previously installed backend) + backendDir := filepath.Join(systemState.Backend.BackendsPath, "test-upgrade-backend") + err := os.MkdirAll(backendDir, 0750) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(backendDir, "run.sh"), []byte("#!/bin/sh\necho v1"), 0755) + Expect(err).NotTo(HaveOccurred()) + + // Write metadata.json for the installed backend (v1) + metadata := map[string]string{ + "name": "test-upgrade-backend", + "version": "1.0.0", + "installed_at": "2024-01-01T00:00:00Z", + } + metadataBytes, err := json.MarshalIndent(metadata, "", " ") + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(backendDir, "metadata.json"), metadataBytes, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Create a "v2" source directory (the upgrade target) + // Must be inside backends path to pass trusted root checks + v2SrcDir := filepath.Join(systemState.Backend.BackendsPath, "v2-backend-src") + err = os.MkdirAll(v2SrcDir, 0750) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(filepath.Join(v2SrcDir, "run.sh"), []byte("#!/bin/sh\necho v2"), 0755) + Expect(err).NotTo(HaveOccurred()) + + // Write gallery YAML pointing to v2 + galleryData := []map[string]any{ + { + "name": "test-upgrade-backend", + "uri": v2SrcDir, + "version": "2.0.0", + }, + } + yamlBytes, err := yaml.Marshal(galleryData) + Expect(err).NotTo(HaveOccurred()) + err = os.WriteFile(galleryFile, yamlBytes, 0644) + Expect(err).NotTo(HaveOccurred()) + + // Configure the gallery in appConfig BEFORE creating the gallery service + // so the backend manager captures the correct galleries + appConfig.BackendGalleries = []config.Gallery{ + {Name: "test", URL: "file://" + galleryFile}, + } + + // Create a fresh gallery service with the upgrade gallery configured + upgradeGallerySvc = galleryop.NewGalleryService(appConfig, modelLoader) + err = upgradeGallerySvc.Start(context.Background(), configLoader, systemState) + Expect(err).NotTo(HaveOccurred()) + + // Register routes with the upgrade-aware gallery service + upgradeApp = echo.New() + opcache := galleryop.NewOpCache(upgradeGallerySvc) + noopMw := func(next echo.HandlerFunc) echo.HandlerFunc { return next } + routes.RegisterUIAPIRoutes(upgradeApp, configLoader, modelLoader, appConfig, upgradeGallerySvc, opcache, nil, noopMw) + }) + + Describe("GET /api/backends/upgrades", func() { + It("should return available upgrades", func() { + req := httptest.NewRequest(http.MethodGet, "/api/backends/upgrades", nil) + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + // Response is empty (upgrade checker not running in test), + // but the endpoint should not error + }) + }) + + Describe("POST /api/backends/upgrade/:name", func() { + It("should accept upgrade request and return job ID", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrade/test-upgrade-backend", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + Expect(response["uuid"]).NotTo(BeEmpty()) + Expect(response["statusUrl"]).NotTo(BeEmpty()) + }) + + It("should upgrade the backend and update metadata", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrade/test-upgrade-backend", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + Expect(rec.Code).To(Equal(http.StatusOK)) + + var response map[string]any + err := json.Unmarshal(rec.Body.Bytes(), &response) + Expect(err).NotTo(HaveOccurred()) + jobID := response["uuid"].(string) + + // Wait for the upgrade job to complete + Eventually(func() bool { + jobReq := httptest.NewRequest(http.MethodGet, "/api/backends/job/"+jobID, nil) + jobRec := httptest.NewRecorder() + upgradeApp.ServeHTTP(jobRec, jobReq) + + var jobResp map[string]any + json.Unmarshal(jobRec.Body.Bytes(), &jobResp) + + processed, _ := jobResp["processed"].(bool) + return processed + }, "10s", "200ms").Should(BeTrue()) + + // Verify the backend was upgraded: run.sh should now contain "v2" + runContent, err := os.ReadFile(filepath.Join( + systemState.Backend.BackendsPath, "test-upgrade-backend", "run.sh")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(runContent)).To(ContainSubstring("v2")) + + // Verify metadata was updated with new version + metadataContent, err := os.ReadFile(filepath.Join( + systemState.Backend.BackendsPath, "test-upgrade-backend", "metadata.json")) + Expect(err).NotTo(HaveOccurred()) + Expect(string(metadataContent)).To(ContainSubstring(`"version": "2.0.0"`)) + }) + }) + + Describe("POST /api/backends/upgrades/check", func() { + It("should trigger an upgrade check and return 200", func() { + req := httptest.NewRequest(http.MethodPost, "/api/backends/upgrades/check", nil) + req.Header.Set("Content-Type", "application/json") + rec := httptest.NewRecorder() + + upgradeApp.ServeHTTP(rec, req) + + Expect(rec.Code).To(Equal(http.StatusOK)) + }) + }) + }) }) // Helper function to make POST request diff --git a/core/services/advisorylock/keys.go b/core/services/advisorylock/keys.go index ebca65932..d5378a5d1 100644 --- a/core/services/advisorylock/keys.go +++ b/core/services/advisorylock/keys.go @@ -9,5 +9,6 @@ const ( KeyGalleryDedup int64 = 102 KeyAgentScheduler int64 = 103 KeyHealthCheck int64 = 104 - KeySchemaMigrate int64 = 105 + KeySchemaMigrate int64 = 105 + KeyBackendUpgradeCheck int64 = 106 ) diff --git a/core/services/galleryop/backends.go b/core/services/galleryop/backends.go index 05430026b..b8680f08a 100644 --- a/core/services/galleryop/backends.go +++ b/core/services/galleryop/backends.go @@ -69,7 +69,9 @@ func (g *GalleryService) backendHandler(op *ManagementOp[gallery.GalleryBackend, } var err error - if op.Delete { + if op.Upgrade { + err = g.backendManager.UpgradeBackend(ctx, op.GalleryElementName, progressCallback) + } else if op.Delete { err = g.backendManager.DeleteBackend(op.GalleryElementName) } else { err = g.backendManager.InstallBackend(ctx, op, progressCallback) diff --git a/core/services/galleryop/managers.go b/core/services/galleryop/managers.go index 7857fba19..5adbef8c7 100644 --- a/core/services/galleryop/managers.go +++ b/core/services/galleryop/managers.go @@ -15,9 +15,11 @@ type ModelManager interface { DeleteModel(name string) error } -// BackendManager handles backend install, delete, and listing lifecycle. +// BackendManager handles backend install, delete, upgrade, and listing lifecycle. type BackendManager interface { InstallBackend(ctx context.Context, op *ManagementOp[gallery.GalleryBackend, any], progressCb ProgressCallback) error DeleteBackend(name string) error ListBackends() (gallery.SystemBackends, error) + UpgradeBackend(ctx context.Context, name string, progressCb ProgressCallback) error + CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) } diff --git a/core/services/galleryop/managers_local.go b/core/services/galleryop/managers_local.go index 3f8055372..2d86f2dc6 100644 --- a/core/services/galleryop/managers_local.go +++ b/core/services/galleryop/managers_local.go @@ -92,6 +92,14 @@ func (b *LocalBackendManager) ListBackends() (gallery.SystemBackends, error) { return gallery.ListSystemBackends(b.systemState) } +func (b *LocalBackendManager) UpgradeBackend(ctx context.Context, name string, progressCb ProgressCallback) error { + return gallery.UpgradeBackend(ctx, b.systemState, b.modelLoader, b.backendGalleries, name, progressCb) +} + +func (b *LocalBackendManager) CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) { + return gallery.CheckBackendUpgrades(ctx, b.backendGalleries, b.systemState) +} + func (b *LocalBackendManager) InstallBackend(ctx context.Context, op *ManagementOp[gallery.GalleryBackend, any], progressCb ProgressCallback) error { if op.ExternalURI != "" { return InstallExternalBackend(ctx, b.backendGalleries, b.systemState, b.modelLoader, diff --git a/core/services/galleryop/operation.go b/core/services/galleryop/operation.go index 758a5bc5c..1c766eade 100644 --- a/core/services/galleryop/operation.go +++ b/core/services/galleryop/operation.go @@ -29,6 +29,9 @@ type ManagementOp[T any, E any] struct { ExternalURI string // The OCI image, URL, or path ExternalName string // Custom name for the backend ExternalAlias string // Custom alias for the backend + + // Upgrade is true if this is an upgrade operation (not a fresh install) + Upgrade bool } type OpStatus struct { diff --git a/core/services/nodes/managers_distributed.go b/core/services/nodes/managers_distributed.go index 0f7967e11..62cb32552 100644 --- a/core/services/nodes/managers_distributed.go +++ b/core/services/nodes/managers_distributed.go @@ -49,17 +49,19 @@ func (d *DistributedModelManager) InstallModel(ctx context.Context, op *galleryo // DistributedBackendManager wraps a local BackendManager and adds NATS fan-out // for backend deletion so worker nodes clean up stale files. type DistributedBackendManager struct { - local galleryop.BackendManager - adapter *RemoteUnloaderAdapter - registry *NodeRegistry + local galleryop.BackendManager + adapter *RemoteUnloaderAdapter + registry *NodeRegistry + backendGalleries []config.Gallery } // NewDistributedBackendManager creates a DistributedBackendManager. func NewDistributedBackendManager(appConfig *config.ApplicationConfig, ml *model.ModelLoader, adapter *RemoteUnloaderAdapter, registry *NodeRegistry) *DistributedBackendManager { return &DistributedBackendManager{ - local: galleryop.NewLocalBackendManager(appConfig, ml), - adapter: adapter, - registry: registry, + local: galleryop.NewLocalBackendManager(appConfig, ml), + adapter: adapter, + registry: registry, + backendGalleries: appConfig.BackendGalleries, } } @@ -172,3 +174,43 @@ func (d *DistributedBackendManager) InstallBackend(ctx context.Context, op *gall } return nil } + +// UpgradeBackend fans out a backend upgrade to all healthy worker nodes. +// TODO: Add dedicated NATS subject for upgrade (currently reuses install with force flag) +func (d *DistributedBackendManager) UpgradeBackend(ctx context.Context, name string, progressCb galleryop.ProgressCallback) error { + allNodes, err := d.registry.List(context.Background()) + if err != nil { + return err + } + + galleriesJSON, _ := json.Marshal(d.backendGalleries) + var errs []error + + for _, node := range allNodes { + if node.Status != StatusHealthy { + continue + } + // Reuse install endpoint which will re-download the backend (force mode) + reply, err := d.adapter.InstallBackend(node.ID, name, "", string(galleriesJSON)) + if err != nil { + if errors.Is(err, nats.ErrNoResponders) { + xlog.Warn("No NATS responders for node during upgrade, marking unhealthy", "node", node.Name, "nodeID", node.ID) + d.registry.MarkUnhealthy(context.Background(), node.ID) + continue + } + errs = append(errs, fmt.Errorf("node %s: %w", node.Name, err)) + continue + } + if !reply.Success { + errs = append(errs, fmt.Errorf("node %s: %s", node.Name, reply.Error)) + } + } + + return errors.Join(errs...) +} + +// CheckUpgrades checks for available backend upgrades. +// Gallery comparison is global (not per-node), so we delegate to the local manager. +func (d *DistributedBackendManager) CheckUpgrades(ctx context.Context) (map[string]gallery.UpgradeInfo, error) { + return d.local.CheckUpgrades(ctx) +} diff --git a/pkg/oci/image.go b/pkg/oci/image.go index 90d433a05..2d00c3479 100644 --- a/pkg/oci/image.go +++ b/pkg/oci/image.go @@ -188,6 +188,56 @@ func GetImage(targetImage, targetPlatform string, auth *registrytypes.AuthConfig return image, err } +// GetImageDigest returns the OCI image digest for the given image reference without downloading it. +// It uses remote.Head to fetch only the descriptor, which is much cheaper than pulling the full image. +func GetImageDigest(targetImage, targetPlatform string, auth *registrytypes.AuthConfig, t http.RoundTripper) (string, error) { + var platform *v1.Platform + var err error + + if targetPlatform != "" { + platform, err = v1.ParsePlatform(targetPlatform) + if err != nil { + return "", err + } + } else { + platform, err = v1.ParsePlatform(fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH)) + if err != nil { + return "", err + } + } + + ref, err := name.ParseReference(targetImage) + if err != nil { + return "", err + } + + if t == nil { + t = http.DefaultTransport + } + + tr := transport.NewRetry(t, + transport.WithRetryBackoff(defaultRetryBackoff), + transport.WithRetryPredicate(defaultRetryPredicate), + ) + + opts := []remote.Option{ + remote.WithTransport(tr), + remote.WithPlatform(*platform), + } + if auth != nil { + opts = append(opts, remote.WithAuth(staticAuth{auth})) + } else { + opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + } + + desc, err := remote.Head(ref, opts...) + if err != nil { + return "", err + } + + return desc.Digest.String(), nil +} + func GetOCIImageSize(targetImage, targetPlatform string, auth *registrytypes.AuthConfig, t http.RoundTripper) (int64, error) { var size int64 var img v1.Image diff --git a/pkg/oci/image_test.go b/pkg/oci/image_test.go index 8b26c2b87..447bc90f6 100644 --- a/pkg/oci/image_test.go +++ b/pkg/oci/image_test.go @@ -5,6 +5,7 @@ import ( "os" "runtime" + "github.com/mudler/LocalAI/pkg/oci" . "github.com/mudler/LocalAI/pkg/oci" // Update with your module path . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -36,3 +37,10 @@ var _ = Describe("OCI", func() { }) }) }) + +var _ = Describe("GetImageDigest", func() { + It("returns an error for an invalid image reference", func() { + _, err := oci.GetImageDigest("!!!invalid-ref!!!", "", nil, nil) + Expect(err).To(HaveOccurred()) + }) +})