From c604f3a1339e11e83e47b4c44795da7e5da361ff Mon Sep 17 00:00:00 2001 From: Ettore Di Giacinto Date: Tue, 3 Nov 2026 22:30:15 +0000 Subject: [PATCH] feat(addons): install, upgrade and downgrade from a staged zip file The manager takes a staged file as a release: sourceId local-file and the file ID as the version. It runs through the one lifecycle path, so generations, backup, recovery and the host access approval apply as for a library package. The approval digest is the sha256 of the zip. The same version is refused, and an older one needs a confirmed downgrade. Files wait in a private per-user staging area for an hour. An add-on from a file has no library, so no update check answers for it. Assisted-by: Claude:claude-sonnet-5-5 go gofmt --- core/application/application.go | 6 + core/services/addons/fileinstall.go | 129 +++++++ core/services/addons/fileservice.go | 161 +++++++++ core/services/addons/hostaccess.go | 5 + core/services/addons/manager.go | 55 ++- core/services/addons/manager_file_test.go | 210 ++++++++++++ core/services/addons/packages/file.go | 38 ++- core/services/addons/state.go | 3 + core/services/addons/updates.go | 5 +- core/services/addons/uploads/uploads.go | 332 +++++++++++++++++++ core/services/addons/uploads/uploads_test.go | 126 +++++++ 11 files changed, 1059 insertions(+), 11 deletions(-) create mode 100644 core/services/addons/fileinstall.go create mode 100644 core/services/addons/fileservice.go create mode 100644 core/services/addons/manager_file_test.go create mode 100644 core/services/addons/uploads/uploads.go create mode 100644 core/services/addons/uploads/uploads_test.go diff --git a/core/application/application.go b/core/application/application.go index ab9a2eb262..5677b84721 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -35,6 +35,7 @@ import ( addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime" "github.com/mudler/LocalAI/core/services/addons/updates" "github.com/mudler/LocalAI/core/services/addons/usersettings" + "github.com/mudler/LocalAI/core/services/addons/uploads" "github.com/mudler/LocalAI/core/services/advisorylock" "github.com/mudler/LocalAI/core/services/cloudproxy/mitm" "github.com/mudler/LocalAI/core/services/facerecognition" @@ -1051,6 +1052,11 @@ func (a *Application) startAddons() error { // The update check reads the libraries and the journal the owner // holds; it applies nothing, and its cache is bounded. m.EnableUpdates(updates.Config{TTL: c.UpdateCacheTTL, MinRefresh: c.UpdateRefreshInterval}) + // The staging area of zip files waits empty at start: a file that was + // uploaded before a restart has to be uploaded again. + files := &uploads.Store{Root: filepath.Join(c.Root, "uploads")} + files.Purge() + m.SetUploads(files) if err = m.Reconcile(a.applicationConfig.Context); err != nil { return err } diff --git a/core/services/addons/fileinstall.go b/core/services/addons/fileinstall.go new file mode 100644 index 0000000000..5c7b0862e4 --- /dev/null +++ b/core/services/addons/fileinstall.go @@ -0,0 +1,129 @@ +// SPDX-License-Identifier: MIT +package addons + +import ( + "context" + "errors" + + "github.com/mudler/LocalAI/core/services/addons/packages" + "github.com/mudler/LocalAI/core/services/addons/uploads" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/xlog" +) + +// ErrNoFileInstall: this owner has no staging area for local files. +var ErrNoFileInstall = errors.New("this add-on owner cannot install from a file") + +// Relations between a staged file and what is installed. +const ( + FileNew = "new" + FileUpgrade = "upgrade" + FileSame = "same" + FileDowngrade = "downgrade" +) + +type fileOwnerKey struct{} + +// WithFileOwner names the user whose staged files a call may read. The host +// access service sets it from the authenticated actor; a call without it +// cannot stage a file. +func WithFileOwner(ctx context.Context, userID string) context.Context { + return context.WithValue(ctx, fileOwnerKey{}, userID) +} + +func fileOwner(ctx context.Context) string { + v, _ := ctx.Value(fileOwnerKey{}).(string) + return v +} + +// SetUploads sets the staging area of local files. The first call wins. +// It is read under the manager lock and outside it, so it has its own guard. +func (m *Manager) SetUploads(s *uploads.Store) { + if s != nil { + m.uploads.CompareAndSwap(nil, s) + } +} + +func (m *Manager) uploadStore() *uploads.Store { return m.uploads.Load() } + +// stageUpload stages the package of a staged file of the user. It does not +// hold the manager lock: it reads and writes the disk only. +func (m *Manager) stageUpload(ctx context.Context, actor contract.Actor, fileID string) (packages.StagedPackage, error) { + store := m.uploadStore() + if store == nil || m.installer == nil { + return packages.StagedPackage{}, ErrNoFileInstall + } + p, entry, err := store.Open(fileID, actor.UserID) + if err != nil { + return packages.StagedPackage{}, err + } + return m.installer.StageFile(ctx, p, entry.Name, nil) +} + +func (m *Manager) dropUpload(actor contract.Actor, fileID string) { + if store := m.uploadStore(); store != nil { + if err := store.Remove(fileID, actor.UserID); err != nil && !errors.Is(err, uploads.ErrNotFound) { + xlog.Warn("Could not remove a staged add-on file", "error", err) + } + } +} + +// FileRelation says what installing the file would do to the add-on. +func FileRelation(installed bool, current, candidate string) string { + if !installed { + return FileNew + } + switch c := contract.CompareVersions(candidate, current); { + case c > 0: + return FileUpgrade + case c < 0: + return FileDowngrade + } + return FileSame +} + +// fileVersionRefusal applies the version rules of an install from a file: +// the same version is never reinstalled, and an older one only after the +// administrator confirmed the downgrade. It returns nil when the file may go on. +func (m *Manager) fileVersionRefusal(cmd LifecycleCommand, next Status, staged packages.PackageReceipt) *FailureCause { + cur, ok := currentReceipt(next) + if !next.Installed || !ok { + return nil + } + switch FileRelation(true, cur.Manifest.Version, staged.Manifest.Version) { + case FileSame: + return &FailureCause{Code: CauseVersionNotNewer} + case FileDowngrade: + if !cmd.Downgrade { + return &FailureCause{Code: CauseDowngradeRefused} + } + } + return nil +} + +// previewFile stages the file of the user named by the context, reads its +// verified manifest and discards the stage. It changes no state of the add-ons. +func (m *Manager) previewFile(ctx context.Context, fileID string) (HostAccessRelease, error) { + user := fileOwner(ctx) + if user == "" { + return HostAccessRelease{}, uploads.ErrNotFound + } + staged, e := m.stageUpload(ctx, contract.Actor{UserID: user}, fileID) + if e != nil { + return HostAccessRelease{}, e + } + rc := staged.Receipt() + if e = m.installer.Discard(staged); e != nil { + return HostAccessRelease{}, e + } + m.mu.Lock() + installed := m.journal.Addons[rc.Manifest.ID].Installed + m.mu.Unlock() + out := HostAccessRelease{AddonID: rc.Manifest.ID, Title: rc.Manifest.ID, Version: rc.Manifest.Version, Digest: rc.Digest, + Official: false, Installed: installed} + if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" { + out.Title = u.Title.Default() + } + out.Required, out.Optional = declared(rc) + return out, nil +} diff --git a/core/services/addons/fileservice.go b/core/services/addons/fileservice.go new file mode 100644 index 0000000000..f9e7625aa1 --- /dev/null +++ b/core/services/addons/fileservice.go @@ -0,0 +1,161 @@ +// SPDX-License-Identifier: MIT +package addons + +import ( + "context" + "errors" + + "github.com/mudler/LocalAI/core/services/addons/packages" + "github.com/mudler/LocalAI/core/services/addons/uploads" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/addons/zipformat" +) + +// FileView is what the administrator sees of a staged zip before the install +// dialog: what it is, how it relates to the installed add-on, and the digest +// the approval binds to. It carries no path and no credential. +type FileView struct { + FileID string `json:"fileId"` + Name string `json:"name"` + Size int64 `json:"size"` + // SHA256 is the digest of the whole file, in hex. The install approval + // binds to "sha256:" plus this value. + SHA256 string `json:"sha256"` + AddonID string `json:"addonId"` + Title string `json:"title"` + Version string `json:"version"` + Platform string `json:"platform"` + // API is the host API range the manifest declares. + API contract.VersionRange `json:"api"` + // Required and Optional are the host operations the manifest asks for. + Required []string `json:"required"` + Optional []string `json:"optional"` + Installed bool `json:"installed"` + InstalledVersion string `json:"installedVersion,omitempty"` + // Relation is new, upgrade, same or downgrade. + Relation string `json:"relation"` + // Source and Origin say that nothing signs the file. + Source string `json:"source"` + Origin string `json:"origin"` +} + +// FileBegin opens an upload. It returns the ID of the upload and the byte limit. +func (m *Manager) FileBegin(ctx context.Context, actor contract.Actor, name string) (string, int64, error) { + if e := m.authorize(ctx, actor); e != nil { + return "", 0, e + } + store := m.uploadStore() + if store == nil || m.installer == nil { + return "", 0, ErrNoFileInstall + } + max, _ := m.installer.Limits() + id, e := store.Begin(actor.UserID, packages.SafeFileName(name), max) + return id, max, e +} + +// FileChunk adds the next chunk to an upload. +func (m *Manager) FileChunk(ctx context.Context, actor contract.Actor, id string, offset int64, data []byte) error { + if e := m.authorize(ctx, actor); e != nil { + return e + } + store := m.uploadStore() + if store == nil { + return ErrNoFileInstall + } + return store.Append(id, actor.UserID, offset, data) +} + +// FileAbort drops an upload that did not finish. +func (m *Manager) FileAbort(ctx context.Context, actor contract.Actor, id string) { + if store := m.uploadStore(); store != nil && m.authorize(ctx, actor) == nil { + store.Abort(id, actor.UserID) + } +} + +// FileFinish closes the upload and verifies the file as the install will: the +// staged package is built and thrown away. A file that fails is removed. +func (m *Manager) FileFinish(ctx context.Context, actor contract.Actor, id string) (FileView, error) { + if e := m.authorize(ctx, actor); e != nil { + return FileView{}, e + } + store := m.uploadStore() + if store == nil { + return FileView{}, ErrNoFileInstall + } + entry, e := store.Finish(id, actor.UserID) + if e != nil { + return FileView{}, e + } + v, e := m.describeStaged(ctx, actor, entry) + if e != nil { + _ = store.Remove(entry.ID, actor.UserID) + return FileView{}, e + } + return v, nil +} + +// FileDescribe reads a staged file of the user again. +func (m *Manager) FileDescribe(ctx context.Context, actor contract.Actor, id string) (FileView, error) { + if e := m.authorize(ctx, actor); e != nil { + return FileView{}, e + } + store := m.uploadStore() + if store == nil { + return FileView{}, ErrNoFileInstall + } + _, entry, e := store.Open(id, actor.UserID) + if e != nil { + return FileView{}, e + } + return m.describeStaged(ctx, actor, entry) +} + +// FileDrop removes a staged file of the user. +func (m *Manager) FileDrop(ctx context.Context, actor contract.Actor, id string) error { + if e := m.authorize(ctx, actor); e != nil { + return e + } + store := m.uploadStore() + if store == nil { + return ErrNoFileInstall + } + return store.Remove(id, actor.UserID) +} + +func (m *Manager) describeStaged(ctx context.Context, actor contract.Actor, entry uploads.Entry) (FileView, error) { + staged, e := m.stageUpload(ctx, actor, entry.ID) + if e != nil { + return FileView{}, e + } + rc := staged.Receipt() + if e = m.installer.Discard(staged); e != nil { + return FileView{}, e + } + m.mu.Lock() + st := m.journal.Addons[rc.Manifest.ID] + m.mu.Unlock() + v := FileView{FileID: entry.ID, Name: entry.Name, Size: entry.Size, SHA256: entry.SHA256, AddonID: rc.Manifest.ID, Title: rc.Manifest.ID, + Version: rc.Manifest.Version, Platform: rc.Manifest.OS + "/" + rc.Manifest.Arch, API: rc.Manifest.API, + Source: zipformat.SourceID, Origin: zipformat.OriginLabel, Relation: FileNew} + if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" { + v.Title = u.Title.Default() + } + v.Required, v.Optional = declared(rc) + if v.Required == nil { + v.Required = []string{} + } + if v.Optional == nil { + v.Optional = []string{} + } + if cur, ok := currentReceipt(st); ok && st.Installed { + v.Installed, v.InstalledVersion = true, cur.Manifest.Version + v.Relation = FileRelation(true, cur.Manifest.Version, rc.Manifest.Version) + } + return v, nil +} + +// IsFileRefusal reports whether e is a refusal of the content of a file. +func IsFileRefusal(e error) (*packages.FileError, bool) { + var fe *packages.FileError + return fe, errors.As(e, &fe) +} diff --git a/core/services/addons/hostaccess.go b/core/services/addons/hostaccess.go index d987e97fad..48b9666cd7 100644 --- a/core/services/addons/hostaccess.go +++ b/core/services/addons/hostaccess.go @@ -9,6 +9,7 @@ import ( "github.com/mudler/LocalAI/core/services/addons/catalog" "github.com/mudler/LocalAI/core/services/addons/packages" addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime" + "github.com/mudler/LocalAI/pkg/addons/zipformat" ) // OfficialReceipt reports whether a package came from the official add-on @@ -137,6 +138,10 @@ func (m *Manager) PreviewRelease(ctx context.Context, sourceID, addonID, version } installed := m.journal.Addons[addonID].Installed m.mu.Unlock() + if sourceID == zipformat.SourceID { + // A local file: the version field names the staged file. + return m.previewFile(ctx, version) + } r, e := m.resolveRelease(ctx, LifecycleCommand{SourceID: sourceID, AddonID: addonID, Version: version}) if e != nil { return HostAccessRelease{}, e diff --git a/core/services/addons/manager.go b/core/services/addons/manager.go index 4517784b1c..67d93bb68a 100644 --- a/core/services/addons/manager.go +++ b/core/services/addons/manager.go @@ -27,7 +27,9 @@ import ( "github.com/mudler/LocalAI/core/services/addons/recovery" addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime" "github.com/mudler/LocalAI/core/services/addons/updates" + "github.com/mudler/LocalAI/core/services/addons/uploads" "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/addons/zipformat" ) var ( @@ -75,6 +77,14 @@ const ( CausePackageFetch = "package_fetch_failed" // CauseInstallStorage: the verified package could not be written to disk. CauseInstallStorage = "install_storage_failed" + // CauseFileMissing: the staged file of a local file install is gone, has + // expired or belongs to another user. + CauseFileMissing = "staged_file_missing" + // CauseVersionNotNewer: the file holds the version that is installed. + CauseVersionNotNewer = "version_not_newer" + // CauseDowngradeRefused: the file holds an older version and the request + // did not confirm the downgrade. + CauseDowngradeRefused = "downgrade_refused" ) // FailureCause is the typed cause of a failed operation. Its text is fixed and @@ -177,6 +187,9 @@ type Manager struct { // identities maps an installed add-on to its trusted identity; see // TrustedIdentity. identities map[string]string + // uploads holds the zip files waiting for an install. Nil means the owner + // does not install from a file. + uploads atomic.Pointer[uploads.Store] } // SetAuthority sets the Authority that admission and the apps list ask. The @@ -614,23 +627,47 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl return op.Status, e } m.mu.Unlock() - if release != nil { - r = *release - } else { - r, resolveErr = m.resolveRelease(ctx, cmd) - } + fromFile := cmd.SourceID == zipformat.SourceID var staged packages.StagedPackage var stageErr error - if resolveErr == nil { - staged, stageErr = m.installer.Stage(ctx, r, nil) + if fromFile { + // A local file has no library entry: the staged file is the + // release, and the same Stage checks apply to its contents. + staged, stageErr = m.stageUpload(ctx, actor, cmd.Version) + } else { + if release != nil { + r = *release + } else { + r, resolveErr = m.resolveRelease(ctx, cmd) + } + if resolveErr == nil { + staged, stageErr = m.installer.Stage(ctx, r, nil) + } } m.mu.Lock() if resolveErr != nil { return fail(resolveErr, &FailureCause{Code: resolveCauseCode(resolveErr), detail: resolveErr}) } if stageErr != nil { + if errors.Is(stageErr, uploads.ErrNotFound) { + return fail(stageErr, &FailureCause{Code: CauseFileMissing, detail: stageErr}) + } return fail(stageErr, &FailureCause{Code: CausePackageFetch, detail: stageErr}) } + if fromFile { + if cause := m.fileVersionRefusal(cmd, next, staged.Receipt()); cause != nil { + if discardErr := m.installer.Discard(staged); discardErr != nil { + xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr) + } + return fail(cause) + } + if staged.Receipt().Manifest.ID != cmd.AddonID { + if discardErr := m.installer.Discard(staged); discardErr != nil { + xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr) + } + return fail(packages.ErrPackage) + } + } if m.uncertain || m.runtimeRecoveryRequired { return op.Status, ErrRecoveryRequired } @@ -764,6 +801,10 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl case "install", "upgrade", "uninstall": m.syncSkills(ctx, cmd.AddonID) } + if cmd.SourceID == zipformat.SourceID && (cmd.Action == "install" || cmd.Action == "upgrade") { + // The package is installed; the staged file has done its work. + m.dropUpload(actor, cmd.Version) + } return result, m.syncFor(ctx, cmd.AddonID) } diff --git a/core/services/addons/manager_file_test.go b/core/services/addons/manager_file_test.go new file mode 100644 index 0000000000..5f8a6dd579 --- /dev/null +++ b/core/services/addons/manager_file_test.go @@ -0,0 +1,210 @@ +//go:build linux + +// SPDX-License-Identifier: MIT +package addons + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strings" + + "github.com/mudler/LocalAI/core/services/addons/packages" + "github.com/mudler/LocalAI/core/services/addons/uploads" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/LocalAI/pkg/addons/zipformat" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("install from a local zip file", func() { + var r *updateRig + var base map[string]any + BeforeEach(func() { + r = newUpdateRig(false) + r.m.SetUploads(&uploads.Store{Root: filepath.Join(r.cfg.Root, "uploads")}) + raw, e := os.ReadFile("../../../tests/addons/fixtures/packages/manifest.json") + Expect(e).NotTo(HaveOccurred()) + Expect(json.Unmarshal(raw, &base)).To(Succeed()) + }) + // zipFor builds the zip of a sample add-on of the given version that asks + // for the given required host operations. + zipFor := func(version, osName string, required ...string) []byte { + m := map[string]any{} + for k, v := range base { + m[k] = v + } + m["version"], m["os"] = version, osName + if len(required) > 0 { + m["capabilities"] = map[string]any{"service": true, "hostOperations": map[string]any{"required": required}} + } + mb, e := json.Marshal(m) + Expect(e).NotTo(HaveOccurred()) + var b bytes.Buffer + Expect(zipformat.Write(&b, "sample", version, osName, "amd64", "bin/sample", []zipformat.Entry{ + {Name: "manifest.json", Data: mb}, {Name: "bin/sample", Data: []byte("executable " + version), Exec: true}, + })).To(Succeed()) + return b.Bytes() + } + upload := func(user string, data []byte) (FileView, error) { + a := contract.Actor{UserID: user, CredentialID: "key", Mode: "named-user"} + id, _, e := r.m.FileBegin(ctx, a, "sample.zip") + Expect(e).NotTo(HaveOccurred()) + for off := 0; off < len(data); off += 1000 { + end := min(off+1000, len(data)) + Expect(r.m.FileChunk(ctx, a, id, int64(off), data[off:end])).To(Succeed()) + } + return r.m.FileFinish(ctx, a, id) + } + fileCmd := func(op, action string, v FileView) LifecycleCommand { + return LifecycleCommand{OperationID: op, AddonID: "sample", Action: action, SourceID: zipformat.SourceID, Version: v.FileID} + } + + It("installs through the manager and records an unsigned local file", func() { + v, e := upload("admin", zipFor("2.0.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + Expect(v.Relation).To(Equal(FileNew)) + Expect(v.Origin).To(Equal("local file, unsigned")) + Expect(v.Version).To(Equal("2.0.0")) + Expect(v.API.Min).To(Equal(contract.HostAPIVersion)) + _, e = r.m.Submit(ctx, actor, fileCmd("f1", "install", v)) + Expect(e).NotTo(HaveOccurred()) + st, _ := r.m.Status(ctx, "sample") + Expect(st.Installed).To(BeTrue()) + rc := st.Generations[0].Receipt + Expect(rc.SourceID).To(Equal("local-file")) + Expect(rc.Origin).To(Equal("local file, unsigned")) + Expect(rc.Verification).To(Equal("sha256")) + Expect(rc.Digest).To(Equal("sha256:" + v.SHA256)) + // The staged file is gone once it is installed. + _, e = r.m.FileDescribe(ctx, actor, v.FileID) + Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue()) + // A package from a file has no library to ask for updates. + list, e := r.m.Installed(ctx, actor) + Expect(e).NotTo(HaveOccurred()) + Expect(list).To(HaveLen(1)) + Expect(list[0].Update).To(BeNil()) + Expect(list[0].SourceID).To(Equal("local-file")) + }) + + It("keeps a file for the user that uploaded it", func() { + v, e := upload("admin", zipFor("2.0.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + other := contract.Actor{UserID: "admin-two", CredentialID: "key", Mode: "named-user"} + _, e = r.m.stageUpload(ctx, other, v.FileID) + Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue()) + // A command of another user is refused by the administrator check + // before it can reach the file. + _, e = r.m.Submit(ctx, other, fileCmd("f1", "install", v)) + Expect(e).To(MatchError(ErrForbidden)) + st, _ := r.m.Status(ctx, "sample") + Expect(st.Installed).To(BeFalse()) + }) + + It("refuses a file for another platform when it is uploaded, and leaves nothing staged", func() { + _, e := upload("admin", zipFor("2.0.0", "darwin")) + var fe *packages.FileError + Expect(errors.As(e, &fe)).To(BeTrue()) + Expect(fe.Code).To(Equal(packages.FileWrongPlatform)) + ents, _ := os.ReadDir(filepath.Join(r.cfg.Root, "uploads")) + Expect(ents).To(BeEmpty()) + }) + + It("refuses an install when the administrator is not one", func() { + _, _, e := r.m.FileBegin(ctx, contract.Actor{UserID: "user", Mode: "named-user"}, "x.zip") + Expect(e).To(MatchError(ErrForbidden)) + }) + + Describe("versions", func() { + install := func(version string) FileView { + v, e := upload("admin", zipFor(version, "linux")) + Expect(e).NotTo(HaveOccurred()) + _, e = r.m.Submit(ctx, actor, fileCmd("i-"+version, "install", v)) + Expect(e).NotTo(HaveOccurred()) + return v + } + BeforeEach(func() { install("2.0.0") }) + + It("upgrades to a higher version through the same generation machinery", func() { + v, e := upload("admin", zipFor("2.1.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + Expect(v.Relation).To(Equal(FileUpgrade)) + Expect(v.InstalledVersion).To(Equal("2.0.0")) + _, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v)) + Expect(e).NotTo(HaveOccurred()) + st, _ := r.m.Status(ctx, "sample") + Expect(st.Generation).To(Equal(uint64(2))) + Expect(st.Generations[1].Receipt.Manifest.Version).To(Equal("2.1.0")) + }) + + It("refuses the same version", func() { + v, e := upload("admin", zipFor("2.0.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + Expect(v.Relation).To(Equal(FileSame)) + _, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v)) + Expect(e).To(MatchError(ErrOperationFailed)) + Expect(e.Error()).To(ContainSubstring(CauseVersionNotNewer)) + st, _ := r.m.Status(ctx, "sample") + Expect(st.Generation).To(Equal(uint64(1))) + }) + + It("refuses a lower version unless the downgrade is confirmed", func() { + v, e := upload("admin", zipFor("1.5.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + Expect(v.Relation).To(Equal(FileDowngrade)) + _, e = r.m.Submit(ctx, actor, fileCmd("d1", "upgrade", v)) + Expect(e).To(MatchError(ErrOperationFailed)) + Expect(e.Error()).To(ContainSubstring(CauseDowngradeRefused)) + st, _ := r.m.Status(ctx, "sample") + Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("2.0.0")) + // The file stays staged for a retry with the confirmation. + cmd := fileCmd("d2", "upgrade", v) + cmd.Downgrade = true + _, e = r.m.Submit(ctx, actor, cmd) + Expect(e).NotTo(HaveOccurred()) + st, _ = r.m.Status(ctx, "sample") + Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("1.5.0")) + }) + }) + + Describe("host access approval", func() { + It("grants only what the administrator approved for the digest of the file", func() { + g := newGrantSim(r.m) + v, e := upload("admin", zipFor("2.0.0", "linux", "workspace.materialize")) + Expect(e).NotTo(HaveOccurred()) + Expect(v.Required).To(Equal([]string{"workspace.materialize"})) + _, e = r.m.Submit(ctx, actor, fileCmd("f0", "install", v)) + var refusal *HostRequirementsRefusal + Expect(errors.As(e, &refusal)).To(BeTrue(), "an unapproved required operation is refused") + // Previews give the digest the dialog shows. + rel, e := r.m.PreviewRelease(WithFileOwner(ctx, "admin"), zipformat.SourceID, "sample", v.FileID) + Expect(e).NotTo(HaveOccurred()) + Expect(rel.Digest).To(Equal("sha256:" + v.SHA256)) + Expect(rel.Official).To(BeFalse()) + Expect(rel.Required).To(Equal([]string{"workspace.materialize"})) + cmd := fileCmd("f1", "install", v) + cmd.Approval = &HostApproval{Digest: "sha256:" + strings.Repeat("0", 64)} + _, e = r.m.Submit(ctx, actor, cmd) + var stale *ApprovalStaleError + Expect(errors.As(e, &stale)).To(BeTrue(), "another digest is stale") + cmd = fileCmd("f2", "install", v) + cmd.Approval = &HostApproval{Digest: rel.Digest, Ops: []string{"workspace.materialize", "egress.check"}, Revision: 3} + _, e = r.m.Submit(ctx, actor, cmd) + Expect(e).NotTo(HaveOccurred()) + Expect(g.stagedOps("f2")).To(Equal([]string{"workspace.materialize"})) + Expect(r.m.OperationCompleted("f2")).To(BeTrue()) + }) + + It("previews only the files of the calling user", func() { + v, e := upload("admin", zipFor("2.0.0", "linux")) + Expect(e).NotTo(HaveOccurred()) + _, e = r.m.PreviewRelease(WithFileOwner(ctx, "someone"), zipformat.SourceID, "sample", v.FileID) + Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue()) + _, e = r.m.PreviewRelease(context.Background(), zipformat.SourceID, "sample", v.FileID) + Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue()) + }) + }) +}) diff --git a/core/services/addons/packages/file.go b/core/services/addons/packages/file.go index 2288051871..33494a48fa 100644 --- a/core/services/addons/packages/file.go +++ b/core/services/addons/packages/file.go @@ -15,6 +15,7 @@ import ( "regexp" "strings" + "github.com/mudler/LocalAI/pkg/addons/contract" "github.com/mudler/LocalAI/pkg/addons/zipformat" ) @@ -55,6 +56,10 @@ const ( var fileNameRE = regexp.MustCompile(`[^A-Za-z0-9._-]`) +// Limits returns the byte and file caps of a package, the ones that apply to +// a library package and to a local file alike. +func (i *Installer) Limits() (bytes int64, files int) { return i.byteLimit(), i.fileLimit() } + // FileSummary is what a verified zip says about itself. It is computed without // writing the package to disk. type FileSummary struct { @@ -63,6 +68,9 @@ type FileSummary struct { Size int64 `json:"size"` Files int `json:"files"` zipformat.Manifest + // Package is manifest.json of the add-on, parsed. InspectFile fills it; it + // is not checked against a host there. + Package contract.Manifest `json:"package"` } // zipSink receives each file of the package after the path and type checks. @@ -223,10 +231,34 @@ func (c *countingReader) Read(b []byte) (int, error) { // installer; zero takes the defaults. func InspectFile(ctx context.Context, p string, maxBytes int64, maxFiles int) (FileSummary, error) { i := &Installer{MaxBytes: maxBytes, MaxFiles: maxFiles} - return readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(_ string, _ bool, r io.Reader) error { - _, e := io.Copy(io.Discard, r) - return e + var pkg contract.Manifest + sum, e := readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(n string, _ bool, r io.Reader) error { + if n != "manifest.json" { + _, e := io.Copy(io.Discard, r) + return e + } + b, e := io.ReadAll(io.LimitReader(r, contract.DefaultLimits().ManifestBytes+1)) + if e != nil { + return e + } + if int64(len(b)) > contract.DefaultLimits().ManifestBytes { + return fileErr(FileBadManifest, "manifest.json is too large") + } + m, e := contract.ParseManifest(b) + if e != nil { + return fileErr(FileBadManifest, "manifest.json is not a valid add-on manifest") + } + pkg = m + return nil }) + if e != nil { + return sum, e + } + if pkg.ID == "" || pkg.ID != sum.ID || pkg.Version != sum.Version || pkg.OS != sum.OS || pkg.Arch != sum.Arch { + return sum, fileErr(FileBadManifest, "The manifest of the add-on does not match the digest manifest of the zip") + } + sum.Package = pkg + return sum, nil } // SafeFileName reduces the name a client gave a file to characters that are diff --git a/core/services/addons/state.go b/core/services/addons/state.go index a9be360801..215204a877 100644 --- a/core/services/addons/state.go +++ b/core/services/addons/state.go @@ -60,6 +60,9 @@ type LifecycleCommand struct { // registers. It carries no authority by itself: the caller must be a human // surface, which the gateway checks before it sets this field. Approval *HostApproval `json:",omitempty"` + // Downgrade confirms that a local file may install an older version than + // the one installed. Without it such a file is refused. + Downgrade bool `json:",omitempty"` } // HostApproval binds an approval to one staged package and one revision of the diff --git a/core/services/addons/updates.go b/core/services/addons/updates.go index 1d73374146..d2eb7d5b0b 100644 --- a/core/services/addons/updates.go +++ b/core/services/addons/updates.go @@ -4,6 +4,7 @@ package addons import ( "context" "errors" + "github.com/mudler/LocalAI/pkg/addons/zipformat" "time" "github.com/mudler/LocalAI/core/services/addons/apps" @@ -55,7 +56,9 @@ func (r *updateReader) Installation(ctx context.Context, id string) (updates.Ins return updates.Installation{}, false, nil } receipt, ok := currentReceipt(st) - if !ok { + if !ok || receipt.SourceID == zipformat.SourceID { + // A package from a local file has no library to ask, so no check can + // answer for it. The page reads that as unknown, never as current. return updates.Installation{}, false, nil } return updates.Installation{AddonID: id, Generation: st.Generation, Receipt: receipt}, true, nil diff --git a/core/services/addons/uploads/uploads.go b/core/services/addons/uploads/uploads.go new file mode 100644 index 0000000000..59415292ab --- /dev/null +++ b/core/services/addons/uploads/uploads.go @@ -0,0 +1,332 @@ +// SPDX-License-Identifier: MIT + +// Package uploads is the owner's staging area for add-on zip files. A file +// waits here between the upload and the install the administrator confirms. +// It is private to the owner, bound to the user that uploaded it, and +// removed after a short time, after the install and at start. +package uploads + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "hash" + "io" + "os" + "path/filepath" + "regexp" + "sync" + "time" +) + +var ( + // ErrNotFound: no such file, or it belongs to another user, or it expired. + ErrNotFound = errors.New("staged file not found") + // ErrTooLarge: the upload went over the byte limit. + ErrTooLarge = errors.New("file is over the size limit") + // ErrBusy: too many files are waiting. + ErrBusy = errors.New("too many staged files") + // ErrOffset: a chunk did not follow the bytes received so far. + ErrOffset = errors.New("chunk out of order") +) + +// Defaults of a store. +const ( + DefaultTTL = time.Hour + DefaultMaxFiles = 16 +) + +var idRE = regexp.MustCompile(`^[0-9a-f]{32}$`) + +// ValidID reports whether s has the shape of a staged file ID. +func ValidID(s string) bool { return idRE.MatchString(s) } + +// Entry is a staged file. +type Entry struct { + ID string `json:"id"` + Owner string `json:"owner"` + Name string `json:"name"` + Size int64 `json:"size"` + SHA256 string `json:"sha256"` + Created time.Time `json:"created"` +} + +// Store keeps staged files under Root. +type Store struct { + Root string + TTL time.Duration + MaxFiles int + // Now is replaced by tests. + Now func() time.Time + + mu sync.Mutex + open map[string]*upload +} + +type upload struct { + owner, name string + f *os.File + h hash.Hash + n int64 + max int64 + started time.Time +} + +func (s *Store) now() time.Time { + if s.Now != nil { + return s.Now() + } + return time.Now() +} + +func (s *Store) ttl() time.Duration { + if s.TTL > 0 { + return s.TTL + } + return DefaultTTL +} + +func (s *Store) maxFiles() int { + if s.MaxFiles > 0 { + return s.MaxFiles + } + return DefaultMaxFiles +} + +func (s *Store) dir() (string, error) { + if s.Root == "" || !filepath.IsAbs(s.Root) { + return "", errors.New("uploads: root must be an absolute path") + } + if err := os.MkdirAll(s.Root, 0o700); err != nil { + return "", err + } + info, err := os.Lstat(s.Root) + if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 { + return "", errors.New("uploads: root must be a private directory") + } + return s.Root, nil +} + +func newID() (string, error) { + var b [16]byte + if _, err := rand.Read(b[:]); err != nil { + return "", err + } + return hex.EncodeToString(b[:]), nil +} + +func (s *Store) zipPath(id string) string { return filepath.Join(s.Root, id+".zip") } +func (s *Store) metaPath(id string) string { return filepath.Join(s.Root, id+".json") } + +// Begin opens a new upload for owner. max is the byte limit of the file. +func (s *Store) Begin(owner, name string, max int64) (string, error) { + if owner == "" || max <= 0 { + return "", ErrNotFound + } + if _, err := s.dir(); err != nil { + return "", err + } + s.Sweep() + s.mu.Lock() + defer s.mu.Unlock() + if s.open == nil { + s.open = map[string]*upload{} + } + staged, _ := filepath.Glob(filepath.Join(s.Root, "*.zip")) + if len(staged)+len(s.open) >= s.maxFiles() { + return "", ErrBusy + } + id, err := newID() + if err != nil { + return "", err + } + f, err := os.OpenFile(s.zipPath(id), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) // #nosec G304 -- the name is a random ID under the private staging root + if err != nil { + return "", err + } + s.open[id] = &upload{owner: owner, name: name, f: f, h: sha256.New(), max: max, started: s.now()} + return id, nil +} + +// Append adds the next chunk. offset is the number of bytes received before it. +func (s *Store) Append(id, owner string, offset int64, p []byte) error { + s.mu.Lock() + defer s.mu.Unlock() + u, ok := s.open[id] + if !ok || u.owner != owner { + return ErrNotFound + } + if offset != u.n { + return ErrOffset + } + if int64(len(p)) > u.max-u.n { + s.abortLocked(id) + return ErrTooLarge + } + if _, err := u.f.Write(p); err != nil { + s.abortLocked(id) + return err + } + _, _ = u.h.Write(p) + u.n += int64(len(p)) + return nil +} + +// Finish closes the upload and makes it a staged file. +func (s *Store) Finish(id, owner string) (Entry, error) { + s.mu.Lock() + defer s.mu.Unlock() + u, ok := s.open[id] + if !ok || u.owner != owner { + return Entry{}, ErrNotFound + } + delete(s.open, id) + e := Entry{ID: id, Owner: owner, Name: u.name, Size: u.n, SHA256: hex.EncodeToString(u.h.Sum(nil)), Created: s.now().UTC()} + err := u.f.Sync() + if ce := u.f.Close(); err == nil { + err = ce + } + if err == nil { + var b []byte + if b, err = json.Marshal(e); err == nil { + err = os.WriteFile(s.metaPath(id), b, 0o600) + } + } + if err != nil { + _ = os.Remove(s.zipPath(id)) + return Entry{}, err + } + return e, nil +} + +// Abort drops an upload that is not finished. It is safe to call twice. +func (s *Store) Abort(id, owner string) { + s.mu.Lock() + defer s.mu.Unlock() + if u, ok := s.open[id]; ok && u.owner == owner { + s.abortLocked(id) + } +} + +func (s *Store) abortLocked(id string) { + if u, ok := s.open[id]; ok { + _ = u.f.Close() + delete(s.open, id) + } + _ = os.Remove(s.zipPath(id)) +} + +// Put stores the whole of r as a staged file of owner, up to max bytes. +func (s *Store) Put(owner, name string, max int64, r io.Reader) (Entry, error) { + id, err := s.Begin(owner, name, max) + if err != nil { + return Entry{}, err + } + buf := make([]byte, 256<<10) + var off int64 + for { + n, rerr := r.Read(buf) + if n > 0 { + if err = s.Append(id, owner, off, buf[:n]); err != nil { + return Entry{}, err + } + off += int64(n) + } + if rerr == io.EOF { + break + } + if rerr != nil { + s.Abort(id, owner) + return Entry{}, rerr + } + } + return s.Finish(id, owner) +} + +// Open returns the path and the entry of a staged file of owner. +func (s *Store) Open(id, owner string) (string, Entry, error) { + if !ValidID(id) || owner == "" { + return "", Entry{}, ErrNotFound + } + e, err := s.read(id) + if err != nil || e.Owner != owner || s.expired(e) { + return "", Entry{}, ErrNotFound + } + return s.zipPath(id), e, nil +} + +func (s *Store) read(id string) (Entry, error) { + var e Entry + b, err := os.ReadFile(s.metaPath(id)) // #nosec G304 -- the name is a validated random ID under the private staging root + if err != nil { + return e, ErrNotFound + } + if json.Unmarshal(b, &e) != nil || e.ID != id { + return Entry{}, ErrNotFound + } + return e, nil +} + +func (s *Store) expired(e Entry) bool { return s.now().Sub(e.Created) > s.ttl() } + +// Remove deletes a staged file of owner. +func (s *Store) Remove(id, owner string) error { + if _, _, err := s.Open(id, owner); err != nil { + return err + } + return s.removeFiles(id) +} + +func (s *Store) removeFiles(id string) error { + err := os.Remove(s.zipPath(id)) + _ = os.Remove(s.metaPath(id)) + if err != nil && !os.IsNotExist(err) { + return fmt.Errorf("remove staged file: %w", err) + } + return nil +} + +// Sweep removes the files that are older than the time to live and the +// uploads that never finished. With purge set it removes everything that is +// not an open upload, as the owner does at start. +func (s *Store) Sweep() { s.sweep(false) } + +// Purge removes every staged file. Call it once at start. +func (s *Store) Purge() { s.sweep(true) } + +func (s *Store) sweep(all bool) { + if _, err := s.dir(); err != nil { + return + } + // The lock covers the whole sweep, so a Finish in progress is never seen + // half done. + s.mu.Lock() + defer s.mu.Unlock() + for id, u := range s.open { + if all || s.now().Sub(u.started) > s.ttl() { + s.abortLocked(id) + } + } + open := map[string]bool{} + for id := range s.open { + open[id] = true + } + ents, err := os.ReadDir(s.Root) + if err != nil { + return + } + for _, de := range ents { + name := de.Name() + id := name[:len(name)-len(filepath.Ext(name))] + if !ValidID(id) || open[id] { + continue + } + e, rerr := s.read(id) + if all || rerr != nil || s.expired(e) { + _ = s.removeFiles(id) + } + } +} diff --git a/core/services/addons/uploads/uploads_test.go b/core/services/addons/uploads/uploads_test.go new file mode 100644 index 0000000000..3d57c69ee8 --- /dev/null +++ b/core/services/addons/uploads/uploads_test.go @@ -0,0 +1,126 @@ +// SPDX-License-Identifier: MIT +package uploads + +import ( + "bytes" + "errors" + "os" + "path/filepath" + "strings" + "testing" + "time" +) + +func newStore(t *testing.T) (*Store, *time.Time) { + t.Helper() + now := time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC) + root := filepath.Join(t.TempDir(), "uploads") + return &Store{Root: root, Now: func() time.Time { return now }, MaxFiles: 2}, &now +} + +func TestPutOpenAndOwner(t *testing.T) { + s, _ := newStore(t) + e, err := s.Put("alice", "x.zip", 1<<20, strings.NewReader("hello")) + if err != nil { + t.Fatal(err) + } + if e.Size != 5 || len(e.SHA256) != 64 || !ValidID(e.ID) { + t.Fatalf("entry %+v", e) + } + p, got, err := s.Open(e.ID, "alice") + if err != nil || got.ID != e.ID { + t.Fatalf("open: %v", err) + } + if b, _ := os.ReadFile(p); string(b) != "hello" { + t.Fatalf("content %q", b) + } + if _, _, err = s.Open(e.ID, "bob"); !errors.Is(err, ErrNotFound) { + t.Fatalf("another user must not open it: %v", err) + } + if _, _, err = s.Open("../etc/passwd", "alice"); !errors.Is(err, ErrNotFound) { + t.Fatalf("a path is not an ID: %v", err) + } + info, _ := os.Stat(p) + if info.Mode().Perm() != 0o600 { + t.Fatalf("mode %v", info.Mode()) + } + if err = s.Remove(e.ID, "bob"); !errors.Is(err, ErrNotFound) { + t.Fatal("another user removed it") + } + if err = s.Remove(e.ID, "alice"); err != nil { + t.Fatal(err) + } + if _, _, err = s.Open(e.ID, "alice"); !errors.Is(err, ErrNotFound) { + t.Fatal("still there") + } +} + +func TestSizeLimitStopsTheUpload(t *testing.T) { + s, _ := newStore(t) + _, err := s.Put("alice", "x.zip", 10, bytes.NewReader(make([]byte, 11))) + if !errors.Is(err, ErrTooLarge) { + t.Fatalf("got %v", err) + } + ents, _ := os.ReadDir(s.Root) + if len(ents) != 0 { + t.Fatalf("left files: %v", ents) + } +} + +func TestChunksMustFollowEachOther(t *testing.T) { + s, _ := newStore(t) + id, err := s.Begin("alice", "x.zip", 100) + if err != nil { + t.Fatal(err) + } + if err = s.Append(id, "alice", 0, []byte("ab")); err != nil { + t.Fatal(err) + } + if err = s.Append(id, "alice", 5, []byte("cd")); !errors.Is(err, ErrOffset) { + t.Fatalf("gap: %v", err) + } + if err = s.Append(id, "bob", 2, []byte("cd")); !errors.Is(err, ErrNotFound) { + t.Fatalf("other user: %v", err) + } + if _, err = s.Finish(id, "bob"); !errors.Is(err, ErrNotFound) { + t.Fatalf("other user finish: %v", err) + } + e, err := s.Finish(id, "alice") + if err != nil || e.Size != 2 { + t.Fatalf("finish: %v %+v", err, e) + } +} + +func TestExpiryBusyAndPurge(t *testing.T) { + s, now := newStore(t) + a, _ := s.Put("alice", "a.zip", 100, strings.NewReader("a")) + if _, err := s.Put("alice", "b.zip", 100, strings.NewReader("b")); err != nil { + t.Fatal(err) + } + if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); !errors.Is(err, ErrBusy) { + t.Fatalf("limit: %v", err) + } + *now = now.Add(2 * time.Hour) + if _, _, err := s.Open(a.ID, "alice"); !errors.Is(err, ErrNotFound) { + t.Fatal("an expired file opens") + } + if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); err != nil { + t.Fatalf("sweep did not free room: %v", err) + } + s.Purge() + ents, _ := os.ReadDir(s.Root) + if len(ents) != 0 { + t.Fatalf("purge left %v", ents) + } +} + +func TestRootMustBePrivate(t *testing.T) { + root := filepath.Join(t.TempDir(), "open") + if err := os.Mkdir(root, 0o755); err != nil { + t.Fatal(err) + } + s := &Store{Root: root} + if _, err := s.Put("alice", "x.zip", 10, strings.NewReader("x")); err == nil { + t.Fatal("a world-readable root was accepted") + } +}