feat(addons): install, upgrade and downgrade from a staged zip file

The manager takes a staged file as a release: sourceId local-file and
the file ID as the version. It runs through the one lifecycle path, so
generations, backup, recovery and the host access approval apply as for
a library package. The approval digest is the sha256 of the zip.

The same version is refused, and an older one needs a confirmed
downgrade. Files wait in a private per-user staging area for an hour.
An add-on from a file has no library, so no update check answers for it.

Assisted-by: Claude:claude-sonnet-5-5 go gofmt
This commit is contained in:
Ettore Di Giacinto committed 2026-11-03 22:52:50 +00:00
1 parent e6ff1ab0f9
commit c604f3a133
11 files changed
+1059 -11

No files matched your search

+6
View File
@@ -35,6 +35,7 @@ import (
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
"github.com/mudler/LocalAI/core/services/addons/updates"
"github.com/mudler/LocalAI/core/services/addons/usersettings"
"github.com/mudler/LocalAI/core/services/addons/uploads"
"github.com/mudler/LocalAI/core/services/advisorylock"
"github.com/mudler/LocalAI/core/services/cloudproxy/mitm"
"github.com/mudler/LocalAI/core/services/facerecognition"
@@ -1051,6 +1052,11 @@ func (a *Application) startAddons() error {
// The update check reads the libraries and the journal the owner
// holds; it applies nothing, and its cache is bounded.
m.EnableUpdates(updates.Config{TTL: c.UpdateCacheTTL, MinRefresh: c.UpdateRefreshInterval})
// The staging area of zip files waits empty at start: a file that was
// uploaded before a restart has to be uploaded again.
files := &uploads.Store{Root: filepath.Join(c.Root, "uploads")}
files.Purge()
m.SetUploads(files)
if err = m.Reconcile(a.applicationConfig.Context); err != nil {
return err
}
+129
View File
@@ -0,0 +1,129 @@
// SPDX-License-Identifier: MIT
package addons
import (
"context"
"errors"
"github.com/mudler/LocalAI/core/services/addons/packages"
"github.com/mudler/LocalAI/core/services/addons/uploads"
"github.com/mudler/LocalAI/pkg/addons/contract"
"github.com/mudler/xlog"
)
// ErrNoFileInstall: this owner has no staging area for local files.
var ErrNoFileInstall = errors.New("this add-on owner cannot install from a file")
// Relations between a staged file and what is installed.
const (
FileNew = "new"
FileUpgrade = "upgrade"
FileSame = "same"
FileDowngrade = "downgrade"
)
type fileOwnerKey struct{}
// WithFileOwner names the user whose staged files a call may read. The host
// access service sets it from the authenticated actor; a call without it
// cannot stage a file.
func WithFileOwner(ctx context.Context, userID string) context.Context {
return context.WithValue(ctx, fileOwnerKey{}, userID)
}
func fileOwner(ctx context.Context) string {
v, _ := ctx.Value(fileOwnerKey{}).(string)
return v
}
// SetUploads sets the staging area of local files. The first call wins.
// It is read under the manager lock and outside it, so it has its own guard.
func (m *Manager) SetUploads(s *uploads.Store) {
if s != nil {
m.uploads.CompareAndSwap(nil, s)
}
}
func (m *Manager) uploadStore() *uploads.Store { return m.uploads.Load() }
// stageUpload stages the package of a staged file of the user. It does not
// hold the manager lock: it reads and writes the disk only.
func (m *Manager) stageUpload(ctx context.Context, actor contract.Actor, fileID string) (packages.StagedPackage, error) {
store := m.uploadStore()
if store == nil || m.installer == nil {
return packages.StagedPackage{}, ErrNoFileInstall
}
p, entry, err := store.Open(fileID, actor.UserID)
if err != nil {
return packages.StagedPackage{}, err
}
return m.installer.StageFile(ctx, p, entry.Name, nil)
}
func (m *Manager) dropUpload(actor contract.Actor, fileID string) {
if store := m.uploadStore(); store != nil {
if err := store.Remove(fileID, actor.UserID); err != nil && !errors.Is(err, uploads.ErrNotFound) {
xlog.Warn("Could not remove a staged add-on file", "error", err)
}
}
}
// FileRelation says what installing the file would do to the add-on.
func FileRelation(installed bool, current, candidate string) string {
if !installed {
return FileNew
}
switch c := contract.CompareVersions(candidate, current); {
case c > 0:
return FileUpgrade
case c < 0:
return FileDowngrade
}
return FileSame
}
// fileVersionRefusal applies the version rules of an install from a file:
// the same version is never reinstalled, and an older one only after the
// administrator confirmed the downgrade. It returns nil when the file may go on.
func (m *Manager) fileVersionRefusal(cmd LifecycleCommand, next Status, staged packages.PackageReceipt) *FailureCause {
cur, ok := currentReceipt(next)
if !next.Installed || !ok {
return nil
}
switch FileRelation(true, cur.Manifest.Version, staged.Manifest.Version) {
case FileSame:
return &FailureCause{Code: CauseVersionNotNewer}
case FileDowngrade:
if !cmd.Downgrade {
return &FailureCause{Code: CauseDowngradeRefused}
}
}
return nil
}
// previewFile stages the file of the user named by the context, reads its
// verified manifest and discards the stage. It changes no state of the add-ons.
func (m *Manager) previewFile(ctx context.Context, fileID string) (HostAccessRelease, error) {
user := fileOwner(ctx)
if user == "" {
return HostAccessRelease{}, uploads.ErrNotFound
}
staged, e := m.stageUpload(ctx, contract.Actor{UserID: user}, fileID)
if e != nil {
return HostAccessRelease{}, e
}
rc := staged.Receipt()
if e = m.installer.Discard(staged); e != nil {
return HostAccessRelease{}, e
}
m.mu.Lock()
installed := m.journal.Addons[rc.Manifest.ID].Installed
m.mu.Unlock()
out := HostAccessRelease{AddonID: rc.Manifest.ID, Title: rc.Manifest.ID, Version: rc.Manifest.Version, Digest: rc.Digest,
Official: false, Installed: installed}
if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" {
out.Title = u.Title.Default()
}
out.Required, out.Optional = declared(rc)
return out, nil
}
+161
View File
@@ -0,0 +1,161 @@
// SPDX-License-Identifier: MIT
package addons
import (
"context"
"errors"
"github.com/mudler/LocalAI/core/services/addons/packages"
"github.com/mudler/LocalAI/core/services/addons/uploads"
"github.com/mudler/LocalAI/pkg/addons/contract"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
)
// FileView is what the administrator sees of a staged zip before the install
// dialog: what it is, how it relates to the installed add-on, and the digest
// the approval binds to. It carries no path and no credential.
type FileView struct {
FileID string `json:"fileId"`
Name string `json:"name"`
Size int64 `json:"size"`
// SHA256 is the digest of the whole file, in hex. The install approval
// binds to "sha256:" plus this value.
SHA256 string `json:"sha256"`
AddonID string `json:"addonId"`
Title string `json:"title"`
Version string `json:"version"`
Platform string `json:"platform"`
// API is the host API range the manifest declares.
API contract.VersionRange `json:"api"`
// Required and Optional are the host operations the manifest asks for.
Required []string `json:"required"`
Optional []string `json:"optional"`
Installed bool `json:"installed"`
InstalledVersion string `json:"installedVersion,omitempty"`
// Relation is new, upgrade, same or downgrade.
Relation string `json:"relation"`
// Source and Origin say that nothing signs the file.
Source string `json:"source"`
Origin string `json:"origin"`
}
// FileBegin opens an upload. It returns the ID of the upload and the byte limit.
func (m *Manager) FileBegin(ctx context.Context, actor contract.Actor, name string) (string, int64, error) {
if e := m.authorize(ctx, actor); e != nil {
return "", 0, e
}
store := m.uploadStore()
if store == nil || m.installer == nil {
return "", 0, ErrNoFileInstall
}
max, _ := m.installer.Limits()
id, e := store.Begin(actor.UserID, packages.SafeFileName(name), max)
return id, max, e
}
// FileChunk adds the next chunk to an upload.
func (m *Manager) FileChunk(ctx context.Context, actor contract.Actor, id string, offset int64, data []byte) error {
if e := m.authorize(ctx, actor); e != nil {
return e
}
store := m.uploadStore()
if store == nil {
return ErrNoFileInstall
}
return store.Append(id, actor.UserID, offset, data)
}
// FileAbort drops an upload that did not finish.
func (m *Manager) FileAbort(ctx context.Context, actor contract.Actor, id string) {
if store := m.uploadStore(); store != nil && m.authorize(ctx, actor) == nil {
store.Abort(id, actor.UserID)
}
}
// FileFinish closes the upload and verifies the file as the install will: the
// staged package is built and thrown away. A file that fails is removed.
func (m *Manager) FileFinish(ctx context.Context, actor contract.Actor, id string) (FileView, error) {
if e := m.authorize(ctx, actor); e != nil {
return FileView{}, e
}
store := m.uploadStore()
if store == nil {
return FileView{}, ErrNoFileInstall
}
entry, e := store.Finish(id, actor.UserID)
if e != nil {
return FileView{}, e
}
v, e := m.describeStaged(ctx, actor, entry)
if e != nil {
_ = store.Remove(entry.ID, actor.UserID)
return FileView{}, e
}
return v, nil
}
// FileDescribe reads a staged file of the user again.
func (m *Manager) FileDescribe(ctx context.Context, actor contract.Actor, id string) (FileView, error) {
if e := m.authorize(ctx, actor); e != nil {
return FileView{}, e
}
store := m.uploadStore()
if store == nil {
return FileView{}, ErrNoFileInstall
}
_, entry, e := store.Open(id, actor.UserID)
if e != nil {
return FileView{}, e
}
return m.describeStaged(ctx, actor, entry)
}
// FileDrop removes a staged file of the user.
func (m *Manager) FileDrop(ctx context.Context, actor contract.Actor, id string) error {
if e := m.authorize(ctx, actor); e != nil {
return e
}
store := m.uploadStore()
if store == nil {
return ErrNoFileInstall
}
return store.Remove(id, actor.UserID)
}
func (m *Manager) describeStaged(ctx context.Context, actor contract.Actor, entry uploads.Entry) (FileView, error) {
staged, e := m.stageUpload(ctx, actor, entry.ID)
if e != nil {
return FileView{}, e
}
rc := staged.Receipt()
if e = m.installer.Discard(staged); e != nil {
return FileView{}, e
}
m.mu.Lock()
st := m.journal.Addons[rc.Manifest.ID]
m.mu.Unlock()
v := FileView{FileID: entry.ID, Name: entry.Name, Size: entry.Size, SHA256: entry.SHA256, AddonID: rc.Manifest.ID, Title: rc.Manifest.ID,
Version: rc.Manifest.Version, Platform: rc.Manifest.OS + "/" + rc.Manifest.Arch, API: rc.Manifest.API,
Source: zipformat.SourceID, Origin: zipformat.OriginLabel, Relation: FileNew}
if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" {
v.Title = u.Title.Default()
}
v.Required, v.Optional = declared(rc)
if v.Required == nil {
v.Required = []string{}
}
if v.Optional == nil {
v.Optional = []string{}
}
if cur, ok := currentReceipt(st); ok && st.Installed {
v.Installed, v.InstalledVersion = true, cur.Manifest.Version
v.Relation = FileRelation(true, cur.Manifest.Version, rc.Manifest.Version)
}
return v, nil
}
// IsFileRefusal reports whether e is a refusal of the content of a file.
func IsFileRefusal(e error) (*packages.FileError, bool) {
var fe *packages.FileError
return fe, errors.As(e, &fe)
}
+5
View File
@@ -9,6 +9,7 @@ import (
"github.com/mudler/LocalAI/core/services/addons/catalog"
"github.com/mudler/LocalAI/core/services/addons/packages"
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
)
// OfficialReceipt reports whether a package came from the official add-on
@@ -137,6 +138,10 @@ func (m *Manager) PreviewRelease(ctx context.Context, sourceID, addonID, version
}
installed := m.journal.Addons[addonID].Installed
m.mu.Unlock()
if sourceID == zipformat.SourceID {
// A local file: the version field names the staged file.
return m.previewFile(ctx, version)
}
r, e := m.resolveRelease(ctx, LifecycleCommand{SourceID: sourceID, AddonID: addonID, Version: version})
if e != nil {
return HostAccessRelease{}, e
+48 -7
View File
@@ -27,7 +27,9 @@ import (
"github.com/mudler/LocalAI/core/services/addons/recovery"
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
"github.com/mudler/LocalAI/core/services/addons/updates"
"github.com/mudler/LocalAI/core/services/addons/uploads"
"github.com/mudler/LocalAI/pkg/addons/contract"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
)
var (
@@ -75,6 +77,14 @@ const (
CausePackageFetch = "package_fetch_failed"
// CauseInstallStorage: the verified package could not be written to disk.
CauseInstallStorage = "install_storage_failed"
// CauseFileMissing: the staged file of a local file install is gone, has
// expired or belongs to another user.
CauseFileMissing = "staged_file_missing"
// CauseVersionNotNewer: the file holds the version that is installed.
CauseVersionNotNewer = "version_not_newer"
// CauseDowngradeRefused: the file holds an older version and the request
// did not confirm the downgrade.
CauseDowngradeRefused = "downgrade_refused"
)
// FailureCause is the typed cause of a failed operation. Its text is fixed and
@@ -177,6 +187,9 @@ type Manager struct {
// identities maps an installed add-on to its trusted identity; see
// TrustedIdentity.
identities map[string]string
// uploads holds the zip files waiting for an install. Nil means the owner
// does not install from a file.
uploads atomic.Pointer[uploads.Store]
}
// SetAuthority sets the Authority that admission and the apps list ask. The
@@ -614,23 +627,47 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl
return op.Status, e
}
m.mu.Unlock()
if release != nil {
r = *release
} else {
r, resolveErr = m.resolveRelease(ctx, cmd)
}
fromFile := cmd.SourceID == zipformat.SourceID
var staged packages.StagedPackage
var stageErr error
if resolveErr == nil {
staged, stageErr = m.installer.Stage(ctx, r, nil)
if fromFile {
// A local file has no library entry: the staged file is the
// release, and the same Stage checks apply to its contents.
staged, stageErr = m.stageUpload(ctx, actor, cmd.Version)
} else {
if release != nil {
r = *release
} else {
r, resolveErr = m.resolveRelease(ctx, cmd)
}
if resolveErr == nil {
staged, stageErr = m.installer.Stage(ctx, r, nil)
}
}
m.mu.Lock()
if resolveErr != nil {
return fail(resolveErr, &FailureCause{Code: resolveCauseCode(resolveErr), detail: resolveErr})
}
if stageErr != nil {
if errors.Is(stageErr, uploads.ErrNotFound) {
return fail(stageErr, &FailureCause{Code: CauseFileMissing, detail: stageErr})
}
return fail(stageErr, &FailureCause{Code: CausePackageFetch, detail: stageErr})
}
if fromFile {
if cause := m.fileVersionRefusal(cmd, next, staged.Receipt()); cause != nil {
if discardErr := m.installer.Discard(staged); discardErr != nil {
xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr)
}
return fail(cause)
}
if staged.Receipt().Manifest.ID != cmd.AddonID {
if discardErr := m.installer.Discard(staged); discardErr != nil {
xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr)
}
return fail(packages.ErrPackage)
}
}
if m.uncertain || m.runtimeRecoveryRequired {
return op.Status, ErrRecoveryRequired
}
@@ -764,6 +801,10 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl
case "install", "upgrade", "uninstall":
m.syncSkills(ctx, cmd.AddonID)
}
if cmd.SourceID == zipformat.SourceID && (cmd.Action == "install" || cmd.Action == "upgrade") {
// The package is installed; the staged file has done its work.
m.dropUpload(actor, cmd.Version)
}
return result, m.syncFor(ctx, cmd.AddonID)
}
+210
View File
@@ -0,0 +1,210 @@
//go:build linux
// SPDX-License-Identifier: MIT
package addons
import (
"bytes"
"context"
"encoding/json"
"errors"
"os"
"path/filepath"
"strings"
"github.com/mudler/LocalAI/core/services/addons/packages"
"github.com/mudler/LocalAI/core/services/addons/uploads"
"github.com/mudler/LocalAI/pkg/addons/contract"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)
var _ = Describe("install from a local zip file", func() {
var r *updateRig
var base map[string]any
BeforeEach(func() {
r = newUpdateRig(false)
r.m.SetUploads(&uploads.Store{Root: filepath.Join(r.cfg.Root, "uploads")})
raw, e := os.ReadFile("../../../tests/addons/fixtures/packages/manifest.json")
Expect(e).NotTo(HaveOccurred())
Expect(json.Unmarshal(raw, &base)).To(Succeed())
})
// zipFor builds the zip of a sample add-on of the given version that asks
// for the given required host operations.
zipFor := func(version, osName string, required ...string) []byte {
m := map[string]any{}
for k, v := range base {
m[k] = v
}
m["version"], m["os"] = version, osName
if len(required) > 0 {
m["capabilities"] = map[string]any{"service": true, "hostOperations": map[string]any{"required": required}}
}
mb, e := json.Marshal(m)
Expect(e).NotTo(HaveOccurred())
var b bytes.Buffer
Expect(zipformat.Write(&b, "sample", version, osName, "amd64", "bin/sample", []zipformat.Entry{
{Name: "manifest.json", Data: mb}, {Name: "bin/sample", Data: []byte("executable " + version), Exec: true},
})).To(Succeed())
return b.Bytes()
}
upload := func(user string, data []byte) (FileView, error) {
a := contract.Actor{UserID: user, CredentialID: "key", Mode: "named-user"}
id, _, e := r.m.FileBegin(ctx, a, "sample.zip")
Expect(e).NotTo(HaveOccurred())
for off := 0; off < len(data); off += 1000 {
end := min(off+1000, len(data))
Expect(r.m.FileChunk(ctx, a, id, int64(off), data[off:end])).To(Succeed())
}
return r.m.FileFinish(ctx, a, id)
}
fileCmd := func(op, action string, v FileView) LifecycleCommand {
return LifecycleCommand{OperationID: op, AddonID: "sample", Action: action, SourceID: zipformat.SourceID, Version: v.FileID}
}
It("installs through the manager and records an unsigned local file", func() {
v, e := upload("admin", zipFor("2.0.0", "linux"))
Expect(e).NotTo(HaveOccurred())
Expect(v.Relation).To(Equal(FileNew))
Expect(v.Origin).To(Equal("local file, unsigned"))
Expect(v.Version).To(Equal("2.0.0"))
Expect(v.API.Min).To(Equal(contract.HostAPIVersion))
_, e = r.m.Submit(ctx, actor, fileCmd("f1", "install", v))
Expect(e).NotTo(HaveOccurred())
st, _ := r.m.Status(ctx, "sample")
Expect(st.Installed).To(BeTrue())
rc := st.Generations[0].Receipt
Expect(rc.SourceID).To(Equal("local-file"))
Expect(rc.Origin).To(Equal("local file, unsigned"))
Expect(rc.Verification).To(Equal("sha256"))
Expect(rc.Digest).To(Equal("sha256:" + v.SHA256))
// The staged file is gone once it is installed.
_, e = r.m.FileDescribe(ctx, actor, v.FileID)
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
// A package from a file has no library to ask for updates.
list, e := r.m.Installed(ctx, actor)
Expect(e).NotTo(HaveOccurred())
Expect(list).To(HaveLen(1))
Expect(list[0].Update).To(BeNil())
Expect(list[0].SourceID).To(Equal("local-file"))
})
It("keeps a file for the user that uploaded it", func() {
v, e := upload("admin", zipFor("2.0.0", "linux"))
Expect(e).NotTo(HaveOccurred())
other := contract.Actor{UserID: "admin-two", CredentialID: "key", Mode: "named-user"}
_, e = r.m.stageUpload(ctx, other, v.FileID)
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
// A command of another user is refused by the administrator check
// before it can reach the file.
_, e = r.m.Submit(ctx, other, fileCmd("f1", "install", v))
Expect(e).To(MatchError(ErrForbidden))
st, _ := r.m.Status(ctx, "sample")
Expect(st.Installed).To(BeFalse())
})
It("refuses a file for another platform when it is uploaded, and leaves nothing staged", func() {
_, e := upload("admin", zipFor("2.0.0", "darwin"))
var fe *packages.FileError
Expect(errors.As(e, &fe)).To(BeTrue())
Expect(fe.Code).To(Equal(packages.FileWrongPlatform))
ents, _ := os.ReadDir(filepath.Join(r.cfg.Root, "uploads"))
Expect(ents).To(BeEmpty())
})
It("refuses an install when the administrator is not one", func() {
_, _, e := r.m.FileBegin(ctx, contract.Actor{UserID: "user", Mode: "named-user"}, "x.zip")
Expect(e).To(MatchError(ErrForbidden))
})
Describe("versions", func() {
install := func(version string) FileView {
v, e := upload("admin", zipFor(version, "linux"))
Expect(e).NotTo(HaveOccurred())
_, e = r.m.Submit(ctx, actor, fileCmd("i-"+version, "install", v))
Expect(e).NotTo(HaveOccurred())
return v
}
BeforeEach(func() { install("2.0.0") })
It("upgrades to a higher version through the same generation machinery", func() {
v, e := upload("admin", zipFor("2.1.0", "linux"))
Expect(e).NotTo(HaveOccurred())
Expect(v.Relation).To(Equal(FileUpgrade))
Expect(v.InstalledVersion).To(Equal("2.0.0"))
_, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v))
Expect(e).NotTo(HaveOccurred())
st, _ := r.m.Status(ctx, "sample")
Expect(st.Generation).To(Equal(uint64(2)))
Expect(st.Generations[1].Receipt.Manifest.Version).To(Equal("2.1.0"))
})
It("refuses the same version", func() {
v, e := upload("admin", zipFor("2.0.0", "linux"))
Expect(e).NotTo(HaveOccurred())
Expect(v.Relation).To(Equal(FileSame))
_, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v))
Expect(e).To(MatchError(ErrOperationFailed))
Expect(e.Error()).To(ContainSubstring(CauseVersionNotNewer))
st, _ := r.m.Status(ctx, "sample")
Expect(st.Generation).To(Equal(uint64(1)))
})
It("refuses a lower version unless the downgrade is confirmed", func() {
v, e := upload("admin", zipFor("1.5.0", "linux"))
Expect(e).NotTo(HaveOccurred())
Expect(v.Relation).To(Equal(FileDowngrade))
_, e = r.m.Submit(ctx, actor, fileCmd("d1", "upgrade", v))
Expect(e).To(MatchError(ErrOperationFailed))
Expect(e.Error()).To(ContainSubstring(CauseDowngradeRefused))
st, _ := r.m.Status(ctx, "sample")
Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("2.0.0"))
// The file stays staged for a retry with the confirmation.
cmd := fileCmd("d2", "upgrade", v)
cmd.Downgrade = true
_, e = r.m.Submit(ctx, actor, cmd)
Expect(e).NotTo(HaveOccurred())
st, _ = r.m.Status(ctx, "sample")
Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("1.5.0"))
})
})
Describe("host access approval", func() {
It("grants only what the administrator approved for the digest of the file", func() {
g := newGrantSim(r.m)
v, e := upload("admin", zipFor("2.0.0", "linux", "workspace.materialize"))
Expect(e).NotTo(HaveOccurred())
Expect(v.Required).To(Equal([]string{"workspace.materialize"}))
_, e = r.m.Submit(ctx, actor, fileCmd("f0", "install", v))
var refusal *HostRequirementsRefusal
Expect(errors.As(e, &refusal)).To(BeTrue(), "an unapproved required operation is refused")
// Previews give the digest the dialog shows.
rel, e := r.m.PreviewRelease(WithFileOwner(ctx, "admin"), zipformat.SourceID, "sample", v.FileID)
Expect(e).NotTo(HaveOccurred())
Expect(rel.Digest).To(Equal("sha256:" + v.SHA256))
Expect(rel.Official).To(BeFalse())
Expect(rel.Required).To(Equal([]string{"workspace.materialize"}))
cmd := fileCmd("f1", "install", v)
cmd.Approval = &HostApproval{Digest: "sha256:" + strings.Repeat("0", 64)}
_, e = r.m.Submit(ctx, actor, cmd)
var stale *ApprovalStaleError
Expect(errors.As(e, &stale)).To(BeTrue(), "another digest is stale")
cmd = fileCmd("f2", "install", v)
cmd.Approval = &HostApproval{Digest: rel.Digest, Ops: []string{"workspace.materialize", "egress.check"}, Revision: 3}
_, e = r.m.Submit(ctx, actor, cmd)
Expect(e).NotTo(HaveOccurred())
Expect(g.stagedOps("f2")).To(Equal([]string{"workspace.materialize"}))
Expect(r.m.OperationCompleted("f2")).To(BeTrue())
})
It("previews only the files of the calling user", func() {
v, e := upload("admin", zipFor("2.0.0", "linux"))
Expect(e).NotTo(HaveOccurred())
_, e = r.m.PreviewRelease(WithFileOwner(ctx, "someone"), zipformat.SourceID, "sample", v.FileID)
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
_, e = r.m.PreviewRelease(context.Background(), zipformat.SourceID, "sample", v.FileID)
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
})
})
})
+35 -3
View File
@@ -15,6 +15,7 @@ import (
"regexp"
"strings"
"github.com/mudler/LocalAI/pkg/addons/contract"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
)
@@ -55,6 +56,10 @@ const (
var fileNameRE = regexp.MustCompile(`[^A-Za-z0-9._-]`)
// Limits returns the byte and file caps of a package, the ones that apply to
// a library package and to a local file alike.
func (i *Installer) Limits() (bytes int64, files int) { return i.byteLimit(), i.fileLimit() }
// FileSummary is what a verified zip says about itself. It is computed without
// writing the package to disk.
type FileSummary struct {
@@ -63,6 +68,9 @@ type FileSummary struct {
Size int64 `json:"size"`
Files int `json:"files"`
zipformat.Manifest
// Package is manifest.json of the add-on, parsed. InspectFile fills it; it
// is not checked against a host there.
Package contract.Manifest `json:"package"`
}
// zipSink receives each file of the package after the path and type checks.
@@ -223,10 +231,34 @@ func (c *countingReader) Read(b []byte) (int, error) {
// installer; zero takes the defaults.
func InspectFile(ctx context.Context, p string, maxBytes int64, maxFiles int) (FileSummary, error) {
i := &Installer{MaxBytes: maxBytes, MaxFiles: maxFiles}
return readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(_ string, _ bool, r io.Reader) error {
_, e := io.Copy(io.Discard, r)
return e
var pkg contract.Manifest
sum, e := readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(n string, _ bool, r io.Reader) error {
if n != "manifest.json" {
_, e := io.Copy(io.Discard, r)
return e
}
b, e := io.ReadAll(io.LimitReader(r, contract.DefaultLimits().ManifestBytes+1))
if e != nil {
return e
}
if int64(len(b)) > contract.DefaultLimits().ManifestBytes {
return fileErr(FileBadManifest, "manifest.json is too large")
}
m, e := contract.ParseManifest(b)
if e != nil {
return fileErr(FileBadManifest, "manifest.json is not a valid add-on manifest")
}
pkg = m
return nil
})
if e != nil {
return sum, e
}
if pkg.ID == "" || pkg.ID != sum.ID || pkg.Version != sum.Version || pkg.OS != sum.OS || pkg.Arch != sum.Arch {
return sum, fileErr(FileBadManifest, "The manifest of the add-on does not match the digest manifest of the zip")
}
sum.Package = pkg
return sum, nil
}
// SafeFileName reduces the name a client gave a file to characters that are
+3
View File
@@ -60,6 +60,9 @@ type LifecycleCommand struct {
// registers. It carries no authority by itself: the caller must be a human
// surface, which the gateway checks before it sets this field.
Approval *HostApproval `json:",omitempty"`
// Downgrade confirms that a local file may install an older version than
// the one installed. Without it such a file is refused.
Downgrade bool `json:",omitempty"`
}
// HostApproval binds an approval to one staged package and one revision of the
+4 -1
View File
@@ -4,6 +4,7 @@ package addons
import (
"context"
"errors"
"github.com/mudler/LocalAI/pkg/addons/zipformat"
"time"
"github.com/mudler/LocalAI/core/services/addons/apps"
@@ -55,7 +56,9 @@ func (r *updateReader) Installation(ctx context.Context, id string) (updates.Ins
return updates.Installation{}, false, nil
}
receipt, ok := currentReceipt(st)
if !ok {
if !ok || receipt.SourceID == zipformat.SourceID {
// A package from a local file has no library to ask, so no check can
// answer for it. The page reads that as unknown, never as current.
return updates.Installation{}, false, nil
}
return updates.Installation{AddonID: id, Generation: st.Generation, Receipt: receipt}, true, nil
+332
View File
@@ -0,0 +1,332 @@
// SPDX-License-Identifier: MIT
// Package uploads is the owner's staging area for add-on zip files. A file
// waits here between the upload and the install the administrator confirms.
// It is private to the owner, bound to the user that uploaded it, and
// removed after a short time, after the install and at start.
package uploads
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"hash"
"io"
"os"
"path/filepath"
"regexp"
"sync"
"time"
)
var (
// ErrNotFound: no such file, or it belongs to another user, or it expired.
ErrNotFound = errors.New("staged file not found")
// ErrTooLarge: the upload went over the byte limit.
ErrTooLarge = errors.New("file is over the size limit")
// ErrBusy: too many files are waiting.
ErrBusy = errors.New("too many staged files")
// ErrOffset: a chunk did not follow the bytes received so far.
ErrOffset = errors.New("chunk out of order")
)
// Defaults of a store.
const (
DefaultTTL = time.Hour
DefaultMaxFiles = 16
)
var idRE = regexp.MustCompile(`^[0-9a-f]{32}$`)
// ValidID reports whether s has the shape of a staged file ID.
func ValidID(s string) bool { return idRE.MatchString(s) }
// Entry is a staged file.
type Entry struct {
ID string `json:"id"`
Owner string `json:"owner"`
Name string `json:"name"`
Size int64 `json:"size"`
SHA256 string `json:"sha256"`
Created time.Time `json:"created"`
}
// Store keeps staged files under Root.
type Store struct {
Root string
TTL time.Duration
MaxFiles int
// Now is replaced by tests.
Now func() time.Time
mu sync.Mutex
open map[string]*upload
}
type upload struct {
owner, name string
f *os.File
h hash.Hash
n int64
max int64
started time.Time
}
func (s *Store) now() time.Time {
if s.Now != nil {
return s.Now()
}
return time.Now()
}
func (s *Store) ttl() time.Duration {
if s.TTL > 0 {
return s.TTL
}
return DefaultTTL
}
func (s *Store) maxFiles() int {
if s.MaxFiles > 0 {
return s.MaxFiles
}
return DefaultMaxFiles
}
func (s *Store) dir() (string, error) {
if s.Root == "" || !filepath.IsAbs(s.Root) {
return "", errors.New("uploads: root must be an absolute path")
}
if err := os.MkdirAll(s.Root, 0o700); err != nil {
return "", err
}
info, err := os.Lstat(s.Root)
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 {
return "", errors.New("uploads: root must be a private directory")
}
return s.Root, nil
}
func newID() (string, error) {
var b [16]byte
if _, err := rand.Read(b[:]); err != nil {
return "", err
}
return hex.EncodeToString(b[:]), nil
}
func (s *Store) zipPath(id string) string { return filepath.Join(s.Root, id+".zip") }
func (s *Store) metaPath(id string) string { return filepath.Join(s.Root, id+".json") }
// Begin opens a new upload for owner. max is the byte limit of the file.
func (s *Store) Begin(owner, name string, max int64) (string, error) {
if owner == "" || max <= 0 {
return "", ErrNotFound
}
if _, err := s.dir(); err != nil {
return "", err
}
s.Sweep()
s.mu.Lock()
defer s.mu.Unlock()
if s.open == nil {
s.open = map[string]*upload{}
}
staged, _ := filepath.Glob(filepath.Join(s.Root, "*.zip"))
if len(staged)+len(s.open) >= s.maxFiles() {
return "", ErrBusy
}
id, err := newID()
if err != nil {
return "", err
}
f, err := os.OpenFile(s.zipPath(id), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) // #nosec G304 -- the name is a random ID under the private staging root
if err != nil {
return "", err
}
s.open[id] = &upload{owner: owner, name: name, f: f, h: sha256.New(), max: max, started: s.now()}
return id, nil
}
// Append adds the next chunk. offset is the number of bytes received before it.
func (s *Store) Append(id, owner string, offset int64, p []byte) error {
s.mu.Lock()
defer s.mu.Unlock()
u, ok := s.open[id]
if !ok || u.owner != owner {
return ErrNotFound
}
if offset != u.n {
return ErrOffset
}
if int64(len(p)) > u.max-u.n {
s.abortLocked(id)
return ErrTooLarge
}
if _, err := u.f.Write(p); err != nil {
s.abortLocked(id)
return err
}
_, _ = u.h.Write(p)
u.n += int64(len(p))
return nil
}
// Finish closes the upload and makes it a staged file.
func (s *Store) Finish(id, owner string) (Entry, error) {
s.mu.Lock()
defer s.mu.Unlock()
u, ok := s.open[id]
if !ok || u.owner != owner {
return Entry{}, ErrNotFound
}
delete(s.open, id)
e := Entry{ID: id, Owner: owner, Name: u.name, Size: u.n, SHA256: hex.EncodeToString(u.h.Sum(nil)), Created: s.now().UTC()}
err := u.f.Sync()
if ce := u.f.Close(); err == nil {
err = ce
}
if err == nil {
var b []byte
if b, err = json.Marshal(e); err == nil {
err = os.WriteFile(s.metaPath(id), b, 0o600)
}
}
if err != nil {
_ = os.Remove(s.zipPath(id))
return Entry{}, err
}
return e, nil
}
// Abort drops an upload that is not finished. It is safe to call twice.
func (s *Store) Abort(id, owner string) {
s.mu.Lock()
defer s.mu.Unlock()
if u, ok := s.open[id]; ok && u.owner == owner {
s.abortLocked(id)
}
}
func (s *Store) abortLocked(id string) {
if u, ok := s.open[id]; ok {
_ = u.f.Close()
delete(s.open, id)
}
_ = os.Remove(s.zipPath(id))
}
// Put stores the whole of r as a staged file of owner, up to max bytes.
func (s *Store) Put(owner, name string, max int64, r io.Reader) (Entry, error) {
id, err := s.Begin(owner, name, max)
if err != nil {
return Entry{}, err
}
buf := make([]byte, 256<<10)
var off int64
for {
n, rerr := r.Read(buf)
if n > 0 {
if err = s.Append(id, owner, off, buf[:n]); err != nil {
return Entry{}, err
}
off += int64(n)
}
if rerr == io.EOF {
break
}
if rerr != nil {
s.Abort(id, owner)
return Entry{}, rerr
}
}
return s.Finish(id, owner)
}
// Open returns the path and the entry of a staged file of owner.
func (s *Store) Open(id, owner string) (string, Entry, error) {
if !ValidID(id) || owner == "" {
return "", Entry{}, ErrNotFound
}
e, err := s.read(id)
if err != nil || e.Owner != owner || s.expired(e) {
return "", Entry{}, ErrNotFound
}
return s.zipPath(id), e, nil
}
func (s *Store) read(id string) (Entry, error) {
var e Entry
b, err := os.ReadFile(s.metaPath(id)) // #nosec G304 -- the name is a validated random ID under the private staging root
if err != nil {
return e, ErrNotFound
}
if json.Unmarshal(b, &e) != nil || e.ID != id {
return Entry{}, ErrNotFound
}
return e, nil
}
func (s *Store) expired(e Entry) bool { return s.now().Sub(e.Created) > s.ttl() }
// Remove deletes a staged file of owner.
func (s *Store) Remove(id, owner string) error {
if _, _, err := s.Open(id, owner); err != nil {
return err
}
return s.removeFiles(id)
}
func (s *Store) removeFiles(id string) error {
err := os.Remove(s.zipPath(id))
_ = os.Remove(s.metaPath(id))
if err != nil && !os.IsNotExist(err) {
return fmt.Errorf("remove staged file: %w", err)
}
return nil
}
// Sweep removes the files that are older than the time to live and the
// uploads that never finished. With purge set it removes everything that is
// not an open upload, as the owner does at start.
func (s *Store) Sweep() { s.sweep(false) }
// Purge removes every staged file. Call it once at start.
func (s *Store) Purge() { s.sweep(true) }
func (s *Store) sweep(all bool) {
if _, err := s.dir(); err != nil {
return
}
// The lock covers the whole sweep, so a Finish in progress is never seen
// half done.
s.mu.Lock()
defer s.mu.Unlock()
for id, u := range s.open {
if all || s.now().Sub(u.started) > s.ttl() {
s.abortLocked(id)
}
}
open := map[string]bool{}
for id := range s.open {
open[id] = true
}
ents, err := os.ReadDir(s.Root)
if err != nil {
return
}
for _, de := range ents {
name := de.Name()
id := name[:len(name)-len(filepath.Ext(name))]
if !ValidID(id) || open[id] {
continue
}
e, rerr := s.read(id)
if all || rerr != nil || s.expired(e) {
_ = s.removeFiles(id)
}
}
}
@@ -0,0 +1,126 @@
// SPDX-License-Identifier: MIT
package uploads
import (
"bytes"
"errors"
"os"
"path/filepath"
"strings"
"testing"
"time"
)
func newStore(t *testing.T) (*Store, *time.Time) {
t.Helper()
now := time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC)
root := filepath.Join(t.TempDir(), "uploads")
return &Store{Root: root, Now: func() time.Time { return now }, MaxFiles: 2}, &now
}
func TestPutOpenAndOwner(t *testing.T) {
s, _ := newStore(t)
e, err := s.Put("alice", "x.zip", 1<<20, strings.NewReader("hello"))
if err != nil {
t.Fatal(err)
}
if e.Size != 5 || len(e.SHA256) != 64 || !ValidID(e.ID) {
t.Fatalf("entry %+v", e)
}
p, got, err := s.Open(e.ID, "alice")
if err != nil || got.ID != e.ID {
t.Fatalf("open: %v", err)
}
if b, _ := os.ReadFile(p); string(b) != "hello" {
t.Fatalf("content %q", b)
}
if _, _, err = s.Open(e.ID, "bob"); !errors.Is(err, ErrNotFound) {
t.Fatalf("another user must not open it: %v", err)
}
if _, _, err = s.Open("../etc/passwd", "alice"); !errors.Is(err, ErrNotFound) {
t.Fatalf("a path is not an ID: %v", err)
}
info, _ := os.Stat(p)
if info.Mode().Perm() != 0o600 {
t.Fatalf("mode %v", info.Mode())
}
if err = s.Remove(e.ID, "bob"); !errors.Is(err, ErrNotFound) {
t.Fatal("another user removed it")
}
if err = s.Remove(e.ID, "alice"); err != nil {
t.Fatal(err)
}
if _, _, err = s.Open(e.ID, "alice"); !errors.Is(err, ErrNotFound) {
t.Fatal("still there")
}
}
func TestSizeLimitStopsTheUpload(t *testing.T) {
s, _ := newStore(t)
_, err := s.Put("alice", "x.zip", 10, bytes.NewReader(make([]byte, 11)))
if !errors.Is(err, ErrTooLarge) {
t.Fatalf("got %v", err)
}
ents, _ := os.ReadDir(s.Root)
if len(ents) != 0 {
t.Fatalf("left files: %v", ents)
}
}
func TestChunksMustFollowEachOther(t *testing.T) {
s, _ := newStore(t)
id, err := s.Begin("alice", "x.zip", 100)
if err != nil {
t.Fatal(err)
}
if err = s.Append(id, "alice", 0, []byte("ab")); err != nil {
t.Fatal(err)
}
if err = s.Append(id, "alice", 5, []byte("cd")); !errors.Is(err, ErrOffset) {
t.Fatalf("gap: %v", err)
}
if err = s.Append(id, "bob", 2, []byte("cd")); !errors.Is(err, ErrNotFound) {
t.Fatalf("other user: %v", err)
}
if _, err = s.Finish(id, "bob"); !errors.Is(err, ErrNotFound) {
t.Fatalf("other user finish: %v", err)
}
e, err := s.Finish(id, "alice")
if err != nil || e.Size != 2 {
t.Fatalf("finish: %v %+v", err, e)
}
}
func TestExpiryBusyAndPurge(t *testing.T) {
s, now := newStore(t)
a, _ := s.Put("alice", "a.zip", 100, strings.NewReader("a"))
if _, err := s.Put("alice", "b.zip", 100, strings.NewReader("b")); err != nil {
t.Fatal(err)
}
if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); !errors.Is(err, ErrBusy) {
t.Fatalf("limit: %v", err)
}
*now = now.Add(2 * time.Hour)
if _, _, err := s.Open(a.ID, "alice"); !errors.Is(err, ErrNotFound) {
t.Fatal("an expired file opens")
}
if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); err != nil {
t.Fatalf("sweep did not free room: %v", err)
}
s.Purge()
ents, _ := os.ReadDir(s.Root)
if len(ents) != 0 {
t.Fatalf("purge left %v", ents)
}
}
func TestRootMustBePrivate(t *testing.T) {
root := filepath.Join(t.TempDir(), "open")
if err := os.Mkdir(root, 0o755); err != nil {
t.Fatal(err)
}
s := &Store{Root: root}
if _, err := s.Put("alice", "x.zip", 10, strings.NewReader("x")); err == nil {
t.Fatal("a world-readable root was accepted")
}
}