mirror of
https://github.com/mudler/LocalAI.git
synced 2026-11-05 20:13:23 -05:00
feat(addons): install, upgrade and downgrade from a staged zip file
The manager takes a staged file as a release: sourceId local-file and the file ID as the version. It runs through the one lifecycle path, so generations, backup, recovery and the host access approval apply as for a library package. The approval digest is the sha256 of the zip. The same version is refused, and an older one needs a confirmed downgrade. Files wait in a private per-user staging area for an hour. An add-on from a file has no library, so no update check answers for it. Assisted-by: Claude:claude-sonnet-5-5 go gofmt
This commit is contained in:
11 files changed
+1059
-11
No files matched your search
@@ -35,6 +35,7 @@ import (
|
||||
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
|
||||
"github.com/mudler/LocalAI/core/services/addons/updates"
|
||||
"github.com/mudler/LocalAI/core/services/addons/usersettings"
|
||||
"github.com/mudler/LocalAI/core/services/addons/uploads"
|
||||
"github.com/mudler/LocalAI/core/services/advisorylock"
|
||||
"github.com/mudler/LocalAI/core/services/cloudproxy/mitm"
|
||||
"github.com/mudler/LocalAI/core/services/facerecognition"
|
||||
@@ -1051,6 +1052,11 @@ func (a *Application) startAddons() error {
|
||||
// The update check reads the libraries and the journal the owner
|
||||
// holds; it applies nothing, and its cache is bounded.
|
||||
m.EnableUpdates(updates.Config{TTL: c.UpdateCacheTTL, MinRefresh: c.UpdateRefreshInterval})
|
||||
// The staging area of zip files waits empty at start: a file that was
|
||||
// uploaded before a restart has to be uploaded again.
|
||||
files := &uploads.Store{Root: filepath.Join(c.Root, "uploads")}
|
||||
files.Purge()
|
||||
m.SetUploads(files)
|
||||
if err = m.Reconcile(a.applicationConfig.Context); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package addons
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/addons/packages"
|
||||
"github.com/mudler/LocalAI/core/services/addons/uploads"
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
"github.com/mudler/xlog"
|
||||
)
|
||||
|
||||
// ErrNoFileInstall: this owner has no staging area for local files.
|
||||
var ErrNoFileInstall = errors.New("this add-on owner cannot install from a file")
|
||||
|
||||
// Relations between a staged file and what is installed.
|
||||
const (
|
||||
FileNew = "new"
|
||||
FileUpgrade = "upgrade"
|
||||
FileSame = "same"
|
||||
FileDowngrade = "downgrade"
|
||||
)
|
||||
|
||||
type fileOwnerKey struct{}
|
||||
|
||||
// WithFileOwner names the user whose staged files a call may read. The host
|
||||
// access service sets it from the authenticated actor; a call without it
|
||||
// cannot stage a file.
|
||||
func WithFileOwner(ctx context.Context, userID string) context.Context {
|
||||
return context.WithValue(ctx, fileOwnerKey{}, userID)
|
||||
}
|
||||
|
||||
func fileOwner(ctx context.Context) string {
|
||||
v, _ := ctx.Value(fileOwnerKey{}).(string)
|
||||
return v
|
||||
}
|
||||
|
||||
// SetUploads sets the staging area of local files. The first call wins.
|
||||
// It is read under the manager lock and outside it, so it has its own guard.
|
||||
func (m *Manager) SetUploads(s *uploads.Store) {
|
||||
if s != nil {
|
||||
m.uploads.CompareAndSwap(nil, s)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *Manager) uploadStore() *uploads.Store { return m.uploads.Load() }
|
||||
|
||||
// stageUpload stages the package of a staged file of the user. It does not
|
||||
// hold the manager lock: it reads and writes the disk only.
|
||||
func (m *Manager) stageUpload(ctx context.Context, actor contract.Actor, fileID string) (packages.StagedPackage, error) {
|
||||
store := m.uploadStore()
|
||||
if store == nil || m.installer == nil {
|
||||
return packages.StagedPackage{}, ErrNoFileInstall
|
||||
}
|
||||
p, entry, err := store.Open(fileID, actor.UserID)
|
||||
if err != nil {
|
||||
return packages.StagedPackage{}, err
|
||||
}
|
||||
return m.installer.StageFile(ctx, p, entry.Name, nil)
|
||||
}
|
||||
|
||||
func (m *Manager) dropUpload(actor contract.Actor, fileID string) {
|
||||
if store := m.uploadStore(); store != nil {
|
||||
if err := store.Remove(fileID, actor.UserID); err != nil && !errors.Is(err, uploads.ErrNotFound) {
|
||||
xlog.Warn("Could not remove a staged add-on file", "error", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// FileRelation says what installing the file would do to the add-on.
|
||||
func FileRelation(installed bool, current, candidate string) string {
|
||||
if !installed {
|
||||
return FileNew
|
||||
}
|
||||
switch c := contract.CompareVersions(candidate, current); {
|
||||
case c > 0:
|
||||
return FileUpgrade
|
||||
case c < 0:
|
||||
return FileDowngrade
|
||||
}
|
||||
return FileSame
|
||||
}
|
||||
|
||||
// fileVersionRefusal applies the version rules of an install from a file:
|
||||
// the same version is never reinstalled, and an older one only after the
|
||||
// administrator confirmed the downgrade. It returns nil when the file may go on.
|
||||
func (m *Manager) fileVersionRefusal(cmd LifecycleCommand, next Status, staged packages.PackageReceipt) *FailureCause {
|
||||
cur, ok := currentReceipt(next)
|
||||
if !next.Installed || !ok {
|
||||
return nil
|
||||
}
|
||||
switch FileRelation(true, cur.Manifest.Version, staged.Manifest.Version) {
|
||||
case FileSame:
|
||||
return &FailureCause{Code: CauseVersionNotNewer}
|
||||
case FileDowngrade:
|
||||
if !cmd.Downgrade {
|
||||
return &FailureCause{Code: CauseDowngradeRefused}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// previewFile stages the file of the user named by the context, reads its
|
||||
// verified manifest and discards the stage. It changes no state of the add-ons.
|
||||
func (m *Manager) previewFile(ctx context.Context, fileID string) (HostAccessRelease, error) {
|
||||
user := fileOwner(ctx)
|
||||
if user == "" {
|
||||
return HostAccessRelease{}, uploads.ErrNotFound
|
||||
}
|
||||
staged, e := m.stageUpload(ctx, contract.Actor{UserID: user}, fileID)
|
||||
if e != nil {
|
||||
return HostAccessRelease{}, e
|
||||
}
|
||||
rc := staged.Receipt()
|
||||
if e = m.installer.Discard(staged); e != nil {
|
||||
return HostAccessRelease{}, e
|
||||
}
|
||||
m.mu.Lock()
|
||||
installed := m.journal.Addons[rc.Manifest.ID].Installed
|
||||
m.mu.Unlock()
|
||||
out := HostAccessRelease{AddonID: rc.Manifest.ID, Title: rc.Manifest.ID, Version: rc.Manifest.Version, Digest: rc.Digest,
|
||||
Official: false, Installed: installed}
|
||||
if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" {
|
||||
out.Title = u.Title.Default()
|
||||
}
|
||||
out.Required, out.Optional = declared(rc)
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package addons
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/addons/packages"
|
||||
"github.com/mudler/LocalAI/core/services/addons/uploads"
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
)
|
||||
|
||||
// FileView is what the administrator sees of a staged zip before the install
|
||||
// dialog: what it is, how it relates to the installed add-on, and the digest
|
||||
// the approval binds to. It carries no path and no credential.
|
||||
type FileView struct {
|
||||
FileID string `json:"fileId"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
// SHA256 is the digest of the whole file, in hex. The install approval
|
||||
// binds to "sha256:" plus this value.
|
||||
SHA256 string `json:"sha256"`
|
||||
AddonID string `json:"addonId"`
|
||||
Title string `json:"title"`
|
||||
Version string `json:"version"`
|
||||
Platform string `json:"platform"`
|
||||
// API is the host API range the manifest declares.
|
||||
API contract.VersionRange `json:"api"`
|
||||
// Required and Optional are the host operations the manifest asks for.
|
||||
Required []string `json:"required"`
|
||||
Optional []string `json:"optional"`
|
||||
Installed bool `json:"installed"`
|
||||
InstalledVersion string `json:"installedVersion,omitempty"`
|
||||
// Relation is new, upgrade, same or downgrade.
|
||||
Relation string `json:"relation"`
|
||||
// Source and Origin say that nothing signs the file.
|
||||
Source string `json:"source"`
|
||||
Origin string `json:"origin"`
|
||||
}
|
||||
|
||||
// FileBegin opens an upload. It returns the ID of the upload and the byte limit.
|
||||
func (m *Manager) FileBegin(ctx context.Context, actor contract.Actor, name string) (string, int64, error) {
|
||||
if e := m.authorize(ctx, actor); e != nil {
|
||||
return "", 0, e
|
||||
}
|
||||
store := m.uploadStore()
|
||||
if store == nil || m.installer == nil {
|
||||
return "", 0, ErrNoFileInstall
|
||||
}
|
||||
max, _ := m.installer.Limits()
|
||||
id, e := store.Begin(actor.UserID, packages.SafeFileName(name), max)
|
||||
return id, max, e
|
||||
}
|
||||
|
||||
// FileChunk adds the next chunk to an upload.
|
||||
func (m *Manager) FileChunk(ctx context.Context, actor contract.Actor, id string, offset int64, data []byte) error {
|
||||
if e := m.authorize(ctx, actor); e != nil {
|
||||
return e
|
||||
}
|
||||
store := m.uploadStore()
|
||||
if store == nil {
|
||||
return ErrNoFileInstall
|
||||
}
|
||||
return store.Append(id, actor.UserID, offset, data)
|
||||
}
|
||||
|
||||
// FileAbort drops an upload that did not finish.
|
||||
func (m *Manager) FileAbort(ctx context.Context, actor contract.Actor, id string) {
|
||||
if store := m.uploadStore(); store != nil && m.authorize(ctx, actor) == nil {
|
||||
store.Abort(id, actor.UserID)
|
||||
}
|
||||
}
|
||||
|
||||
// FileFinish closes the upload and verifies the file as the install will: the
|
||||
// staged package is built and thrown away. A file that fails is removed.
|
||||
func (m *Manager) FileFinish(ctx context.Context, actor contract.Actor, id string) (FileView, error) {
|
||||
if e := m.authorize(ctx, actor); e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
store := m.uploadStore()
|
||||
if store == nil {
|
||||
return FileView{}, ErrNoFileInstall
|
||||
}
|
||||
entry, e := store.Finish(id, actor.UserID)
|
||||
if e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
v, e := m.describeStaged(ctx, actor, entry)
|
||||
if e != nil {
|
||||
_ = store.Remove(entry.ID, actor.UserID)
|
||||
return FileView{}, e
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// FileDescribe reads a staged file of the user again.
|
||||
func (m *Manager) FileDescribe(ctx context.Context, actor contract.Actor, id string) (FileView, error) {
|
||||
if e := m.authorize(ctx, actor); e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
store := m.uploadStore()
|
||||
if store == nil {
|
||||
return FileView{}, ErrNoFileInstall
|
||||
}
|
||||
_, entry, e := store.Open(id, actor.UserID)
|
||||
if e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
return m.describeStaged(ctx, actor, entry)
|
||||
}
|
||||
|
||||
// FileDrop removes a staged file of the user.
|
||||
func (m *Manager) FileDrop(ctx context.Context, actor contract.Actor, id string) error {
|
||||
if e := m.authorize(ctx, actor); e != nil {
|
||||
return e
|
||||
}
|
||||
store := m.uploadStore()
|
||||
if store == nil {
|
||||
return ErrNoFileInstall
|
||||
}
|
||||
return store.Remove(id, actor.UserID)
|
||||
}
|
||||
|
||||
func (m *Manager) describeStaged(ctx context.Context, actor contract.Actor, entry uploads.Entry) (FileView, error) {
|
||||
staged, e := m.stageUpload(ctx, actor, entry.ID)
|
||||
if e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
rc := staged.Receipt()
|
||||
if e = m.installer.Discard(staged); e != nil {
|
||||
return FileView{}, e
|
||||
}
|
||||
m.mu.Lock()
|
||||
st := m.journal.Addons[rc.Manifest.ID]
|
||||
m.mu.Unlock()
|
||||
v := FileView{FileID: entry.ID, Name: entry.Name, Size: entry.Size, SHA256: entry.SHA256, AddonID: rc.Manifest.ID, Title: rc.Manifest.ID,
|
||||
Version: rc.Manifest.Version, Platform: rc.Manifest.OS + "/" + rc.Manifest.Arch, API: rc.Manifest.API,
|
||||
Source: zipformat.SourceID, Origin: zipformat.OriginLabel, Relation: FileNew}
|
||||
if u := rc.Manifest.Capabilities.UI; u != nil && u.Title != nil && u.Title.Default() != "" {
|
||||
v.Title = u.Title.Default()
|
||||
}
|
||||
v.Required, v.Optional = declared(rc)
|
||||
if v.Required == nil {
|
||||
v.Required = []string{}
|
||||
}
|
||||
if v.Optional == nil {
|
||||
v.Optional = []string{}
|
||||
}
|
||||
if cur, ok := currentReceipt(st); ok && st.Installed {
|
||||
v.Installed, v.InstalledVersion = true, cur.Manifest.Version
|
||||
v.Relation = FileRelation(true, cur.Manifest.Version, rc.Manifest.Version)
|
||||
}
|
||||
return v, nil
|
||||
}
|
||||
|
||||
// IsFileRefusal reports whether e is a refusal of the content of a file.
|
||||
func IsFileRefusal(e error) (*packages.FileError, bool) {
|
||||
var fe *packages.FileError
|
||||
return fe, errors.As(e, &fe)
|
||||
}
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/addons/catalog"
|
||||
"github.com/mudler/LocalAI/core/services/addons/packages"
|
||||
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
)
|
||||
|
||||
// OfficialReceipt reports whether a package came from the official add-on
|
||||
@@ -137,6 +138,10 @@ func (m *Manager) PreviewRelease(ctx context.Context, sourceID, addonID, version
|
||||
}
|
||||
installed := m.journal.Addons[addonID].Installed
|
||||
m.mu.Unlock()
|
||||
if sourceID == zipformat.SourceID {
|
||||
// A local file: the version field names the staged file.
|
||||
return m.previewFile(ctx, version)
|
||||
}
|
||||
r, e := m.resolveRelease(ctx, LifecycleCommand{SourceID: sourceID, AddonID: addonID, Version: version})
|
||||
if e != nil {
|
||||
return HostAccessRelease{}, e
|
||||
|
||||
@@ -27,7 +27,9 @@ import (
|
||||
"github.com/mudler/LocalAI/core/services/addons/recovery"
|
||||
addonruntime "github.com/mudler/LocalAI/core/services/addons/runtime"
|
||||
"github.com/mudler/LocalAI/core/services/addons/updates"
|
||||
"github.com/mudler/LocalAI/core/services/addons/uploads"
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -75,6 +77,14 @@ const (
|
||||
CausePackageFetch = "package_fetch_failed"
|
||||
// CauseInstallStorage: the verified package could not be written to disk.
|
||||
CauseInstallStorage = "install_storage_failed"
|
||||
// CauseFileMissing: the staged file of a local file install is gone, has
|
||||
// expired or belongs to another user.
|
||||
CauseFileMissing = "staged_file_missing"
|
||||
// CauseVersionNotNewer: the file holds the version that is installed.
|
||||
CauseVersionNotNewer = "version_not_newer"
|
||||
// CauseDowngradeRefused: the file holds an older version and the request
|
||||
// did not confirm the downgrade.
|
||||
CauseDowngradeRefused = "downgrade_refused"
|
||||
)
|
||||
|
||||
// FailureCause is the typed cause of a failed operation. Its text is fixed and
|
||||
@@ -177,6 +187,9 @@ type Manager struct {
|
||||
// identities maps an installed add-on to its trusted identity; see
|
||||
// TrustedIdentity.
|
||||
identities map[string]string
|
||||
// uploads holds the zip files waiting for an install. Nil means the owner
|
||||
// does not install from a file.
|
||||
uploads atomic.Pointer[uploads.Store]
|
||||
}
|
||||
|
||||
// SetAuthority sets the Authority that admission and the apps list ask. The
|
||||
@@ -614,23 +627,47 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl
|
||||
return op.Status, e
|
||||
}
|
||||
m.mu.Unlock()
|
||||
if release != nil {
|
||||
r = *release
|
||||
} else {
|
||||
r, resolveErr = m.resolveRelease(ctx, cmd)
|
||||
}
|
||||
fromFile := cmd.SourceID == zipformat.SourceID
|
||||
var staged packages.StagedPackage
|
||||
var stageErr error
|
||||
if resolveErr == nil {
|
||||
staged, stageErr = m.installer.Stage(ctx, r, nil)
|
||||
if fromFile {
|
||||
// A local file has no library entry: the staged file is the
|
||||
// release, and the same Stage checks apply to its contents.
|
||||
staged, stageErr = m.stageUpload(ctx, actor, cmd.Version)
|
||||
} else {
|
||||
if release != nil {
|
||||
r = *release
|
||||
} else {
|
||||
r, resolveErr = m.resolveRelease(ctx, cmd)
|
||||
}
|
||||
if resolveErr == nil {
|
||||
staged, stageErr = m.installer.Stage(ctx, r, nil)
|
||||
}
|
||||
}
|
||||
m.mu.Lock()
|
||||
if resolveErr != nil {
|
||||
return fail(resolveErr, &FailureCause{Code: resolveCauseCode(resolveErr), detail: resolveErr})
|
||||
}
|
||||
if stageErr != nil {
|
||||
if errors.Is(stageErr, uploads.ErrNotFound) {
|
||||
return fail(stageErr, &FailureCause{Code: CauseFileMissing, detail: stageErr})
|
||||
}
|
||||
return fail(stageErr, &FailureCause{Code: CausePackageFetch, detail: stageErr})
|
||||
}
|
||||
if fromFile {
|
||||
if cause := m.fileVersionRefusal(cmd, next, staged.Receipt()); cause != nil {
|
||||
if discardErr := m.installer.Discard(staged); discardErr != nil {
|
||||
xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr)
|
||||
}
|
||||
return fail(cause)
|
||||
}
|
||||
if staged.Receipt().Manifest.ID != cmd.AddonID {
|
||||
if discardErr := m.installer.Discard(staged); discardErr != nil {
|
||||
xlog.Warn("Could not discard refused addon stage", "addon", cmd.AddonID, "error", discardErr)
|
||||
}
|
||||
return fail(packages.ErrPackage)
|
||||
}
|
||||
}
|
||||
if m.uncertain || m.runtimeRecoveryRequired {
|
||||
return op.Status, ErrRecoveryRequired
|
||||
}
|
||||
@@ -764,6 +801,10 @@ func (m *Manager) submit(ctx context.Context, actor contract.Actor, cmd Lifecycl
|
||||
case "install", "upgrade", "uninstall":
|
||||
m.syncSkills(ctx, cmd.AddonID)
|
||||
}
|
||||
if cmd.SourceID == zipformat.SourceID && (cmd.Action == "install" || cmd.Action == "upgrade") {
|
||||
// The package is installed; the staged file has done its work.
|
||||
m.dropUpload(actor, cmd.Version)
|
||||
}
|
||||
return result, m.syncFor(ctx, cmd.AddonID)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,210 @@
|
||||
//go:build linux
|
||||
|
||||
// SPDX-License-Identifier: MIT
|
||||
package addons
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/addons/packages"
|
||||
"github.com/mudler/LocalAI/core/services/addons/uploads"
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
. "github.com/onsi/ginkgo/v2"
|
||||
. "github.com/onsi/gomega"
|
||||
)
|
||||
|
||||
var _ = Describe("install from a local zip file", func() {
|
||||
var r *updateRig
|
||||
var base map[string]any
|
||||
BeforeEach(func() {
|
||||
r = newUpdateRig(false)
|
||||
r.m.SetUploads(&uploads.Store{Root: filepath.Join(r.cfg.Root, "uploads")})
|
||||
raw, e := os.ReadFile("../../../tests/addons/fixtures/packages/manifest.json")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(json.Unmarshal(raw, &base)).To(Succeed())
|
||||
})
|
||||
// zipFor builds the zip of a sample add-on of the given version that asks
|
||||
// for the given required host operations.
|
||||
zipFor := func(version, osName string, required ...string) []byte {
|
||||
m := map[string]any{}
|
||||
for k, v := range base {
|
||||
m[k] = v
|
||||
}
|
||||
m["version"], m["os"] = version, osName
|
||||
if len(required) > 0 {
|
||||
m["capabilities"] = map[string]any{"service": true, "hostOperations": map[string]any{"required": required}}
|
||||
}
|
||||
mb, e := json.Marshal(m)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
var b bytes.Buffer
|
||||
Expect(zipformat.Write(&b, "sample", version, osName, "amd64", "bin/sample", []zipformat.Entry{
|
||||
{Name: "manifest.json", Data: mb}, {Name: "bin/sample", Data: []byte("executable " + version), Exec: true},
|
||||
})).To(Succeed())
|
||||
return b.Bytes()
|
||||
}
|
||||
upload := func(user string, data []byte) (FileView, error) {
|
||||
a := contract.Actor{UserID: user, CredentialID: "key", Mode: "named-user"}
|
||||
id, _, e := r.m.FileBegin(ctx, a, "sample.zip")
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
for off := 0; off < len(data); off += 1000 {
|
||||
end := min(off+1000, len(data))
|
||||
Expect(r.m.FileChunk(ctx, a, id, int64(off), data[off:end])).To(Succeed())
|
||||
}
|
||||
return r.m.FileFinish(ctx, a, id)
|
||||
}
|
||||
fileCmd := func(op, action string, v FileView) LifecycleCommand {
|
||||
return LifecycleCommand{OperationID: op, AddonID: "sample", Action: action, SourceID: zipformat.SourceID, Version: v.FileID}
|
||||
}
|
||||
|
||||
It("installs through the manager and records an unsigned local file", func() {
|
||||
v, e := upload("admin", zipFor("2.0.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(v.Relation).To(Equal(FileNew))
|
||||
Expect(v.Origin).To(Equal("local file, unsigned"))
|
||||
Expect(v.Version).To(Equal("2.0.0"))
|
||||
Expect(v.API.Min).To(Equal(contract.HostAPIVersion))
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("f1", "install", v))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
st, _ := r.m.Status(ctx, "sample")
|
||||
Expect(st.Installed).To(BeTrue())
|
||||
rc := st.Generations[0].Receipt
|
||||
Expect(rc.SourceID).To(Equal("local-file"))
|
||||
Expect(rc.Origin).To(Equal("local file, unsigned"))
|
||||
Expect(rc.Verification).To(Equal("sha256"))
|
||||
Expect(rc.Digest).To(Equal("sha256:" + v.SHA256))
|
||||
// The staged file is gone once it is installed.
|
||||
_, e = r.m.FileDescribe(ctx, actor, v.FileID)
|
||||
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
|
||||
// A package from a file has no library to ask for updates.
|
||||
list, e := r.m.Installed(ctx, actor)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(list).To(HaveLen(1))
|
||||
Expect(list[0].Update).To(BeNil())
|
||||
Expect(list[0].SourceID).To(Equal("local-file"))
|
||||
})
|
||||
|
||||
It("keeps a file for the user that uploaded it", func() {
|
||||
v, e := upload("admin", zipFor("2.0.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
other := contract.Actor{UserID: "admin-two", CredentialID: "key", Mode: "named-user"}
|
||||
_, e = r.m.stageUpload(ctx, other, v.FileID)
|
||||
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
|
||||
// A command of another user is refused by the administrator check
|
||||
// before it can reach the file.
|
||||
_, e = r.m.Submit(ctx, other, fileCmd("f1", "install", v))
|
||||
Expect(e).To(MatchError(ErrForbidden))
|
||||
st, _ := r.m.Status(ctx, "sample")
|
||||
Expect(st.Installed).To(BeFalse())
|
||||
})
|
||||
|
||||
It("refuses a file for another platform when it is uploaded, and leaves nothing staged", func() {
|
||||
_, e := upload("admin", zipFor("2.0.0", "darwin"))
|
||||
var fe *packages.FileError
|
||||
Expect(errors.As(e, &fe)).To(BeTrue())
|
||||
Expect(fe.Code).To(Equal(packages.FileWrongPlatform))
|
||||
ents, _ := os.ReadDir(filepath.Join(r.cfg.Root, "uploads"))
|
||||
Expect(ents).To(BeEmpty())
|
||||
})
|
||||
|
||||
It("refuses an install when the administrator is not one", func() {
|
||||
_, _, e := r.m.FileBegin(ctx, contract.Actor{UserID: "user", Mode: "named-user"}, "x.zip")
|
||||
Expect(e).To(MatchError(ErrForbidden))
|
||||
})
|
||||
|
||||
Describe("versions", func() {
|
||||
install := func(version string) FileView {
|
||||
v, e := upload("admin", zipFor(version, "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("i-"+version, "install", v))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
return v
|
||||
}
|
||||
BeforeEach(func() { install("2.0.0") })
|
||||
|
||||
It("upgrades to a higher version through the same generation machinery", func() {
|
||||
v, e := upload("admin", zipFor("2.1.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(v.Relation).To(Equal(FileUpgrade))
|
||||
Expect(v.InstalledVersion).To(Equal("2.0.0"))
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
st, _ := r.m.Status(ctx, "sample")
|
||||
Expect(st.Generation).To(Equal(uint64(2)))
|
||||
Expect(st.Generations[1].Receipt.Manifest.Version).To(Equal("2.1.0"))
|
||||
})
|
||||
|
||||
It("refuses the same version", func() {
|
||||
v, e := upload("admin", zipFor("2.0.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(v.Relation).To(Equal(FileSame))
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("u1", "upgrade", v))
|
||||
Expect(e).To(MatchError(ErrOperationFailed))
|
||||
Expect(e.Error()).To(ContainSubstring(CauseVersionNotNewer))
|
||||
st, _ := r.m.Status(ctx, "sample")
|
||||
Expect(st.Generation).To(Equal(uint64(1)))
|
||||
})
|
||||
|
||||
It("refuses a lower version unless the downgrade is confirmed", func() {
|
||||
v, e := upload("admin", zipFor("1.5.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(v.Relation).To(Equal(FileDowngrade))
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("d1", "upgrade", v))
|
||||
Expect(e).To(MatchError(ErrOperationFailed))
|
||||
Expect(e.Error()).To(ContainSubstring(CauseDowngradeRefused))
|
||||
st, _ := r.m.Status(ctx, "sample")
|
||||
Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("2.0.0"))
|
||||
// The file stays staged for a retry with the confirmation.
|
||||
cmd := fileCmd("d2", "upgrade", v)
|
||||
cmd.Downgrade = true
|
||||
_, e = r.m.Submit(ctx, actor, cmd)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
st, _ = r.m.Status(ctx, "sample")
|
||||
Expect(st.Generations[st.Generation-1].Receipt.Manifest.Version).To(Equal("1.5.0"))
|
||||
})
|
||||
})
|
||||
|
||||
Describe("host access approval", func() {
|
||||
It("grants only what the administrator approved for the digest of the file", func() {
|
||||
g := newGrantSim(r.m)
|
||||
v, e := upload("admin", zipFor("2.0.0", "linux", "workspace.materialize"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(v.Required).To(Equal([]string{"workspace.materialize"}))
|
||||
_, e = r.m.Submit(ctx, actor, fileCmd("f0", "install", v))
|
||||
var refusal *HostRequirementsRefusal
|
||||
Expect(errors.As(e, &refusal)).To(BeTrue(), "an unapproved required operation is refused")
|
||||
// Previews give the digest the dialog shows.
|
||||
rel, e := r.m.PreviewRelease(WithFileOwner(ctx, "admin"), zipformat.SourceID, "sample", v.FileID)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(rel.Digest).To(Equal("sha256:" + v.SHA256))
|
||||
Expect(rel.Official).To(BeFalse())
|
||||
Expect(rel.Required).To(Equal([]string{"workspace.materialize"}))
|
||||
cmd := fileCmd("f1", "install", v)
|
||||
cmd.Approval = &HostApproval{Digest: "sha256:" + strings.Repeat("0", 64)}
|
||||
_, e = r.m.Submit(ctx, actor, cmd)
|
||||
var stale *ApprovalStaleError
|
||||
Expect(errors.As(e, &stale)).To(BeTrue(), "another digest is stale")
|
||||
cmd = fileCmd("f2", "install", v)
|
||||
cmd.Approval = &HostApproval{Digest: rel.Digest, Ops: []string{"workspace.materialize", "egress.check"}, Revision: 3}
|
||||
_, e = r.m.Submit(ctx, actor, cmd)
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
Expect(g.stagedOps("f2")).To(Equal([]string{"workspace.materialize"}))
|
||||
Expect(r.m.OperationCompleted("f2")).To(BeTrue())
|
||||
})
|
||||
|
||||
It("previews only the files of the calling user", func() {
|
||||
v, e := upload("admin", zipFor("2.0.0", "linux"))
|
||||
Expect(e).NotTo(HaveOccurred())
|
||||
_, e = r.m.PreviewRelease(WithFileOwner(ctx, "someone"), zipformat.SourceID, "sample", v.FileID)
|
||||
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
|
||||
_, e = r.m.PreviewRelease(context.Background(), zipformat.SourceID, "sample", v.FileID)
|
||||
Expect(errors.Is(e, uploads.ErrNotFound)).To(BeTrue())
|
||||
})
|
||||
})
|
||||
})
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"github.com/mudler/LocalAI/pkg/addons/contract"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
)
|
||||
|
||||
@@ -55,6 +56,10 @@ const (
|
||||
|
||||
var fileNameRE = regexp.MustCompile(`[^A-Za-z0-9._-]`)
|
||||
|
||||
// Limits returns the byte and file caps of a package, the ones that apply to
|
||||
// a library package and to a local file alike.
|
||||
func (i *Installer) Limits() (bytes int64, files int) { return i.byteLimit(), i.fileLimit() }
|
||||
|
||||
// FileSummary is what a verified zip says about itself. It is computed without
|
||||
// writing the package to disk.
|
||||
type FileSummary struct {
|
||||
@@ -63,6 +68,9 @@ type FileSummary struct {
|
||||
Size int64 `json:"size"`
|
||||
Files int `json:"files"`
|
||||
zipformat.Manifest
|
||||
// Package is manifest.json of the add-on, parsed. InspectFile fills it; it
|
||||
// is not checked against a host there.
|
||||
Package contract.Manifest `json:"package"`
|
||||
}
|
||||
|
||||
// zipSink receives each file of the package after the path and type checks.
|
||||
@@ -223,10 +231,34 @@ func (c *countingReader) Read(b []byte) (int, error) {
|
||||
// installer; zero takes the defaults.
|
||||
func InspectFile(ctx context.Context, p string, maxBytes int64, maxFiles int) (FileSummary, error) {
|
||||
i := &Installer{MaxBytes: maxBytes, MaxFiles: maxFiles}
|
||||
return readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(_ string, _ bool, r io.Reader) error {
|
||||
_, e := io.Copy(io.Discard, r)
|
||||
return e
|
||||
var pkg contract.Manifest
|
||||
sum, e := readZip(ctx, p, i.byteLimit(), i.fileLimit(), nil, func(n string, _ bool, r io.Reader) error {
|
||||
if n != "manifest.json" {
|
||||
_, e := io.Copy(io.Discard, r)
|
||||
return e
|
||||
}
|
||||
b, e := io.ReadAll(io.LimitReader(r, contract.DefaultLimits().ManifestBytes+1))
|
||||
if e != nil {
|
||||
return e
|
||||
}
|
||||
if int64(len(b)) > contract.DefaultLimits().ManifestBytes {
|
||||
return fileErr(FileBadManifest, "manifest.json is too large")
|
||||
}
|
||||
m, e := contract.ParseManifest(b)
|
||||
if e != nil {
|
||||
return fileErr(FileBadManifest, "manifest.json is not a valid add-on manifest")
|
||||
}
|
||||
pkg = m
|
||||
return nil
|
||||
})
|
||||
if e != nil {
|
||||
return sum, e
|
||||
}
|
||||
if pkg.ID == "" || pkg.ID != sum.ID || pkg.Version != sum.Version || pkg.OS != sum.OS || pkg.Arch != sum.Arch {
|
||||
return sum, fileErr(FileBadManifest, "The manifest of the add-on does not match the digest manifest of the zip")
|
||||
}
|
||||
sum.Package = pkg
|
||||
return sum, nil
|
||||
}
|
||||
|
||||
// SafeFileName reduces the name a client gave a file to characters that are
|
||||
|
||||
@@ -60,6 +60,9 @@ type LifecycleCommand struct {
|
||||
// registers. It carries no authority by itself: the caller must be a human
|
||||
// surface, which the gateway checks before it sets this field.
|
||||
Approval *HostApproval `json:",omitempty"`
|
||||
// Downgrade confirms that a local file may install an older version than
|
||||
// the one installed. Without it such a file is refused.
|
||||
Downgrade bool `json:",omitempty"`
|
||||
}
|
||||
|
||||
// HostApproval binds an approval to one staged package and one revision of the
|
||||
|
||||
@@ -4,6 +4,7 @@ package addons
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"github.com/mudler/LocalAI/pkg/addons/zipformat"
|
||||
"time"
|
||||
|
||||
"github.com/mudler/LocalAI/core/services/addons/apps"
|
||||
@@ -55,7 +56,9 @@ func (r *updateReader) Installation(ctx context.Context, id string) (updates.Ins
|
||||
return updates.Installation{}, false, nil
|
||||
}
|
||||
receipt, ok := currentReceipt(st)
|
||||
if !ok {
|
||||
if !ok || receipt.SourceID == zipformat.SourceID {
|
||||
// A package from a local file has no library to ask, so no check can
|
||||
// answer for it. The page reads that as unknown, never as current.
|
||||
return updates.Installation{}, false, nil
|
||||
}
|
||||
return updates.Installation{AddonID: id, Generation: st.Generation, Receipt: receipt}, true, nil
|
||||
|
||||
@@ -0,0 +1,332 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
|
||||
// Package uploads is the owner's staging area for add-on zip files. A file
|
||||
// waits here between the upload and the install the administrator confirms.
|
||||
// It is private to the owner, bound to the user that uploaded it, and
|
||||
// removed after a short time, after the install and at start.
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
var (
|
||||
// ErrNotFound: no such file, or it belongs to another user, or it expired.
|
||||
ErrNotFound = errors.New("staged file not found")
|
||||
// ErrTooLarge: the upload went over the byte limit.
|
||||
ErrTooLarge = errors.New("file is over the size limit")
|
||||
// ErrBusy: too many files are waiting.
|
||||
ErrBusy = errors.New("too many staged files")
|
||||
// ErrOffset: a chunk did not follow the bytes received so far.
|
||||
ErrOffset = errors.New("chunk out of order")
|
||||
)
|
||||
|
||||
// Defaults of a store.
|
||||
const (
|
||||
DefaultTTL = time.Hour
|
||||
DefaultMaxFiles = 16
|
||||
)
|
||||
|
||||
var idRE = regexp.MustCompile(`^[0-9a-f]{32}$`)
|
||||
|
||||
// ValidID reports whether s has the shape of a staged file ID.
|
||||
func ValidID(s string) bool { return idRE.MatchString(s) }
|
||||
|
||||
// Entry is a staged file.
|
||||
type Entry struct {
|
||||
ID string `json:"id"`
|
||||
Owner string `json:"owner"`
|
||||
Name string `json:"name"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Created time.Time `json:"created"`
|
||||
}
|
||||
|
||||
// Store keeps staged files under Root.
|
||||
type Store struct {
|
||||
Root string
|
||||
TTL time.Duration
|
||||
MaxFiles int
|
||||
// Now is replaced by tests.
|
||||
Now func() time.Time
|
||||
|
||||
mu sync.Mutex
|
||||
open map[string]*upload
|
||||
}
|
||||
|
||||
type upload struct {
|
||||
owner, name string
|
||||
f *os.File
|
||||
h hash.Hash
|
||||
n int64
|
||||
max int64
|
||||
started time.Time
|
||||
}
|
||||
|
||||
func (s *Store) now() time.Time {
|
||||
if s.Now != nil {
|
||||
return s.Now()
|
||||
}
|
||||
return time.Now()
|
||||
}
|
||||
|
||||
func (s *Store) ttl() time.Duration {
|
||||
if s.TTL > 0 {
|
||||
return s.TTL
|
||||
}
|
||||
return DefaultTTL
|
||||
}
|
||||
|
||||
func (s *Store) maxFiles() int {
|
||||
if s.MaxFiles > 0 {
|
||||
return s.MaxFiles
|
||||
}
|
||||
return DefaultMaxFiles
|
||||
}
|
||||
|
||||
func (s *Store) dir() (string, error) {
|
||||
if s.Root == "" || !filepath.IsAbs(s.Root) {
|
||||
return "", errors.New("uploads: root must be an absolute path")
|
||||
}
|
||||
if err := os.MkdirAll(s.Root, 0o700); err != nil {
|
||||
return "", err
|
||||
}
|
||||
info, err := os.Lstat(s.Root)
|
||||
if err != nil || !info.IsDir() || info.Mode()&os.ModeSymlink != 0 || info.Mode().Perm()&0o077 != 0 {
|
||||
return "", errors.New("uploads: root must be a private directory")
|
||||
}
|
||||
return s.Root, nil
|
||||
}
|
||||
|
||||
func newID() (string, error) {
|
||||
var b [16]byte
|
||||
if _, err := rand.Read(b[:]); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return hex.EncodeToString(b[:]), nil
|
||||
}
|
||||
|
||||
func (s *Store) zipPath(id string) string { return filepath.Join(s.Root, id+".zip") }
|
||||
func (s *Store) metaPath(id string) string { return filepath.Join(s.Root, id+".json") }
|
||||
|
||||
// Begin opens a new upload for owner. max is the byte limit of the file.
|
||||
func (s *Store) Begin(owner, name string, max int64) (string, error) {
|
||||
if owner == "" || max <= 0 {
|
||||
return "", ErrNotFound
|
||||
}
|
||||
if _, err := s.dir(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
s.Sweep()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if s.open == nil {
|
||||
s.open = map[string]*upload{}
|
||||
}
|
||||
staged, _ := filepath.Glob(filepath.Join(s.Root, "*.zip"))
|
||||
if len(staged)+len(s.open) >= s.maxFiles() {
|
||||
return "", ErrBusy
|
||||
}
|
||||
id, err := newID()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
f, err := os.OpenFile(s.zipPath(id), os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0o600) // #nosec G304 -- the name is a random ID under the private staging root
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
s.open[id] = &upload{owner: owner, name: name, f: f, h: sha256.New(), max: max, started: s.now()}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
// Append adds the next chunk. offset is the number of bytes received before it.
|
||||
func (s *Store) Append(id, owner string, offset int64, p []byte) error {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
u, ok := s.open[id]
|
||||
if !ok || u.owner != owner {
|
||||
return ErrNotFound
|
||||
}
|
||||
if offset != u.n {
|
||||
return ErrOffset
|
||||
}
|
||||
if int64(len(p)) > u.max-u.n {
|
||||
s.abortLocked(id)
|
||||
return ErrTooLarge
|
||||
}
|
||||
if _, err := u.f.Write(p); err != nil {
|
||||
s.abortLocked(id)
|
||||
return err
|
||||
}
|
||||
_, _ = u.h.Write(p)
|
||||
u.n += int64(len(p))
|
||||
return nil
|
||||
}
|
||||
|
||||
// Finish closes the upload and makes it a staged file.
|
||||
func (s *Store) Finish(id, owner string) (Entry, error) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
u, ok := s.open[id]
|
||||
if !ok || u.owner != owner {
|
||||
return Entry{}, ErrNotFound
|
||||
}
|
||||
delete(s.open, id)
|
||||
e := Entry{ID: id, Owner: owner, Name: u.name, Size: u.n, SHA256: hex.EncodeToString(u.h.Sum(nil)), Created: s.now().UTC()}
|
||||
err := u.f.Sync()
|
||||
if ce := u.f.Close(); err == nil {
|
||||
err = ce
|
||||
}
|
||||
if err == nil {
|
||||
var b []byte
|
||||
if b, err = json.Marshal(e); err == nil {
|
||||
err = os.WriteFile(s.metaPath(id), b, 0o600)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
_ = os.Remove(s.zipPath(id))
|
||||
return Entry{}, err
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
// Abort drops an upload that is not finished. It is safe to call twice.
|
||||
func (s *Store) Abort(id, owner string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if u, ok := s.open[id]; ok && u.owner == owner {
|
||||
s.abortLocked(id)
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Store) abortLocked(id string) {
|
||||
if u, ok := s.open[id]; ok {
|
||||
_ = u.f.Close()
|
||||
delete(s.open, id)
|
||||
}
|
||||
_ = os.Remove(s.zipPath(id))
|
||||
}
|
||||
|
||||
// Put stores the whole of r as a staged file of owner, up to max bytes.
|
||||
func (s *Store) Put(owner, name string, max int64, r io.Reader) (Entry, error) {
|
||||
id, err := s.Begin(owner, name, max)
|
||||
if err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
buf := make([]byte, 256<<10)
|
||||
var off int64
|
||||
for {
|
||||
n, rerr := r.Read(buf)
|
||||
if n > 0 {
|
||||
if err = s.Append(id, owner, off, buf[:n]); err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
off += int64(n)
|
||||
}
|
||||
if rerr == io.EOF {
|
||||
break
|
||||
}
|
||||
if rerr != nil {
|
||||
s.Abort(id, owner)
|
||||
return Entry{}, rerr
|
||||
}
|
||||
}
|
||||
return s.Finish(id, owner)
|
||||
}
|
||||
|
||||
// Open returns the path and the entry of a staged file of owner.
|
||||
func (s *Store) Open(id, owner string) (string, Entry, error) {
|
||||
if !ValidID(id) || owner == "" {
|
||||
return "", Entry{}, ErrNotFound
|
||||
}
|
||||
e, err := s.read(id)
|
||||
if err != nil || e.Owner != owner || s.expired(e) {
|
||||
return "", Entry{}, ErrNotFound
|
||||
}
|
||||
return s.zipPath(id), e, nil
|
||||
}
|
||||
|
||||
func (s *Store) read(id string) (Entry, error) {
|
||||
var e Entry
|
||||
b, err := os.ReadFile(s.metaPath(id)) // #nosec G304 -- the name is a validated random ID under the private staging root
|
||||
if err != nil {
|
||||
return e, ErrNotFound
|
||||
}
|
||||
if json.Unmarshal(b, &e) != nil || e.ID != id {
|
||||
return Entry{}, ErrNotFound
|
||||
}
|
||||
return e, nil
|
||||
}
|
||||
|
||||
func (s *Store) expired(e Entry) bool { return s.now().Sub(e.Created) > s.ttl() }
|
||||
|
||||
// Remove deletes a staged file of owner.
|
||||
func (s *Store) Remove(id, owner string) error {
|
||||
if _, _, err := s.Open(id, owner); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.removeFiles(id)
|
||||
}
|
||||
|
||||
func (s *Store) removeFiles(id string) error {
|
||||
err := os.Remove(s.zipPath(id))
|
||||
_ = os.Remove(s.metaPath(id))
|
||||
if err != nil && !os.IsNotExist(err) {
|
||||
return fmt.Errorf("remove staged file: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// Sweep removes the files that are older than the time to live and the
|
||||
// uploads that never finished. With purge set it removes everything that is
|
||||
// not an open upload, as the owner does at start.
|
||||
func (s *Store) Sweep() { s.sweep(false) }
|
||||
|
||||
// Purge removes every staged file. Call it once at start.
|
||||
func (s *Store) Purge() { s.sweep(true) }
|
||||
|
||||
func (s *Store) sweep(all bool) {
|
||||
if _, err := s.dir(); err != nil {
|
||||
return
|
||||
}
|
||||
// The lock covers the whole sweep, so a Finish in progress is never seen
|
||||
// half done.
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
for id, u := range s.open {
|
||||
if all || s.now().Sub(u.started) > s.ttl() {
|
||||
s.abortLocked(id)
|
||||
}
|
||||
}
|
||||
open := map[string]bool{}
|
||||
for id := range s.open {
|
||||
open[id] = true
|
||||
}
|
||||
ents, err := os.ReadDir(s.Root)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, de := range ents {
|
||||
name := de.Name()
|
||||
id := name[:len(name)-len(filepath.Ext(name))]
|
||||
if !ValidID(id) || open[id] {
|
||||
continue
|
||||
}
|
||||
e, rerr := s.read(id)
|
||||
if all || rerr != nil || s.expired(e) {
|
||||
_ = s.removeFiles(id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
// SPDX-License-Identifier: MIT
|
||||
package uploads
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func newStore(t *testing.T) (*Store, *time.Time) {
|
||||
t.Helper()
|
||||
now := time.Date(2026, 11, 3, 10, 0, 0, 0, time.UTC)
|
||||
root := filepath.Join(t.TempDir(), "uploads")
|
||||
return &Store{Root: root, Now: func() time.Time { return now }, MaxFiles: 2}, &now
|
||||
}
|
||||
|
||||
func TestPutOpenAndOwner(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
e, err := s.Put("alice", "x.zip", 1<<20, strings.NewReader("hello"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if e.Size != 5 || len(e.SHA256) != 64 || !ValidID(e.ID) {
|
||||
t.Fatalf("entry %+v", e)
|
||||
}
|
||||
p, got, err := s.Open(e.ID, "alice")
|
||||
if err != nil || got.ID != e.ID {
|
||||
t.Fatalf("open: %v", err)
|
||||
}
|
||||
if b, _ := os.ReadFile(p); string(b) != "hello" {
|
||||
t.Fatalf("content %q", b)
|
||||
}
|
||||
if _, _, err = s.Open(e.ID, "bob"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("another user must not open it: %v", err)
|
||||
}
|
||||
if _, _, err = s.Open("../etc/passwd", "alice"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("a path is not an ID: %v", err)
|
||||
}
|
||||
info, _ := os.Stat(p)
|
||||
if info.Mode().Perm() != 0o600 {
|
||||
t.Fatalf("mode %v", info.Mode())
|
||||
}
|
||||
if err = s.Remove(e.ID, "bob"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("another user removed it")
|
||||
}
|
||||
if err = s.Remove(e.ID, "alice"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, _, err = s.Open(e.ID, "alice"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("still there")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSizeLimitStopsTheUpload(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
_, err := s.Put("alice", "x.zip", 10, bytes.NewReader(make([]byte, 11)))
|
||||
if !errors.Is(err, ErrTooLarge) {
|
||||
t.Fatalf("got %v", err)
|
||||
}
|
||||
ents, _ := os.ReadDir(s.Root)
|
||||
if len(ents) != 0 {
|
||||
t.Fatalf("left files: %v", ents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestChunksMustFollowEachOther(t *testing.T) {
|
||||
s, _ := newStore(t)
|
||||
id, err := s.Begin("alice", "x.zip", 100)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = s.Append(id, "alice", 0, []byte("ab")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err = s.Append(id, "alice", 5, []byte("cd")); !errors.Is(err, ErrOffset) {
|
||||
t.Fatalf("gap: %v", err)
|
||||
}
|
||||
if err = s.Append(id, "bob", 2, []byte("cd")); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("other user: %v", err)
|
||||
}
|
||||
if _, err = s.Finish(id, "bob"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatalf("other user finish: %v", err)
|
||||
}
|
||||
e, err := s.Finish(id, "alice")
|
||||
if err != nil || e.Size != 2 {
|
||||
t.Fatalf("finish: %v %+v", err, e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExpiryBusyAndPurge(t *testing.T) {
|
||||
s, now := newStore(t)
|
||||
a, _ := s.Put("alice", "a.zip", 100, strings.NewReader("a"))
|
||||
if _, err := s.Put("alice", "b.zip", 100, strings.NewReader("b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); !errors.Is(err, ErrBusy) {
|
||||
t.Fatalf("limit: %v", err)
|
||||
}
|
||||
*now = now.Add(2 * time.Hour)
|
||||
if _, _, err := s.Open(a.ID, "alice"); !errors.Is(err, ErrNotFound) {
|
||||
t.Fatal("an expired file opens")
|
||||
}
|
||||
if _, err := s.Put("alice", "c.zip", 100, strings.NewReader("c")); err != nil {
|
||||
t.Fatalf("sweep did not free room: %v", err)
|
||||
}
|
||||
s.Purge()
|
||||
ents, _ := os.ReadDir(s.Root)
|
||||
if len(ents) != 0 {
|
||||
t.Fatalf("purge left %v", ents)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRootMustBePrivate(t *testing.T) {
|
||||
root := filepath.Join(t.TempDir(), "open")
|
||||
if err := os.Mkdir(root, 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s := &Store{Root: root}
|
||||
if _, err := s.Put("alice", "x.zip", 10, strings.NewReader("x")); err == nil {
|
||||
t.Fatal("a world-readable root was accepted")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user