diff --git a/core/application/addon_gateway.go b/core/application/addon_gateway.go index 1f4d5e9ada..9562f07a5c 100644 --- a/core/application/addon_gateway.go +++ b/core/application/addon_gateway.go @@ -67,11 +67,7 @@ func (a *Application) AddonHostOps() *addonruntime.HostOps { return a.addonHostO func (a *Application) addonAuthority() hostops.Authority { a.addonAuthorityOnce.Do(func() { if a.hostAuthority == nil { - var floor map[string][]string - if a.applicationConfig != nil { - floor = a.applicationConfig.AddonHostOps - } - a.hostAuthority = hostops.NewStaticAuthority(floor) + a.hostAuthority = a.buildAddonAuthority() } }) return a.hostAuthority @@ -380,6 +376,9 @@ func (a *Application) startAddonGateway() error { if e := a.wireExecutionBinding(g); e != nil { return e } + if e := a.wireHostGrantsOwner(g); e != nil { + return e + } // Only the owner holds the policy store. Other frontends reach it through the // gateway, and without it the gateway keeps denying every resource. if !filepath.IsAbs(cfg.Root) || cfg.ListenAddress == "" { diff --git a/core/application/addon_hostgrants.go b/core/application/addon_hostgrants.go new file mode 100644 index 0000000000..f9c381f9c5 --- /dev/null +++ b/core/application/addon_hostgrants.go @@ -0,0 +1,619 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "context" + "errors" + "fmt" + "path/filepath" + "regexp" + "slices" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/mudler/LocalAI/core/services/addons" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/core/services/addons/hostgrants" + "github.com/mudler/LocalAI/core/services/addons/hostops" + "github.com/mudler/LocalAI/pkg/addons/contract" + "github.com/mudler/xlog" +) + +// File names of the owner's host grants, beside the host settings. +const ( + addonHostGrantsFile = "host-grants.json" + addonHostGrantsAudit = "host-grants-audit.jsonl" +) + +// errHostGrantsLocked is what a write answers when the deployment keeps the +// policy file and flags the only source of grants. +var errHostGrantsLocked = errors.New("host grants are locked by the deployment") + +// addonHostGrants is the owner's side of the UI host grants: the document, its +// audit file, the deployment floor and the glue to the add-on manager. Only +// the owner has one. A frontend never stores or enforces grants. +type addonHostGrants struct { + floor hostgrants.Floor + locked bool + override string + + store *hostgrants.Store + storeErr error + audit *hostgrants.Audit + + mu sync.RWMutex + manager *addons.Manager + + // refresh coalesces the calls that follow a grant write. + refreshMu sync.Mutex + refreshQueued bool + refreshRunning bool + warned atomic.Bool +} + +var supportedOps = func() map[string]bool { + m := map[string]bool{} + for _, op := range hostops.SupportedOperations() { + m[op] = true + } + return m +}() + +// hostGrantsFloor builds the floor of the deployment: the grants and deny +// rules of the policy file and flags, read once at startup. Empty prefixes are +// dropped, since an empty prefix would match every operation. +func hostGrantsFloor(grants, deny map[string][]string) hostgrants.Floor { + clean := func(in map[string][]string) map[string][]string { + out := map[string][]string{} + for id, list := range in { + for _, e := range list { + if e != "" { + out[id] = append(out[id], e) + } + } + } + return out + } + return hostgrants.Floor{Grants: clean(grants), Deny: clean(deny)} +} + +func (a *Application) hostGrantsFloorFromConfig() hostgrants.Floor { + if a.applicationConfig == nil { + return hostgrants.Floor{} + } + return hostGrantsFloor(a.applicationConfig.AddonHostOps, a.applicationConfig.AddonHostDeny) +} + +// openAddonHostGrants opens the document and the audit file under the owner's +// state directory. Call it on the owner only, after NewManager took the +// storage guard and before the Authority is first used. A missing document is +// the first start: it is not created until the first write. An unreadable one +// does not stop the owner; the Authority then refuses every operation, because +// the deny rules in it cannot be read. +func (a *Application) openAddonHostGrants(root string) { + cfg := a.applicationConfig + g := &addonHostGrants{floor: a.hostGrantsFloorFromConfig(), locked: cfg.AddonHostUIGrantsLocked, override: cfg.AddonHostBlanket} + g.store, g.storeErr = hostgrants.Open(filepath.Join(root, addonHostGrantsFile)) + if g.storeErr == nil { + var err error + if g.audit, err = hostgrants.OpenAudit(filepath.Join(root, addonHostGrantsAudit)); err != nil { + // Without the audit file nothing may be granted: the store is shut + // so the Authority fails closed, like for an unreadable document. + _ = g.store.Close() + g.store, g.storeErr = nil, fmt.Errorf("%w: audit file", hostgrants.ErrUnavailable) + } + } + a.addonGrants = g + g.logStartup(cfg.AddonHostOpsSource) + if g.storeErr == nil { + a.addonGrantStore = g + } +} + +func (g *addonHostGrants) logStartup(source string) { + if source == "" { + source = "empty" + } + prefixes, denyRules := 0, 0 + for _, l := range g.floor.Grants { + prefixes += len(l) + } + for _, l := range g.floor.Deny { + denyRules += len(l) + } + mode, byDeployment := hostgrants.BlanketOff, false + if g.override != "" { + mode, byDeployment = g.override, true + } + fields := []any{"floorSource", source, "floorAddons", len(g.floor.Grants), "floorPrefixes", prefixes, "floorDenyRules", denyRules, "uiGrants", map[bool]string{true: "locked", false: "enabled"}[g.locked]} + if g.storeErr != nil { + xlog.Warn("Addon host grants store is unreadable: no add-on can call a host operation until it is repaired", "reason", describeStoreError(g.storeErr)) + xlog.Info("Addon host grants", fields...) + return + } + snap, _ := g.store.Snapshot(context.Background()) + ops := 0 + for _, e := range snap.Grants { + ops += len(e.Ops) + } + if !byDeployment { + mode = snap.Blanket.Mode + } + uiDeny := 0 + for _, l := range snap.Deny { + uiDeny += len(l) + } + fields = append(fields, "revision", snap.Revision, "uiGrantAddons", len(snap.Grants), "uiGrantOperations", ops, "uiDenyRules", uiDeny, "blanket", mode, "blanketSetByDeployment", byDeployment) + xlog.Info("Addon host grants", fields...) + if g.locked && len(snap.Grants)+uiDeny > 0 { + xlog.Warn("Addon host grants are locked: the document keeps applying as it is, and no UI write changes it") + } +} + +// describeStoreError gives a reason that names no path. +func describeStoreError(err error) string { + switch { + case errors.Is(err, hostgrants.ErrInvalid): + return "the document is not valid" + case errors.Is(err, hostgrants.ErrUncertain): + return "the last write is in doubt" + case errors.Is(err, hostgrants.ErrUnavailable): + return "the document cannot be opened or locked" + } + return "unreadable" +} + +func (a *Application) closeAddonHostGrants() { + if g := a.addonGrants; g != nil { + if g.store != nil { + _ = g.store.Close() + } + _ = g.audit.Close() + } +} + +func (g *addonHostGrants) bind(m *addons.Manager) { + g.mu.Lock() + defer g.mu.Unlock() + g.manager = m +} + +func (g *addonHostGrants) mgr() *addons.Manager { + g.mu.RLock() + defer g.mu.RUnlock() + return g.manager +} + +// context is what the evaluator needs for one add-on. Until the manager is +// bound no identity is known, so no UI entry counts. +func (g *addonHostGrants) context(id string) hostgrants.Context { + c := hostgrants.Context{Registered: func(op string) bool { return supportedOps[op] }} + if m := g.mgr(); m != nil { + c.Identity, c.Completed = m.TrustedIdentity(id), m.OperationCompleted + } else { + c.Completed = func(string) bool { return false } + } + return c +} + +// failClosed wraps the Authority so that a store that cannot be read refuses +// every operation: the UI deny rules in it are unknown, and an empty view would +// let a denied operation through. +type failClosed struct { + inner hostops.Authority + g *addonHostGrants +} + +func (f failClosed) broken() bool { + if f.g.storeErr != nil || (f.g.store != nil && f.g.store.Usable() != nil) { + if f.g.warned.CompareAndSwap(false, true) { + xlog.Error("Addon host grants store is unreadable: every host operation is refused") + } + return true + } + return false +} + +func (f failClosed) Allows(id, op string) (bool, hostops.Reason) { + if id == "" || f.broken() { + return false, hostops.ReasonNotGranted + } + return f.inner.Allows(id, op) +} + +func (f failClosed) Snapshot(id string) hostops.EffectiveSet { + if id == "" || f.broken() { + return hostops.EffectiveSet{} + } + return f.inner.Snapshot(id) +} + +func (f failClosed) Entitled(id string) bool { + return id != "" && !f.broken() && f.inner.Entitled(id) +} + +// buildAddonAuthority is the Authority every reader asks. On the owner it reads +// the store on every call; elsewhere it is the floor of the deployment alone. +func (a *Application) buildAddonAuthority() hostops.Authority { + g := a.addonGrants + if g == nil { + return hostgrants.NewAuthority(hostgrants.AuthorityConfig{Floor: a.hostGrantsFloorFromConfig()}) + } + inner := hostgrants.NewAuthority(hostgrants.AuthorityConfig{Floor: g.floor, Store: g.store, Context: g.context}) + return failClosed{inner: inner, g: g} +} + +// AddonHostGrantsStore is the owner's document of UI grants, or nil in a process that has none. +func (a *Application) AddonHostGrantsStore() *hostgrants.Store { + if a.addonGrants == nil { + return nil + } + return a.addonGrants.store +} + +// AddonHostGrantsAfterChange is the hook for every grant write. It asks the +// manager to look again at what each add-on holds, in the background, and never +// blocks. The Authority needs no call: it reads the store on every operation. +func (a *Application) AddonHostGrantsAfterChange(addonIDs []string) { + if g := a.addonGrants; g != nil { + g.refreshSoon(a.applicationConfig.Context, addonIDs) + } +} + +// ApplyAddonHostGrants runs the refresh now. +func (a *Application) ApplyAddonHostGrants(ctx context.Context) error { + if a.addonGrants == nil || a.addonGrants.mgr() == nil { + return addons.ErrConfiguration + } + return a.addonGrants.mgr().HostGrantsChanged(ctx) +} + +func (g *addonHostGrants) refreshSoon(ctx context.Context, ids []string) { + m := g.mgr() + if m == nil { + return + } + g.refreshMu.Lock() + g.refreshQueued = true + if g.refreshRunning { + g.refreshMu.Unlock() + return + } + g.refreshRunning = true + g.refreshMu.Unlock() + go func() { + for { + g.refreshMu.Lock() + if !g.refreshQueued { + g.refreshRunning = false + g.refreshMu.Unlock() + return + } + g.refreshQueued = false + g.refreshMu.Unlock() + if ctx == nil || ctx.Err() == nil { + if err := m.HostGrantsChanged(context.Background()); err != nil && !errors.Is(err, addons.ErrRecoveryRequired) { + xlog.Warn("Could not apply a change of host grants to the add-ons", "addons", ids, "error", err) + } + } + } + }() +} + +// --- addons.GrantStore --- + +var blanketName = regexp.MustCompile(`^blanket:(official|any)@rev([0-9]+)$`) + +func (g *addonHostGrants) now() string { return time.Now().UTC().Format(time.RFC3339) } + +func (g *addonHostGrants) auditAppend(e hostgrants.AuditEntry) { _ = g.audit.Append(e) } + +// Stage records pending grants for one lifecycle operation. They are part of +// the entry of the add-on but take effect only when the operation completed. +// An entry that exists keeps working: the new operations carry the ID of the +// operation, the old ones their own. +func (g *addonHostGrants) Stage(ctx context.Context, s addons.GrantStage) error { + if g.locked { + g.auditAppend(hostgrants.AuditEntry{Event: hostgrants.AuditRefused, AddonID: s.AddonID, OperationID: s.OperationID, Reason: "locked_by_deployment", By: s.Actor.UserID}) + return errHostGrantsLocked + } + source := hostgrants.SourceInstall + if s.Action == "upgrade" { + source = hostgrants.SourceUpgrade + } + var blanketMode string + var blanketRev uint64 + if s.Blanket != "" { + m := blanketName.FindStringSubmatch(s.Blanket) + if m == nil { + return fmt.Errorf("record host grants: unknown blanket %q", s.Blanket) + } + blanketMode = m[1] + blanketRev, _ = strconv.ParseUint(m[2], 10, 64) + source = hostgrants.SourceBlanket + } + for range 3 { + snap, err := g.store.Snapshot(ctx) + if err != nil { + return err + } + prev := snap.Grants[s.AddonID] + if prev.Identity == s.Identity && stagedBy(prev, s.OperationID) { + return nil + } + if s.ExpectedRevision != 0 && s.ExpectedRevision != snap.Revision { + g.auditAppend(hostgrants.AuditEntry{Event: hostgrants.AuditRefused, AddonID: s.AddonID, OperationID: s.OperationID, Reason: "revision_conflict", By: s.Actor.UserID}) + return addons.ErrGrantsConflict + } + patch := hostgrants.Patch{} + if blanketRev != 0 { + switch { + case blanketRev == snap.Blanket.Revision: + case blanketRev == snap.Blanket.Revision+1 && snap.Blanket.Mode != blanketMode: + // The deployment fixes a mode the document does not hold yet. + b := snap.Blanket + b.Mode, b.ChangedAt, b.ChangedBy = blanketMode, g.now(), "deployment" + patch.Blanket = &b + default: + return addons.ErrGrantsConflict + } + } + entry := hostgrants.AddonGrants{Identity: s.Identity, Generation: s.Generation, OperationID: s.OperationID} + have := map[string]bool{} + fresh := true + if len(prev.Ops) > 0 && prev.Identity == s.Identity { + entry.Ops, entry.Generation, entry.OperationID = slices.Clone(prev.Ops), s.Generation, prev.OperationID + for _, op := range prev.Ops { + have[op.Name] = true + } + fresh = false + } + var added []string + for _, name := range s.Ops { + if have[name] { + continue + } + op := hostgrants.OpGrant{Name: name, Source: source, At: g.now(), By: s.Actor.UserID, Version: s.Version, BlanketRevision: blanketRev} + if !fresh { + op.OperationID = s.OperationID + } + entry.Ops = append(entry.Ops, op) + added = append(added, name) + } + if len(added) == 0 { + return nil + } + patch.Addons = map[string]*hostgrants.AddonGrants{s.AddonID: &entry} + res, err := g.store.Update(ctx, patch, snap.Revision, "stage-"+s.OperationID+"-r"+strconv.FormatUint(snap.Revision, 10)) + if errors.Is(err, hostgrants.ErrConflict) && s.ExpectedRevision == 0 { + continue + } + if errors.Is(err, hostgrants.ErrConflict) { + return addons.ErrGrantsConflict + } + if err != nil { + return err + } + ev := hostgrants.AuditGrant + tag := "" + if source == hostgrants.SourceBlanket { + ev, tag = hostgrants.AuditBlanketGrant, blanketMode+"@rev"+strconv.FormatUint(blanketRev, 10) + } + g.auditAppend(hostgrants.AuditEntry{Event: ev, AddonID: s.AddonID, Ops: added, Source: string(source), Version: s.Version, Digest: s.Digest, + OperationID: s.OperationID, RevisionBefore: snap.Revision, RevisionAfter: res.Snapshot.Revision, By: s.Actor.UserID, Credential: s.Actor.Mode, Blanket: tag}) + return nil + } + return addons.ErrGrantsConflict +} + +// stagedBy reports whether the entry already holds the operations of the +// lifecycle operation, so a continued operation does not stage twice. +func stagedBy(e hostgrants.AddonGrants, opID string) bool { + for _, op := range e.Ops { + if hostgrants.EffectiveOperationID(e, op) == opID { + return true + } + } + return false +} + +// Abort removes what an operation that did not complete staged, and nothing +// else: the operations the add-on held before keep working. +func (g *addonHostGrants) Abort(ctx context.Context, operationID string) error { + for range 3 { + snap, err := g.store.Snapshot(ctx) + if err != nil { + return err + } + patch := hostgrants.Patch{Addons: map[string]*hostgrants.AddonGrants{}} + var audits []hostgrants.AuditEntry + for id, e := range snap.Grants { + var kept []hostgrants.OpGrant + var dropped []string + for _, op := range e.Ops { + if hostgrants.EffectiveOperationID(e, op) == operationID { + dropped = append(dropped, op.Name) + } else { + kept = append(kept, op) + } + } + if len(dropped) == 0 { + continue + } + if len(kept) == 0 { + patch.Addons[id] = nil + } else { + e.Ops = kept + patch.Addons[id] = &e + } + audits = append(audits, hostgrants.AuditEntry{Event: hostgrants.AuditRefused, AddonID: id, Ops: dropped, OperationID: operationID, Reason: "operation_not_completed", RevisionBefore: snap.Revision}) + } + if len(patch.Addons) == 0 { + return nil + } + res, err := g.store.Update(ctx, patch, snap.Revision, "abort-"+operationID+"-r"+strconv.FormatUint(snap.Revision, 10)) + if errors.Is(err, hostgrants.ErrConflict) { + continue + } + if err != nil { + return err + } + for _, e := range audits { + e.RevisionAfter = res.Snapshot.Revision + g.auditAppend(e) + } + return nil + } + return hostgrants.ErrConflict +} + +// Remove deletes the grants of an uninstalled add-on and, when asked, keeps a +// hint that pre-ticks a later install. A hint grants nothing. +func (g *addonHostGrants) Remove(ctx context.Context, addonID string, keepHint bool) error { + for range 3 { + snap, err := g.store.Snapshot(ctx) + if err != nil { + return err + } + entry, had := snap.Grants[addonID] + _, hinted := snap.Hints[addonID] + if !had && !(hinted && !keepHint) { + return nil + } + patch := hostgrants.Patch{Addons: map[string]*hostgrants.AddonGrants{addonID: nil}} + var names []string + for _, op := range entry.Ops { + names = append(names, op.Name) + } + if keepHint && len(names) > 0 { + patch.Hints = map[string][]string{addonID: names} + } else if hinted { + patch.Hints = map[string][]string{addonID: nil} + } + res, err := g.store.Update(ctx, patch, snap.Revision, "remove-"+addonID+"-r"+strconv.FormatUint(snap.Revision, 10)) + if errors.Is(err, hostgrants.ErrConflict) { + continue + } + if err != nil { + return err + } + if had { + g.auditAppend(hostgrants.AuditEntry{Event: hostgrants.AuditRevoke, AddonID: addonID, Ops: names, Reason: "uninstalled", RevisionBefore: snap.Revision, RevisionAfter: res.Snapshot.Revision}) + } + return nil + } + return hostgrants.ErrConflict +} + +// Reconcile drops the entries that no longer match the journal. +func (g *addonHostGrants) Reconcile(ctx context.Context, installed map[string]string, completed func(string) bool) error { + removed, err := g.store.Reconcile(ctx, hostgrants.ReconcileInput{ + Installed: func(id string) (string, bool) { identity, ok := installed[id]; return identity, ok }, + OperationCompleted: completed, + }) + for _, r := range removed { + g.auditAppend(hostgrants.AuditEntry{Event: hostgrants.AuditReconcileRemove, AddonID: r.AddonID, Ops: r.Ops, Reason: r.Reason}) + } + return err +} + +// Blanket says what the blanket switch grants to a package, when it is on and +// the package is in its scope. +func (g *addonHostGrants) Blanket(ctx context.Context, q addons.BlanketQuery) ([]string, string) { + if g.locked || g.store.Usable() != nil { + return nil, "" + } + snap := g.store.View() + id := q.Receipt.Manifest.ID + res := g.floor.EvaluateBlanket(snap, g.context(id), g.override, hostgrants.BlanketInput{ + AddonID: id, Official: addons.OfficialReceipt(q.Receipt), Required: q.Required, Optional: q.Optional, + }) + if !res.Applies || len(res.Plan.Grant) == 0 { + return nil, "" + } + mode, rev := snap.Blanket.Mode, res.Revision + if g.override != "" { + mode = g.override + if snap.Blanket.Mode != mode { + // Stage first writes the mode, which moves the revision by one. + rev = snap.Blanket.Revision + 1 + } + } + return res.Plan.Grant, fmt.Sprintf("blanket:%s@rev%d", mode, rev) +} + +// --- gateway.HostAccessManager --- + +type hostAccessManager struct{ m *addons.Manager } + +func (h hostAccessManager) HostAccess(ctx context.Context) ([]gateway.InstalledAccess, error) { + list, err := h.m.HostAccessInstalled(ctx) + if err != nil { + return nil, err + } + out := make([]gateway.InstalledAccess, 0, len(list)) + for _, in := range list { + a := gateway.InstalledAccess{AddonID: in.AddonID, Title: in.Title, Version: in.Version, State: in.State, Identity: in.Identity, + Generation: in.Generation, OperationID: in.OperationID, Official: in.Official, Required: in.Required, Optional: in.Optional, + SourceID: in.SourceID, NewOptional: in.NewOptional} + if in.HeldVersion != "" { + a.PendingUpgrade = &gateway.PendingUpgrade{Version: in.HeldVersion, Delta: in.HeldDelta} + } + out = append(out, a) + } + return out, nil +} + +func (h hostAccessManager) PreviewHostAccess(ctx context.Context, sourceID, addonID, version string) (gateway.ReleaseAccess, error) { + r, err := h.m.PreviewRelease(ctx, sourceID, addonID, version) + if err != nil { + return gateway.ReleaseAccess{}, err + } + return gateway.ReleaseAccess{AddonID: r.AddonID, Title: r.Title, Version: r.Version, Digest: r.Digest, Official: r.Official, + Required: r.Required, Optional: r.Optional, Installed: r.Installed}, nil +} + +func (h hostAccessManager) SubmitWithApproval(ctx context.Context, actor contract.Actor, cmd addons.LifecycleCommand, ap gateway.HostApproval) (addons.OperationStatus, error) { + cmd.Approval = &addons.HostApproval{Digest: ap.Digest, Ops: ap.Ops, Revision: ap.Revision, Nonce: ap.Nonce} + st, err := h.m.Submit(ctx, actor, cmd) + // The owner service types its refusals by these errors. + var stale *addons.ApprovalStaleError + var refusal *addons.HostRequirementsRefusal + switch { + case errors.As(err, &stale): + return st, gateway.ErrApprovalStale + case errors.As(err, &refusal) && refusal.Blocked(): + return st, gateway.ErrBlockedByPolicy + case errors.As(err, &refusal): + return st, gateway.ErrApprovalRequired + } + return st, err +} + +// wireHostGrantsOwner gives the gateway of the owner the host access routes. +// A frontend has no grants and forwards. +func (a *Application) wireHostGrantsOwner(g *gateway.Gateway) error { + hg := a.addonGrants + if hg == nil || hg.store == nil || a.addonManager == nil { + return nil + } + owner, err := gateway.NewHostGrantsOwner(gateway.HostGrantsConfig{ + Store: hg.store, Floor: hg.floor, Audit: hg.audit, Manager: hostAccessManager{a.addonManager}, + Supported: hostops.SupportedOperations(), + Describe: func(op string) (gateway.HostOpDescription, bool) { + title, text, risk, ok := hostops.DescribeOperation(op) + return gateway.HostOpDescription{Title: title, Text: text, Risk: risk}, ok + }, + BlanketOverride: hg.override, Locked: hg.locked, + Completed: a.addonManager.OperationCompleted, + Admin: g.HostSettingsAdmin, + AfterChange: a.AddonHostGrantsAfterChange, + }) + if err != nil { + return err + } + g.HostGrantsOwner = owner + return nil +} diff --git a/core/application/addon_hostgrants_test.go b/core/application/addon_hostgrants_test.go new file mode 100644 index 0000000000..c8ee80cf3d --- /dev/null +++ b/core/application/addon_hostgrants_test.go @@ -0,0 +1,343 @@ +// SPDX-License-Identifier: MIT +package application + +import ( + "archive/tar" + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "time" + + "github.com/mudler/LocalAI/core/config" + "github.com/mudler/LocalAI/core/http/auth" + "github.com/mudler/LocalAI/core/services/addons" + "github.com/mudler/LocalAI/core/services/addons/catalog" + "github.com/mudler/LocalAI/core/services/addons/gateway" + "github.com/mudler/LocalAI/core/services/addons/hostops" + "github.com/mudler/LocalAI/pkg/addons/contract" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +// These specs run the owner the way local-ai run does, install a package +// through the real host access service and ask the Authority through a broker. +var _ = Describe("host grants on the owner", func() { + var ( + ownerData string + keys string + port string + sourceURL string + digests map[string]string + actor = contract.Actor{UserID: auth.AddonLocalOwnerID, CredentialID: auth.AddonLocalOwnerID, Mode: auth.AddonModeLocalOwner} + ) + + pack := func(version string, required, optional []string) (data []byte) { + manifest, e := os.ReadFile("../../tests/addons/fixtures/packages/manifest.json") + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + var m map[string]any + ExpectWithOffset(1, json.Unmarshal(manifest, &m)).To(Succeed()) + m["version"] = version + m["capabilities"] = map[string]any{"service": true, "hostOperations": map[string]any{"required": required, "optional": optional}} + manifest, _ = json.Marshal(m) + var b bytes.Buffer + tw := tar.NewWriter(&b) + for name, content := range map[string][]byte{"manifest.json": manifest, "bin/sample": []byte("binary " + version)} { + ExpectWithOffset(1, tw.WriteHeader(&tar.Header{Name: name, Mode: 0755, Size: int64(len(content))})).To(Succeed()) + _, e = tw.Write(content) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + } + ExpectWithOffset(1, tw.Close()).To(Succeed()) + return b.Bytes() + } + + BeforeEach(func() { + requireAddonRuntime() + keys = filepath.Join(realTempDir(), "keys") + Expect(gateway.GenerateCarrierKeys(keys, []string{"127.0.0.1"})).To(Succeed()) + ownerData = realTempDir() + l, e := net.Listen("tcp", "127.0.0.1:0") + Expect(e).NotTo(HaveOccurred()) + port = strconv.Itoa(l.Addr().(*net.TCPAddr).Port) + Expect(l.Close()).To(Succeed()) + + archives := map[string][]byte{ + "/1.2.3.tar": pack("1.2.3", []string{"workspace.materialize"}, []string{"egress.check"}), + "/1.3.0.tar": pack("1.3.0", []string{"workspace.materialize", "egress.audit"}, []string{"egress.check"}), + } + digests = map[string]string{} + for path, data := range archives { + digests[path] = fmt.Sprintf("sha256:%x", sha256.Sum256(data)) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if data, ok := archives[r.URL.Path]; ok { + _, _ = w.Write(data) + return + } + rel := func(v string) catalog.Release { + return catalog.Release{ID: "sample", Name: "Sample", Version: v, OS: "linux", Arch: "amd64", Artifact: sourceURL + "/" + v + ".tar", Digest: digests["/"+v+".tar"]} + } + _ = catalog.WriteIndex(w, []catalog.Release{rel("1.2.3"), rel("1.3.0")}) + })) + DeferCleanup(server.Close) + sourceURL = server.URL + root := filepath.Join(ownerData, "addons") + DeferCleanup(func() { + _ = filepath.Walk(root, func(p string, i os.FileInfo, e error) error { + if e == nil && i.IsDir() { + _ = os.Chmod(p, 0700) + } + return nil + }) + }) + Expect(os.MkdirAll(root, 0700)).To(Succeed()) + sources, e := catalog.NewSources(filepath.Join(root, "sources.json"), func(context.Context, contract.Actor) error { return nil }, catalog.CredentialResolver{}) + Expect(e).NotTo(HaveOccurred()) + Expect(sources.Save(context.Background(), actor, catalog.Source{ID: "test", Name: "Test", URL: sourceURL, Enabled: true, AllowLoopbackHTTP: true})).To(Succeed()) + }) + + // startOwner builds the owner on the same data directory. Each start + // stands for one run of the process. + startOwner := func(opts ...func(*config.ApplicationConfig)) (*Application, func()) { + transport, e := gateway.LoadCarrierConfig(keys, "owner", "https://127.0.0.1:"+port) + Expect(e).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + cfg := &config.ApplicationConfig{Context: ctx, DataPath: ownerData} + for _, o := range opts { + o(cfg) + } + cfg.Addons = config.AddonConfig{ + Enabled: true, + InstanceID: transport.InstanceID, OwnerID: gateway.OwnerIdentity, OwnerAddress: "https://127.0.0.1:" + port, + StorageID: "local", Root: filepath.Join(ownerData, "addons"), Transport: transport, OwnershipMode: auth.AddonModeLocalOwner, + Host: contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}, + ListenAddress: "127.0.0.1:" + port, + } + a := &Application{applicationConfig: cfg} + stopped := false + stop := func() { + if !stopped { + stopped = true + cancel() + _ = a.Shutdown() + } + } + if e = a.startAddons(); e != nil { + stop() + Fail("owner did not start: " + e.Error()) + } + DeferCleanup(stop) + return a, stop + } + + human := func() context.Context { return gateway.WithHumanSurface(context.Background()) } + call := func(a *Application, c gateway.HostGrantsCall) (map[string]any, error) { + raw, e := a.AddonGateway().HostGrants(human(), actor, c) + if e != nil { + return nil, e + } + out := map[string]any{} + ExpectWithOffset(1, json.Unmarshal(raw, &out)).To(Succeed()) + return out, nil + } + body := func(v any) json.RawMessage { + b, e := json.Marshal(v) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + return b + } + revision := func(a *Application) uint64 { + out, e := call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsRead}) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + return uint64(out["revision"].(float64)) + } + // install previews a release and installs it with the approval of ops. + install := func(a *Application, opID, action, version string, ops []string, expected string) error { + pv, e := call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsPreview, Body: body(gateway.HostGrantsPreviewRequest{SourceID: "test", AddonID: "sample", Version: version})}) + ExpectWithOffset(1, e).NotTo(HaveOccurred()) + req := gateway.HostGrantsInstallRequest{OperationID: opID, Action: action, SourceID: "test", AddonID: "sample", Version: version, + Approve: gateway.HostGrantsApprove{Digest: pv["digest"].(string), Ops: ops, Revision: uint64(pv["revision"].(float64)), Nonce: pv["nonce"].(string)}} + if expected != "" { + req.ExpectedDigest, req.CandidateDigest = expected, pv["digest"].(string) + } + _, e = call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsInstall, OperationID: opID, Body: body(req)}) + return e + } + // ask makes a host operation call the way the broker serves it, with the + // Authority of the application. + ask := func(a *Application, op string) int { + in := hostops.NewInFlight() + b := hostops.New(in, func(string, string) bool { return true }, contract.DefaultLimits()) + ExpectWithOffset(1, b.Register(hostops.Spec{Op: op, Scope: "read", Feature: "x", Handler: func(context.Context, hostops.Call) (json.RawMessage, error) { + return json.RawMessage(`{"ok":true}`), nil + }})).To(Succeed()) + h := b.HandlerForRuntime("sample", "secret", a.addonAuthority(), 0, "") + end := in.Begin(contract.Envelope{ + Request: contract.Request{AddonID: "sample", OperationID: "op1", Generation: 1, Action: "start", Arguments: json.RawMessage(`{}`)}, + Authority: contract.Authority{Actor: contract.Actor{UserID: "u1", CredentialID: "c1", Mode: "named-user"}, Scope: "read", ExpiresAt: time.Now().Add(time.Minute)}, + }) + defer end() + raw, _ := json.Marshal(contract.HostRequest{OperationID: "op1", Op: op, Arguments: json.RawMessage(`{}`)}) + req := httptest.NewRequest(http.MethodPost, "/host", bytes.NewReader(raw)) + req.Header.Set("X-Addon-Host-Credential", "secret") + rec := httptest.NewRecorder() + h.ServeHTTP(rec, req) + return rec.Code + } + submit := func(a *Application, opID, action, version, expected, candidate string) error { + _, e := a.addonManager.Submit(context.Background(), actor, addons.LifecycleCommand{OperationID: opID, AddonID: "sample", Action: action, SourceID: "test", Version: version, ExpectedDigest: expected, CandidateDigest: candidate}) + return e + } + + It("grants at install, revokes and denies without a restart, and keeps the grants across restarts", func() { + a, stop := startOwner() + Expect(a.AddonGateway().HostGrantsOwner).NotTo(BeNil()) + + // Nothing approved: the install is refused and says what to approve. + pv, e := call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsPreview, Body: body(gateway.HostGrantsPreviewRequest{SourceID: "test", AddonID: "sample", Version: "1.2.3"})}) + Expect(e).NotTo(HaveOccurred()) + Expect(pv["canInstall"]).To(BeTrue()) + Expect(install(a, "i-none", "install", "1.2.3", nil, "")).To(HaveOccurred()) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusForbidden)) + + // Approved: the required operation works, the unticked optional one does not. + Expect(install(a, "i-1", "install", "1.2.3", []string{"workspace.materialize"}, "")).To(Succeed()) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK)) + Expect(ask(a, "egress.check")).To(Equal(http.StatusForbidden)) + Expect(a.addonAuthority().Snapshot("sample").Operations).To(Equal([]string{"workspace.materialize"})) + + // Revoke: the very next call is refused, with no restart. + _, e = call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsRevoke, AddonID: "sample", OperationID: "r-1", Body: body(gateway.HostGrantsRevokeRequest{OperationID: "r-1", Revision: revision(a)})}) + Expect(e).NotTo(HaveOccurred()) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusForbidden)) + + // Edit gives it back; a dry run of a deny names who loses what and writes nothing. + _, e = call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsEdit, AddonID: "sample", OperationID: "e-1", Body: body(gateway.HostGrantsEditRequest{OperationID: "e-1", Revision: revision(a), Ops: []string{"workspace.materialize", "egress.check"}})}) + Expect(e).NotTo(HaveOccurred()) + Expect(ask(a, "egress.check")).To(Equal(http.StatusOK)) + before := revision(a) + dry, e := call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsDenyPreview, Body: body(gateway.HostGrantsDenyRequest{Revision: before, Deny: map[string][]string{"sample": {"egress."}}, DryRun: true})}) + Expect(e).NotTo(HaveOccurred()) + Expect(dry["affects"]).NotTo(BeEmpty()) + Expect(revision(a)).To(Equal(before)) + Expect(ask(a, "egress.check")).To(Equal(http.StatusOK)) + + // A deny set in the UI beats the grant at once; removing it restores the grant. + _, e = call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsDeny, OperationID: "d-1", Body: body(gateway.HostGrantsDenyRequest{OperationID: "d-1", Revision: before, Deny: map[string][]string{"sample": {"egress."}}})}) + Expect(e).NotTo(HaveOccurred()) + Expect(ask(a, "egress.check")).To(Equal(http.StatusForbidden)) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK)) + _, e = call(a, gateway.HostGrantsCall{Op: gateway.HostGrantsDeny, OperationID: "d-2", Body: body(gateway.HostGrantsDenyRequest{OperationID: "d-2", Revision: revision(a), Deny: map[string][]string{}})}) + Expect(e).NotTo(HaveOccurred()) + Expect(ask(a, "egress.check")).To(Equal(http.StatusOK)) + + // The audit file records the changes and names no path or secret. + audit, e := os.ReadFile(filepath.Join(ownerData, "addons", addonHostGrantsAudit)) + Expect(e).NotTo(HaveOccurred()) + Expect(string(audit)).To(ContainSubstring(`"event":"grant"`)) + Expect(string(audit)).To(ContainSubstring(`"event":"revoke"`)) + Expect(string(audit)).NotTo(ContainSubstring(ownerData)) + + // A restart reopens the document: the grants are still there. + stop() + again, _ := startOwner() + Expect(ask(again, "workspace.materialize")).To(Equal(http.StatusOK)) + Expect(ask(again, "egress.check")).To(Equal(http.StatusOK)) + }) + + It("lets a deny of the deployment beat a grant of the UI", func() { + a, stop := startOwner() + Expect(install(a, "i-1", "install", "1.2.3", []string{"workspace.materialize"}, "")).To(Succeed()) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK)) + stop() + denied, stopDenied := startOwner(func(c *config.ApplicationConfig) { + c.AddonHostDeny = map[string][]string{"*": {"workspace."}} + }) + Expect(ask(denied, "workspace.materialize")).To(Equal(http.StatusForbidden)) + stopDenied() + // The grant was not lost: without the deny it works again. + last, _ := startOwner() + Expect(ask(last, "workspace.materialize")).To(Equal(http.StatusOK)) + }) + + It("refuses everything when the document cannot be read", func() { + Expect(os.MkdirAll(filepath.Join(ownerData, "addons"), 0700)).To(Succeed()) + Expect(os.WriteFile(filepath.Join(ownerData, "addons", addonHostGrantsFile), []byte("{not json"), 0600)).To(Succeed()) + a, _ := startOwner(func(c *config.ApplicationConfig) { + c.AddonHostOps = map[string][]string{"sample": {"workspace."}} + }) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusForbidden), "the deny rules in the document are unknown") + Expect(a.AddonHostGrantsStore()).To(BeNil()) + }) + + It("holds an upgrade that asks for more and keeps the old version running", func() { + a, _ := startOwner() + Expect(install(a, "i-1", "install", "1.2.3", []string{"workspace.materialize"}, "")).To(Succeed()) + old := digests["/1.2.3.tar"] + newer := digests["/1.3.0.tar"] + // Without an approval the upgrade is refused and nothing changes. + e := submit(a, "u-0", "upgrade", "1.3.0", old, newer) + var refusal *addons.HostRequirementsRefusal + Expect(e).To(HaveOccurred()) + Expect(errors.As(e, &refusal)).To(BeTrue(), e.Error()) + st, e := a.addonManager.Status(context.Background(), "sample") + Expect(e).NotTo(HaveOccurred()) + Expect(st.Generation).To(Equal(uint64(1))) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK), "the old version keeps its grant") + Expect(ask(a, "egress.audit")).To(Equal(http.StatusForbidden)) + + // Approved in the browser: the new operation counts once the upgrade completed. + Expect(install(a, "u-1", "upgrade", "1.3.0", []string{"egress.audit"}, old)).To(Succeed()) + st, _ = a.addonManager.Status(context.Background(), "sample") + Expect(st.Generation).To(Equal(uint64(2))) + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK)) + Expect(ask(a, "egress.audit")).To(Equal(http.StatusOK)) + }) + + It("applies the blanket mode a deployment fixes without a dialog, and records it", func() { + a, _ := startOwner(func(c *config.ApplicationConfig) { c.AddonHostBlanket = "any" }) + Expect(submit(a, "b-1", "install", "1.2.3", "", "")).To(Succeed()) + // Required operations are granted; the optional one needs includeOptional. + Expect(ask(a, "workspace.materialize")).To(Equal(http.StatusOK)) + Expect(ask(a, "egress.check")).To(Equal(http.StatusForbidden)) + audit, e := os.ReadFile(filepath.Join(ownerData, "addons", addonHostGrantsAudit)) + Expect(e).NotTo(HaveOccurred()) + Expect(string(audit)).To(ContainSubstring(`"event":"blanket-grant"`)) + Expect(string(audit)).To(ContainSubstring(`"blanket":"any@rev1"`)) + }) + + It("keeps the grants of a split deployment on the owner only", func() { + a, _ := startOwner() + Expect(install(a, "i-1", "install", "1.2.3", []string{"workspace.materialize"}, "")).To(Succeed()) + transport, e := gateway.LoadCarrierConfig(keys, "frontend", "https://127.0.0.1:"+port) + Expect(e).NotTo(HaveOccurred()) + ctx, cancel := context.WithCancel(context.Background()) + DeferCleanup(cancel) + frontendData := realTempDir() + f := &Application{applicationConfig: &config.ApplicationConfig{Context: ctx, DataPath: frontendData, + AddonHostOps: map[string][]string{"sample": {"egress."}}}} + f.applicationConfig.Addons = config.AddonConfig{Enabled: true, InstanceID: transport.InstanceID, OwnerID: gateway.OwnerIdentity, + OwnerAddress: "https://127.0.0.1:" + port, StorageID: "local", Transport: transport, OwnershipMode: auth.AddonModeLocalOwner, + Host: contract.HostCompatibility{Host: 1, API: contract.HostAPIVersion, Protocol: 1, SDK: 1, DataFormat: 1, OS: "linux", Arch: "amd64"}} + Expect(f.startAddons()).To(Succeed()) + DeferCleanup(func() { _ = f.Shutdown() }) + // The frontend stores nothing and enforces only its own floor, never the grants of the owner. + Expect(f.AddonHostGrantsStore()).To(BeNil()) + Expect(f.AddonGateway().HostGrantsOwner).To(BeNil()) + Expect(ask(f, "workspace.materialize")).To(Equal(http.StatusForbidden)) + entries, e := os.ReadDir(frontendData) + Expect(e).NotTo(HaveOccurred()) + Expect(entries).To(BeEmpty()) + // Reading the owner's grants through the frontend works, and writes forward. + out, e := call(f, gateway.HostGrantsCall{Op: gateway.HostGrantsRead}) + Expect(e).NotTo(HaveOccurred()) + Expect(out["revision"]).NotTo(BeZero()) + }) +}) diff --git a/core/application/application.go b/core/application/application.go index ba1f2ceb70..bc27c073d4 100644 --- a/core/application/application.go +++ b/core/application/application.go @@ -118,6 +118,8 @@ type Application struct { hostAuthority hostops.Authority addonAuthorityOnce sync.Once addonGrantStore addons.GrantStore + // addonGrants is the owner's host grants document and its glue; nil elsewhere. + addonGrants *addonHostGrants // addonTransfers holds the handles of connector file transfers; see transfers. addonTransfers *hostops.Transfers addonTransfersOnce sync.Once @@ -735,6 +737,7 @@ func (a *Application) Shutdown() error { if a.addonManager != nil { err = a.addonManager.Close() } + a.closeAddonHostGrants() // Before distributed shutdown: the sync's subscriptions live on the // NATS connection that closes there. a.stopFailoverDistributed() @@ -1005,6 +1008,7 @@ func (a *Application) startAddons() error { a.addonHostSettingsClose() } _ = m.Close() + a.closeAddonHostGrants() } }() if c.InstanceID == c.OwnerID && c.Transport != nil { @@ -1012,6 +1016,10 @@ func (a *Application) startAddons() error { return err } } + if c.InstanceID == c.OwnerID && c.Root != "" { + a.openAddonHostGrants(c.Root) + a.addonGrants.bind(m) + } m.SetHostOps(a.applicationConfig.AddonHostOps) m.SetAuthority(a.addonAuthority()) m.SetGrantStore(a.addonGrantStore)