Bump reva

This commit is contained in:
André Duffeck committed 2026-09-21 09:44:58 +02:00
1 parent be3c7b674e
commit e3e7465d2c
11 files changed
+258 -26

No files matched your search

+1 -1
View File
@@ -64,7 +64,7 @@ require (
github.com/open-policy-agent/opa v1.19.1
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067
github.com/opencloud-eu/reva/v2 v2.50.0
github.com/opencloud-eu/reva/v2 v2.50.1-0.20260921072732-1796c81fa9f2
github.com/opensearch-project/opensearch-go/v4 v4.7.3
github.com/orcaman/concurrent-map v1.0.0
github.com/pkg/errors v0.9.1
+2 -2
View File
@@ -942,8 +942,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l
github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067 h1:UkNMKauyJAzY6RE6mmthz9bQZLYkbvBuApm7ZDCparE=
github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067/go.mod h1:lTM8JeGblNpoMySTW7Lui2+c5TTLI95mwxtdUIHHrhU=
github.com/opencloud-eu/reva/v2 v2.50.0 h1:N5e0AuqACiZdQZAlEk7JqrSQfqwQVt26xAsCGzzOxOo=
github.com/opencloud-eu/reva/v2 v2.50.0/go.mod h1:PZFZK90dVfh+pkelCbgtGjmUQots9x7JOd1I2tt2aIQ=
github.com/opencloud-eu/reva/v2 v2.50.1-0.20260921072732-1796c81fa9f2 h1:4z91hPzO9itzPHYVr+w/++pewG9V3Df58FLq2b8oTlM=
github.com/opencloud-eu/reva/v2 v2.50.1-0.20260921072732-1796c81fa9f2/go.mod h1:PZFZK90dVfh+pkelCbgtGjmUQots9x7JOd1I2tt2aIQ=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4 h1:l2oB/RctH+t8r7QBj5p8thfEHCM/jF35aAY3WQ3hADI=
github.com/opencloud-eu/secure v0.0.0-20260312082735-b6f5cb2244e4/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
@@ -245,7 +245,9 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke
}
}
log.Warn().Msg("core access token not set")
if !isUnprotectedEndpoint {
log.Debug().Msg("core access token not set")
}
userAgentCredKeys := getCredsForUserAgent(r.UserAgent(), conf.CredentialsByUserAgent, conf.CredentialChain)
@@ -300,7 +302,7 @@ func authenticateUser(w http.ResponseWriter, r *http.Request, conf *config, toke
return nil, err
}
log.Info().Msg("core access token generated") // write token to response
log.Debug().Msg("core access token generated") // write token to response
// write token to response
token := res.Token
@@ -27,6 +27,7 @@ import (
"time"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/auth"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/downloader"
"github.com/opencloud-eu/reva/v2/pkg/storage/utils/walker"
"github.com/opencloud-eu/reva/v2/pkg/utils"
@@ -72,7 +73,7 @@ func (a *Archiver) CreateTar(ctx context.Context, dst io.Writer) (func(), error)
for _, root := range a.resources {
err := a.walker.Walk(ctx, root, func(wd string, info *provider.ResourceInfo, err error) error {
err := a.walker.Walk(auth.NewStaticSession(ctx), root, func(wd string, info *provider.ResourceInfo, err error) error {
if err != nil {
return err
}
@@ -147,7 +148,7 @@ func (a *Archiver) CreateZip(ctx context.Context, dst io.Writer) (func(), error)
for _, root := range a.resources {
err := a.walker.Walk(ctx, root, func(wd string, info *provider.ResourceInfo, err error) error {
err := a.walker.Walk(auth.NewStaticSession(ctx), root, func(wd string, info *provider.ResourceInfo, err error) error {
if err != nil {
return err
}
+116
View File
@@ -0,0 +1,116 @@
// Copyright 2026 OpenCloud GmbH <mail@opencloud.eu>
// SPDX-License-Identifier: Apache-2.0
package auth
import (
"context"
"sync"
"time"
)
const (
// RefreshLeeway is how long before a token's expiry a Session refreshes it.
// Refreshing ahead of time avoids races where a request is made with a token
// that expires while it is in flight.
RefreshLeeway = 30 * time.Second
retryBackoff = 5 * time.Second
)
// Authenticator produces an authenticated context together with the time at which
// its token expires. A Session calls it once up front and then again shortly
// before each expiry to keep the context fresh.
type Authenticator func(ctx context.Context) (context.Context, time.Time, error)
// Session keeps an authenticated context valid for the whole duration of a
// long-running task.
//
// Many operations (indexing a space, creating an archive, bulk migrations, …)
// can outlive the lifetime of a single authentication token. Passing a static
// context to such a task means its token expires halfway through and every
// subsequent call fails. Session solves this by owning the context: it
// authenticates once up front via an Authenticator and then refreshes the
// context in the background shortly before the token expires. Callers always
// read the current context through Ctx, so they transparently pick up refreshed
// tokens without having to thread a new context around.
// A Session is safe for concurrent use.
type Session struct {
mu sync.RWMutex
ctx context.Context
cancel context.CancelFunc
}
// NewSession authenticates once via auth and then refreshes the context in the
// background shortly before the token expires. It returns an error if the initial
// authentication fails.
func NewSession(parent context.Context, auth Authenticator) (*Session, error) {
sessionCtx, cancel := context.WithCancel(parent)
ctx, expiry, err := auth(sessionCtx)
if err != nil {
cancel()
return nil, err
}
s := &Session{ctx: ctx, cancel: cancel}
go s.refresh(sessionCtx, auth, expiry)
return s, nil
}
// NewStaticSession returns a Session that always returns ctx and never refreshes.
// It is useful for callers that manage their own (typically short-lived) context
// but need to satisfy a Session-based API.
func NewStaticSession(ctx context.Context) *Session {
return &Session{ctx: ctx, cancel: func() {}}
}
// Ctx returns the current authentication context. It always carries a valid token
// as long as the Authenticator keeps succeeding.
func (s *Session) Ctx() context.Context {
s.mu.RLock()
defer s.mu.RUnlock()
return s.ctx
}
// Close stops the background refresher and cancels the session-owned context,
// unblocking any in-flight authentication call.
func (s *Session) Close() {
s.cancel()
}
func (s *Session) set(ctx context.Context) {
s.mu.Lock()
defer s.mu.Unlock()
s.ctx = ctx
}
func (s *Session) refresh(ctx context.Context, auth Authenticator, expiry time.Time) {
for {
wait := max(time.Until(expiry)-RefreshLeeway, 0)
timer := time.NewTimer(wait)
select {
case <-ctx.Done():
timer.Stop()
return
case <-timer.C:
}
newCtx, exp, err := auth(ctx)
// If the session was closed while auth was in flight, drop the result so a
// slow authenticator can no longer update the session after Close.
if ctx.Err() != nil {
return
}
if err != nil {
// keep using the current context and retry after a short backoff
select {
case <-ctx.Done():
return
case <-time.After(retryBackoff):
}
continue
}
s.set(newCtx)
expiry = exp
}
}
+1
View File
@@ -76,6 +76,7 @@ var mimeTypes = map[string]string{
"atx": "application/vnd.antix.game-component",
"au": "audio/basic",
"avi": "video/x-msvideo",
"avif": "image/avif",
"aw": "application/applixware",
"azf": "application/vnd.airzip.filesecure.azf",
"azs": "application/vnd.airzip.filesecure.azs",
@@ -767,6 +767,14 @@ assimilate:
}
n = node.New(spaceID, id, parentID, filepath.Base(path), treeSize, "", provider.ResourceType_RESOURCE_TYPE_CONTAINER, nil, t.lookup)
} else {
// CalculateChecksums reads the whole file, so skip an unchanged file that recently failed after
// this point, e.g. because the service user can't set its xattrs
if err := t.assimilationFailures.Recent(path, fi); err != nil {
return nil, nil, err
}
// Record sees err because every failure below assigns it before returning
defer func() { t.assimilationFailures.Record(path, fi, err) }()
sha1h, md5h, adler32h, err := node.CalculateChecksums(context.Background(), path)
if err == nil {
attributes[prefixes.ChecksumPrefix+"sha1"] = sha1h.Sum(nil)
@@ -859,17 +867,19 @@ assimilate:
}()
}
err = t.Propagate(context.Background(), n, sizeDiff)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to propagate")
}
t.log.Debug().Str("path", path).Interface("attributes", attributes).Msg("setting attributes")
err = t.lookup.MetadataBackend().SetMultiple(context.Background(), bn, attributes)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to set attributes")
}
// only propagate once the attributes are stored. If storing them failed after propagating, the file
// would still have no blobsize and the next attempt would propagate its whole size again.
err = t.Propagate(context.Background(), n, sizeDiff)
if err != nil {
return nil, nil, errors.Wrap(err, "failed to propagate")
}
// clear the status attribute if it was set before, if there was any upload to this file in progress
// it needs notice that this file was changes meanwhile.
if _, ok := previousAttribs[prefixes.StatusPrefix]; ok {
@@ -0,0 +1,95 @@
// Package assimilation contains helpers for assimilating files into the posix fs
package assimilation
import (
"io/fs"
"syscall"
"time"
"github.com/hashicorp/golang-lru/v2/expirable"
"github.com/pkg/errors"
)
const (
// a failed file's retry delay starts at minRetryDelay and doubles per failure, up to maxRetryDelay
minRetryDelay = time.Minute
maxRetryDelay = 24 * time.Hour
// maxFailures caps the memory use at about 8 MB. Files that fail beyond it aren't remembered.
maxFailures = 10000
)
// Failures remembers files that failed to assimilate, so scans can skip them while they are unchanged
type Failures struct {
lru *expirable.LRU[string, failure]
}
// failure is the state of a file when it last failed to assimilate
type failure struct {
err error
modTime time.Time
size int64
mode fs.FileMode
uid uint32
gid uint32
delay time.Duration
retryAt time.Time
}
// NewFailures returns a new Failures
func NewFailures() *Failures {
// entries expire so that deleted files don't pile up
return &Failures{lru: expirable.NewLRU[string, failure](maxFailures, nil, 2*maxRetryDelay)}
}
// Recent returns the last error of the file at path if it is unchanged and not due for a retry yet
func (f *Failures) Recent(path string, fi fs.FileInfo) error {
last, ok := f.lru.Get(path)
if !ok || !last.unchanged(fi) || !time.Now().Before(last.retryAt) {
return nil
}
return errors.Wrapf(last.err, "item is unchanged since it failed to assimilate, not retrying before %s", last.retryAt.Format(time.RFC3339))
}
// Record stores the result of assimilating the file at path. fi is its state before the attempt.
func (f *Failures) Record(path string, fi fs.FileInfo, err error) {
if err == nil || errors.Is(err, fs.ErrNotExist) {
f.lru.Remove(path)
return
}
// when full, don't evict: a scan over more failing files than maxFailures would evict each one
// before it comes around again
if !f.lru.Contains(path) && f.lru.Len() >= maxFailures {
return
}
delay := minRetryDelay
if last, ok := f.lru.Peek(path); ok && last.unchanged(fi) {
delay = min(2*last.delay, maxRetryDelay)
}
uid, gid := owner(fi)
f.lru.Add(path, failure{
err: err,
modTime: fi.ModTime(),
size: fi.Size(),
mode: fi.Mode(),
uid: uid,
gid: gid,
delay: delay,
retryAt: time.Now().Add(delay),
})
}
// unchanged reports whether fi still matches the file that failed. It includes the owner, so a chown
// that fixes the file triggers a retry.
func (f failure) unchanged(fi fs.FileInfo) bool {
uid, gid := owner(fi)
return f.modTime.Equal(fi.ModTime()) && f.size == fi.Size() && f.mode == fi.Mode() && f.uid == uid && f.gid == gid
}
func owner(fi fs.FileInfo) (uint32, uint32) {
if st, ok := fi.Sys().(*syscall.Stat_t); ok {
return st.Uid, st.Gid
}
return 0, 0
}
+7 -3
View File
@@ -48,6 +48,7 @@ import (
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation"
"github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs"
"github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/metadata"
@@ -104,6 +105,8 @@ type Tree struct {
watcher Watcher
scanQueue chan scanItem
scanDebouncer *ScanDebouncer
// files that failed to assimilate, keyed by path, see updateFile()
assimilationFailures *assimilation.Failures
es events.Stream
log *zerolog.Logger
@@ -130,9 +133,10 @@ func New(lu node.PathLookup, bs node.Blobstore, um usermapper.Mapper, trashbin *
scanDebouncer: NewScanDebouncer(o.ScanDebounceDelay, func(item scanItem) {
scanQueue <- item
}),
es: es,
log: log,
Ignorer: ignore.NewIgnorer(o, log),
assimilationFailures: assimilation.NewFailures(),
es: es,
log: log,
Ignorer: ignore.NewIgnorer(o, log),
}
t.idResolver = t.lookup
t.assimilateFunc = t.assimilate
+12 -10
View File
@@ -19,13 +19,13 @@
package walker
import (
"context"
"path/filepath"
provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1"
gateway "github.com/cs3org/go-cs3apis/cs3/gateway/v1beta1"
rpc "github.com/cs3org/go-cs3apis/cs3/rpc/v1beta1"
"github.com/opencloud-eu/reva/v2/pkg/auth"
"github.com/opencloud-eu/reva/v2/pkg/errtypes"
"github.com/opencloud-eu/reva/v2/pkg/rgrpc/todo/pool"
)
@@ -43,7 +43,7 @@ type WalkFunc func(wd string, info *provider.ResourceInfo, err error) error
// Walker is an interface implemented by objects that are able to walk from a dir rooted into the passed path
type Walker interface {
// Walk walks the file tree rooted at root, calling fn for each file or folder in the tree, including the root.
Walk(ctx context.Context, root *provider.ResourceId, fn WalkFunc) error
Walk(session *auth.Session, root *provider.ResourceId, fn WalkFunc) error
}
type revaWalker struct {
@@ -56,14 +56,14 @@ func NewWalker(gatewaySelector pool.Selectable[gateway.GatewayAPIClient]) Walker
}
// Walk walks the file tree rooted at root, calling fn for each file or folder in the tree, including the root.
func (r *revaWalker) Walk(ctx context.Context, root *provider.ResourceId, fn WalkFunc) error {
info, err := r.stat(ctx, root)
func (r *revaWalker) Walk(session *auth.Session, root *provider.ResourceId, fn WalkFunc) error {
info, err := r.stat(session, root)
if err != nil {
return fn("", nil, err)
}
err = r.walkRecursively(ctx, "", info, fn)
err = r.walkRecursively(session, "", info, fn)
if err == filepath.SkipDir {
return nil
@@ -72,7 +72,7 @@ func (r *revaWalker) Walk(ctx context.Context, root *provider.ResourceId, fn Wal
return err
}
func (r *revaWalker) walkRecursively(ctx context.Context, wd string, info *provider.ResourceInfo, fn WalkFunc) error {
func (r *revaWalker) walkRecursively(session *auth.Session, wd string, info *provider.ResourceInfo, fn WalkFunc) error {
if info.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER {
return fn(wd, info, nil)
@@ -83,12 +83,12 @@ func (r *revaWalker) walkRecursively(ctx context.Context, wd string, info *provi
return err
}
list, err := r.readDir(ctx, info.Id)
list, err := r.readDir(session, info.Id)
if err != nil {
return err
}
for _, file := range list {
err = r.walkRecursively(ctx, filepath.Join(wd, info.Path), file, fn)
err = r.walkRecursively(session, filepath.Join(wd, info.Path), file, fn)
if err != nil && (file.Type != provider.ResourceType_RESOURCE_TYPE_CONTAINER || err != filepath.SkipDir) {
return err
}
@@ -97,7 +97,8 @@ func (r *revaWalker) walkRecursively(ctx context.Context, wd string, info *provi
return nil
}
func (r *revaWalker) readDir(ctx context.Context, id *provider.ResourceId) ([]*provider.ResourceInfo, error) {
func (r *revaWalker) readDir(session *auth.Session, id *provider.ResourceId) ([]*provider.ResourceInfo, error) {
ctx := session.Ctx()
gatewayClient, err := r.gatewaySelector.Next()
if err != nil {
return nil, err
@@ -114,7 +115,8 @@ func (r *revaWalker) readDir(ctx context.Context, id *provider.ResourceId) ([]*p
return resp.Infos, nil
}
func (r *revaWalker) stat(ctx context.Context, id *provider.ResourceId) (*provider.ResourceInfo, error) {
func (r *revaWalker) stat(session *auth.Session, id *provider.ResourceId) (*provider.ResourceInfo, error) {
ctx := session.Ctx()
gatewayClient, err := r.gatewaySelector.Next()
if err != nil {
return nil, err
+2 -1
View File
@@ -1367,7 +1367,7 @@ github.com/opencloud-eu/icap-client
# github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20260902170011-45af3945a067
## explicit; go 1.23
github.com/opencloud-eu/libre-graph-api-go
# github.com/opencloud-eu/reva/v2 v2.50.0
# github.com/opencloud-eu/reva/v2 v2.50.1-0.20260921072732-1796c81fa9f2
## explicit; go 1.25.8
github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace
github.com/opencloud-eu/reva/v2/cmd/revad/runtime
@@ -1640,6 +1640,7 @@ github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/timemanager
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/assimilation
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher
github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/watcher/natswatcher
github.com/opencloud-eu/reva/v2/pkg/storage/fs/registry