From fb8af22073cee757a1aeed54cd30aa2fda9615eb Mon Sep 17 00:00:00 2001 From: Florian Schade Date: Mon, 27 Oct 2025 12:01:27 +0100 Subject: [PATCH] chore: bump reva (#1701) * chore: bump reva * enhancement(test): add postprocessing wait helper --- go.mod | 3 +- go.sum | 4 +- tests/acceptance/bootstrap/WebDav.php | 11 ++ .../lowLevelUpload.feature | 1 + .../storage/fs/posix/blobstore/blobstore.go | 149 ++++++++++++------ .../reva/v2/pkg/storage/fs/posix/tree/tree.go | 8 +- .../storage/pkg/decomposedfs/upload/upload.go | 11 ++ vendor/modules.txt | 4 +- 8 files changed, 135 insertions(+), 56 deletions(-) diff --git a/go.mod b/go.mod index abfb7f5a5f..82d40ff888 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( github.com/open-policy-agent/opa v1.9.0 github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76 - github.com/opencloud-eu/reva/v2 v2.39.1-0.20251023125727-bf6471473de6 + github.com/opencloud-eu/reva/v2 v2.39.1-0.20251027101010-5f317aacd03b github.com/opensearch-project/opensearch-go/v4 v4.5.0 github.com/orcaman/concurrent-map v1.0.0 github.com/pkg/errors v0.9.1 @@ -380,7 +380,6 @@ require ( golang.org/x/sys v0.37.0 // indirect golang.org/x/time v0.13.0 // indirect golang.org/x/tools v0.37.0 // indirect - golang.org/x/tools/godoc v0.1.0-deprecated // indirect google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20250929231259-57b25ae835d4 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 08cae8cbf9..4b0919060a 100644 --- a/go.sum +++ b/go.sum @@ -948,8 +948,8 @@ github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89 h1:W1ms+l github.com/opencloud-eu/icap-client v0.0.0-20250930132611-28a2afe62d89/go.mod h1:vigJkNss1N2QEceCuNw/ullDehncuJNFB6mEnzfq9UI= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76 h1:vD/EdfDUrv4omSFjrinT8Mvf+8D7f9g4vgQ2oiDrVUI= github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76/go.mod h1:pzatilMEHZFT3qV7C/X3MqOa3NlRQuYhlRhZTL+hN6Q= -github.com/opencloud-eu/reva/v2 v2.39.1-0.20251023125727-bf6471473de6 h1:L3OSIbnxc12GCJtV6NItBRCjdVCHXXWf+nUOMIIRiJs= -github.com/opencloud-eu/reva/v2 v2.39.1-0.20251023125727-bf6471473de6/go.mod h1:W1dC+qc2D0WPK3vfZ5ZAcKHxbwII3L9MG8ouGqrd1z4= +github.com/opencloud-eu/reva/v2 v2.39.1-0.20251027101010-5f317aacd03b h1:pgVcBeiF5PN8tnQIbPyykLgvgJwqryT0iK/AljKXvoI= +github.com/opencloud-eu/reva/v2 v2.39.1-0.20251027101010-5f317aacd03b/go.mod h1:4CgDhO6Pc+HLdNI7+Rep8N0bc7qP9novdcv764IMpmM= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= diff --git a/tests/acceptance/bootstrap/WebDav.php b/tests/acceptance/bootstrap/WebDav.php index 1b167642ec..e75210179c 100644 --- a/tests/acceptance/bootstrap/WebDav.php +++ b/tests/acceptance/bootstrap/WebDav.php @@ -741,6 +741,17 @@ trait WebDav { $this->setResponse($this->downloadFileWithRange($user, $fileSource, $range)); } + /** + * @When the user waits for :time seconds for postprocessing to finish + * + * @param int $time + * + * @return void + */ + public function waitForCertainSeconds(int $time): void { + \sleep($time); + } + /** * @Then /^user "([^"]*)" using password "([^"]*)" should not be able to download file "([^"]*)"$/ * diff --git a/tests/acceptance/features/coreApiWebdavUploadTUS/lowLevelUpload.feature b/tests/acceptance/features/coreApiWebdavUploadTUS/lowLevelUpload.feature index d2324791fb..3e5780e54c 100644 --- a/tests/acceptance/features/coreApiWebdavUploadTUS/lowLevelUpload.feature +++ b/tests/acceptance/features/coreApiWebdavUploadTUS/lowLevelUpload.feature @@ -50,6 +50,7 @@ Feature: low level tests for upload of chunks | Upload-Metadata | filename ZmlsZS50eHQ= | When user "Alice" sends a chunk to the last created TUS Location with offset "0" and data "123" using the WebDAV API And user "Alice" sends a chunk to the last created TUS Location with offset "3" and data "4567890" using the WebDAV API + And the user waits for "2" seconds for postprocessing to finish And user "Alice" sends a chunk to the last created TUS Location with offset "3" and data "0000000" using the WebDAV API Then the HTTP status code should be "404" And the content of file "/file.txt" for user "Alice" should be "1234567890" diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go index 8501196b5c..d24d162409 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore/blobstore.go @@ -20,13 +20,21 @@ package blobstore import ( - "bufio" + "context" + "fmt" "io" "os" "path/filepath" + "time" + + "github.com/pkg/errors" + "github.com/pkg/xattr" "github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/node" - "github.com/pkg/errors" +) + +const ( + TMPDir = ".oc-tmp" ) // Blobstore provides an interface to an filesystem based blobstore @@ -41,61 +49,106 @@ func New(root string) (*Blobstore, error) { }, nil } -// Upload stores some data in the blobstore under the given key -func (bs *Blobstore) Upload(node *node.Node, source, copyTarget string) error { - path := node.InternalPath() +// Upload is responsible for transferring data from a source file (upload) to its final location; +// the file operation is done atomically using a temporary file followed by a rename +func (bs *Blobstore) Upload(n *node.Node, source, copyTarget string) error { + tempName := filepath.Join(n.SpaceRoot.InternalPath(), TMPDir, filepath.Base(source)) - // preserve the mtime of the file - fi, _ := os.Stat(path) - - file, err := os.Open(source) - if err != nil { - return errors.Wrap(err, "Decomposedfs: posix blobstore: Can not open source file to upload") - } - defer file.Close() - - f, err := os.OpenFile(node.InternalPath(), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0700) - if err != nil { - return errors.Wrapf(err, "could not open blob '%s' for writing", node.InternalPath()) - } - defer f.Close() - - w := bufio.NewWriter(f) - _, err = w.ReadFrom(file) - if err != nil { - return errors.Wrapf(err, "could not write blob '%s'", node.InternalPath()) - } - err = w.Flush() - if err != nil { - return err - } - err = os.Chtimes(path, fi.ModTime(), fi.ModTime()) - if err != nil { + // there is no guarantee that the space root TMPDir exists at this point, so we create the directory if needed + if err := os.MkdirAll(filepath.Dir(tempName), 0700); err != nil { return err } - if copyTarget != "" { - // also "upload" the file to a local path, e.g. for keeping the "current" version of the file - err := os.MkdirAll(filepath.Dir(copyTarget), 0700) - if err != nil { - return err + sourceFile, err := os.Open(source) + if err != nil { + return fmt.Errorf("failed to open source file '%s': %v", source, err) + } + defer func() { + _ = sourceFile.Close() + }() + + tempFile, err := os.OpenFile(tempName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0700) + if err != nil { + return fmt.Errorf("unable to create temp file '%s': %v", tempName, err) + } + + if _, err := tempFile.ReadFrom(sourceFile); err != nil { + return fmt.Errorf("failed to write data from source file '%s' to temp file '%s' - %v", source, tempName, err) + } + + if err := tempFile.Sync(); err != nil { + return fmt.Errorf("failed to sync temp file '%s' - %v", tempName, err) + } + + if err := tempFile.Close(); err != nil { + return fmt.Errorf("failed to close temp file '%s' - %v", tempName, err) + } + + nodeAttributes, err := n.Xattrs(context.Background()) + if err != nil { + return fmt.Errorf("failed to get xattrs for node '%s': %v", n.InternalPath(), err) + } + + var mtime *time.Time + for k, v := range nodeAttributes { + if err := xattr.Set(tempName, k, v); err != nil { + return fmt.Errorf("failed to set xattr '%s' on temp file '%s' - %v", k, tempName, err) } - _, err = file.Seek(0, 0) - if err != nil { - return err + if k == "user.oc.mtime" { + tv, err := time.Parse(time.RFC3339Nano, string(v)) + if err == nil { + mtime = &tv + } + } + } + + // the extended attributes should always contain a mtime, but in case they don't, we fetch it from the node + if mtime == nil { + switch nodeMtime, err := n.GetMTime(context.Background()); { + case err != nil: + return fmt.Errorf("failed to get mtime for node '%s' - %v", n.InternalPath(), err) + default: + mtime = &nodeMtime } - copyFile, err := os.OpenFile(copyTarget, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) - if err != nil { - return errors.Wrapf(err, "could not open copy target '%s' for writing", copyTarget) - } - defer copyFile.Close() + } - _, err = copyFile.ReadFrom(file) - if err != nil { - return errors.Wrapf(err, "could not write blob copy of '%s' to '%s'", node.InternalPath(), copyTarget) - } + // etags rely on the id and the mtime, so we need to ensure the mtime is set correctly + if err := os.Chtimes(tempName, *mtime, *mtime); err != nil { + return fmt.Errorf("failed to set mtime on temp file '%s' - %v", tempName, err) + } + + // atomically move the file to its final location, + // on Windows systems (unsupported oc os) os.Rename is not atomic + if err := os.Rename(tempName, n.InternalPath()); err != nil { + return fmt.Errorf("failed to move temp file '%s' to node '%s' - %v", tempName, n.InternalPath(), err) + } + + // upload successfully, now handle the copy target if set + if copyTarget == "" { + return nil + } + + // also "upload" the file to a local path, e.g., for keeping the "current" version of the file + if err := os.MkdirAll(filepath.Dir(copyTarget), 0700); err != nil { + return err + } + + if _, err := sourceFile.Seek(0, 0); err != nil { + return err + } + + copyFile, err := os.OpenFile(copyTarget, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0600) + if err != nil { + return errors.Wrapf(err, "could not open copy target '%s' for writing", copyTarget) + } + defer func() { + _ = copyFile.Close() + }() + + if _, err := copyFile.ReadFrom(sourceFile); err != nil { + return errors.Wrapf(err, "could not write blob copy of '%s' to '%s'", n.InternalPath(), copyTarget) } return nil diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go index 936a4ed29d..1c48eeb732 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/tree/tree.go @@ -39,9 +39,11 @@ import ( "golang.org/x/sync/errgroup" provider "github.com/cs3org/go-cs3apis/cs3/storage/provider/v1beta1" + "github.com/opencloud-eu/reva/v2/pkg/appctx" "github.com/opencloud-eu/reva/v2/pkg/errtypes" "github.com/opencloud-eu/reva/v2/pkg/events" + "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/blobstore" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/lookup" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/options" "github.com/opencloud-eu/reva/v2/pkg/storage/fs/posix/trashbin" @@ -724,6 +726,10 @@ func (t *Tree) isIndex(path string) bool { return strings.HasPrefix(path, filepath.Join(t.options.Root, "indexes")) } +func (t *Tree) isTemporary(path string) bool { + return path == blobstore.TMPDir +} + func (t *Tree) isRootPath(path string) bool { return path == t.options.Root || path == t.personalSpacesRoot || @@ -736,7 +742,7 @@ func (t *Tree) isSpaceRoot(path string) bool { } func (t *Tree) isInternal(path string) bool { - return t.isIndex(path) || strings.Contains(path, lookup.MetadataDir) + return t.isIndex(path) || strings.Contains(path, lookup.MetadataDir) || t.isTemporary(path) } func isLockFile(path string) bool { diff --git a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go index 3027893e9e..c847ffd743 100644 --- a/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go +++ b/vendor/github.com/opencloud-eu/reva/v2/pkg/storage/pkg/decomposedfs/upload/upload.go @@ -292,6 +292,17 @@ func (session *DecomposedFsSession) Finalize(ctx context.Context) (err error) { revisionNode := node.New(session.SpaceID(), session.NodeID(), "", "", session.Size(), session.ID(), provider.ResourceType_RESOURCE_TYPE_FILE, session.SpaceOwner(), session.store.lu) + switch spaceRoot, err := session.store.lu.NodeFromSpaceID(ctx, session.SpaceID()); { + case err != nil: + return fmt.Errorf("failed to get space root for space id %s: %v", session.SpaceID(), err) + case spaceRoot == nil: + return fmt.Errorf("space root for space id %s not found", session.SpaceID()) + case spaceRoot.InternalPath() == "": + return fmt.Errorf("space root for space id %s has no valid internal path", session.SpaceID()) + default: + revisionNode.SpaceRoot = spaceRoot + } + // lock the node before writing the blob unlock, err := session.store.lu.MetadataBackend().Lock(revisionNode) if err != nil { diff --git a/vendor/modules.txt b/vendor/modules.txt index c7c1f133ab..23bf238cb5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1340,7 +1340,7 @@ github.com/opencloud-eu/icap-client # github.com/opencloud-eu/libre-graph-api-go v1.0.8-0.20250724122329-41ba6b191e76 ## explicit; go 1.18 github.com/opencloud-eu/libre-graph-api-go -# github.com/opencloud-eu/reva/v2 v2.39.1-0.20251023125727-bf6471473de6 +# github.com/opencloud-eu/reva/v2 v2.39.1-0.20251027101010-5f317aacd03b ## explicit; go 1.24.1 github.com/opencloud-eu/reva/v2/cmd/revad/internal/grace github.com/opencloud-eu/reva/v2/cmd/revad/runtime @@ -2516,8 +2516,6 @@ golang.org/x/tools/internal/stdlib golang.org/x/tools/internal/typeparams golang.org/x/tools/internal/typesinternal golang.org/x/tools/internal/versions -# golang.org/x/tools/godoc v0.1.0-deprecated -## explicit; go 1.24.0 # google.golang.org/genproto v0.0.0-20250303144028-a0af3efb3deb ## explicit; go 1.23.0 google.golang.org/genproto/protobuf/field_mask