diff --git a/backend/dropbox/dropbox.go b/backend/dropbox/dropbox.go index 1d0489f1c..0f7815a22 100644 --- a/backend/dropbox/dropbox.go +++ b/backend/dropbox/dropbox.go @@ -222,17 +222,14 @@ All other operations will be disabled.`, }, { Name: "shared_folders", Help: `Instructs rclone to work on shared folders. - -When this flag is used with no path only the List operation is supported and -all available shared folders will be listed. If you specify a path the first part -will be interpreted as the name of shared folder. Rclone will then try to mount this -shared to the root namespace. On success shared folder rclone proceeds normally. -The shared folder is now pretty much a normal folder and all normal operations -are supported. -Note that we don't unmount the shared folder afterwards so the ---dropbox-shared-folders can be omitted after the first use of a particular -shared folder. +When this flag is used with no path only the List operation is supported and +all available shared folders will be listed. If you specify a path the first part +will be interpreted as the name of shared folder and rclone will list and access +files within it using the shared folder's namespace. This works for all shared +folder types including team folders. + +Note that write operations (upload, delete, etc.) are not supported in this mode. See also --dropbox-root-namespace for an alternative way to work with shared folders.`, @@ -336,6 +333,7 @@ type Fs struct { ns string // The namespace we are using or "" for none batcher *batcher.Batcher[*files.UploadSessionFinishArg, *files.FileMetadata] exportExts []exportExtension + cfg dropbox.Config // SDK config for creating per-namespace clients } type exportType int @@ -358,11 +356,23 @@ type Object struct { bytes int64 // size of the object modTime time.Time // time it was last modified hash string // content_hash of the object + nsID string // shared folder namespace ID, if applicable exportType exportType exportAPIFormat exportAPIFormat } +// fileSrv returns the files client to use for this object. +// If the object belongs to a shared folder, it returns a +// namespace-scoped client; otherwise it returns the default client. +func (o *Object) fileSrv() files.Client { + if o.nsID != "" { + nsCfg := o.fs.cfg.WithNamespaceID(o.nsID) + return files.New(nsCfg) + } + return o.fs.srv +} + // Name of the remote (as passed into NewFs) func (f *Fs) Name() string { return f.name @@ -537,6 +547,7 @@ func NewFs(ctx context.Context, name, root string, m configmap.Mapper) (fs.Fs, e cfg.AsMemberID = memberIDs[0].MemberInfo.Profile.MemberProfile.TeamMemberId } + f.cfg = cfg f.srv = files.New(cfg) f.svc = files.New(ucfg) f.sharing = sharing.New(cfg) @@ -830,9 +841,53 @@ func (f *Fs) NewObject(ctx context.Context, remote string) (fs.Object, error) { if f.opt.SharedFiles { return f.findSharedFile(ctx, remote) } + if f.opt.SharedFolders { + return f.newSharedFolderObject(ctx, remote) + } return f.newObjectWithInfo(ctx, remote, nil) } +// newSharedFolderObject finds an Object inside a shared folder by +// using the shared folder's namespace ID to resolve the path. +func (f *Fs) newSharedFolderObject(ctx context.Context, remote string) (fs.Object, error) { + firstDir, subPath, ok := strings.Cut(remote, "/") + if !ok { + return nil, fs.ErrorObjectNotFound + } + id, err := f.findSharedFolder(ctx, firstDir) + if err != nil { + return nil, err + } + nsCfg := f.cfg.WithNamespaceID(id) + nsSrv := files.New(nsCfg) + + filePath := f.opt.Enc.FromStandardPath("/" + subPath) + var entry files.IsMetadata + err = f.pacer.Call(func() (bool, error) { + entry, err = nsSrv.GetMetadata(&files.GetMetadataArg{Path: filePath}) + return shouldRetry(ctx, err) + }) + if err != nil { + switch e := err.(type) { + case files.GetMetadataAPIError: + if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound { + return nil, fs.ErrorObjectNotFound + } + } + return nil, err + } + fileInfo, ok := entry.(*files.FileMetadata) + if !ok { + return nil, fs.ErrorObjectNotFound + } + o, err := f.newObjectWithInfo(ctx, remote, fileInfo) + if err != nil { + return nil, err + } + o.(*Object).nsID = id + return o, nil +} + // listSharedFolders lists all available shared folders mounted and not mounted // we'll need the id later so we have to return them in original format func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) error) (err error) { @@ -879,6 +934,93 @@ func (f *Fs) listSharedFolders(ctx context.Context, callback func(fs.DirEntry) e return nil } +// listDir lists the contents of a directory using the given files client. +// root is the Dropbox API path to list, dir is the rclone directory prefix for results. +// nsID is the shared folder namespace ID to set on created objects (empty for normal listings). +func (f *Fs) listDir(ctx context.Context, srv files.Client, root string, dir string, nsID string, list *list.Helper) (err error) { + started := false + var res *files.ListFolderResult + for { + if !started { + arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root)) + arg.Recursive = false + arg.Limit = 1000 + if root == "/" || root == "" { + arg.Path = "" // Specify root folder as empty string + } + err = f.pacer.Call(func() (bool, error) { + res, err = srv.ListFolder(arg) + return shouldRetry(ctx, err) + }) + if err != nil { + switch e := err.(type) { + case files.ListFolderAPIError: + if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound { + err = fs.ErrorDirNotFound + } + } + return err + } + started = true + } else { + arg := files.ListFolderContinueArg{ + Cursor: res.Cursor, + } + err = f.pacer.Call(func() (bool, error) { + res, err = srv.ListFolderContinue(&arg) + return shouldRetry(ctx, err) + }) + if err != nil { + return fmt.Errorf("list continue: %w", err) + } + } + for _, entry := range res.Entries { + var fileInfo *files.FileMetadata + var folderInfo *files.FolderMetadata + var metadata *files.Metadata + switch info := entry.(type) { + case *files.FolderMetadata: + folderInfo = info + metadata = &info.Metadata + case *files.FileMetadata: + fileInfo = info + metadata = &info.Metadata + default: + fs.Errorf(f, "Unknown type %T", entry) + continue + } + + // Only the last element is reliably cased in PathDisplay + entryPath := metadata.PathDisplay + leaf := f.opt.Enc.ToStandardName(path.Base(entryPath)) + remote := path.Join(dir, leaf) + if folderInfo != nil { + d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id) + err = list.Add(d) + if err != nil { + return err + } + } else if fileInfo != nil { + o, err := f.newObjectWithInfo(ctx, remote, fileInfo) + if err != nil { + return err + } + o.(*Object).nsID = nsID + if o.(*Object).exportType.listable() { + err = list.Add(o) + if err != nil { + return err + } + } + } + } + if !res.HasMore { + break + } + } + return list.Flush() +} + // findSharedFolder find the id for a given shared folder name // somewhat annoyingly there is no endpoint to query a shared folder by it's name // so our only option is to iterate over all shared folders @@ -1018,99 +1160,34 @@ func (f *Fs) ListP(ctx context.Context, dir string, callback fs.ListRCallback) ( return list.Flush() } if f.opt.SharedFolders { - err := f.listSharedFolders(ctx, list.Add) + if dir == "" { + err := f.listSharedFolders(ctx, list.Add) + if err != nil { + return err + } + return list.Flush() + } + // For subdirectories, use the shared folder's namespace + // to list its contents via the files API. + firstDir, subDir, _ := strings.Cut(dir, "/") + id, err := f.findSharedFolder(ctx, firstDir) if err != nil { return err } - return list.Flush() + nsCfg := f.cfg.WithNamespaceID(id) + nsSrv := files.New(nsCfg) + root := "" + if subDir != "" { + root = "/" + subDir + } + return f.listDir(ctx, nsSrv, root, dir, id, list) } root := f.slashRoot if dir != "" { root += "/" + dir } - - started := false - var res *files.ListFolderResult - for { - if !started { - arg := files.NewListFolderArg(f.opt.Enc.FromStandardPath(root)) - arg.Recursive = false - arg.Limit = 1000 - - if root == "/" { - arg.Path = "" // Specify root folder as empty string - } - err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.ListFolder(arg) - return shouldRetry(ctx, err) - }) - if err != nil { - switch e := err.(type) { - case files.ListFolderAPIError: - if e.EndpointError != nil && e.EndpointError.Path != nil && e.EndpointError.Path.Tag == files.LookupErrorNotFound { - err = fs.ErrorDirNotFound - } - } - return err - } - started = true - } else { - arg := files.ListFolderContinueArg{ - Cursor: res.Cursor, - } - err = f.pacer.Call(func() (bool, error) { - res, err = f.srv.ListFolderContinue(&arg) - return shouldRetry(ctx, err) - }) - if err != nil { - return fmt.Errorf("list continue: %w", err) - } - } - for _, entry := range res.Entries { - var fileInfo *files.FileMetadata - var folderInfo *files.FolderMetadata - var metadata *files.Metadata - switch info := entry.(type) { - case *files.FolderMetadata: - folderInfo = info - metadata = &info.Metadata - case *files.FileMetadata: - fileInfo = info - metadata = &info.Metadata - default: - fs.Errorf(f, "Unknown type %T", entry) - continue - } - - // Only the last element is reliably cased in PathDisplay - entryPath := metadata.PathDisplay - leaf := f.opt.Enc.ToStandardName(path.Base(entryPath)) - remote := path.Join(dir, leaf) - if folderInfo != nil { - d := fs.NewDir(remote, time.Time{}).SetID(folderInfo.Id) - err = list.Add(d) - if err != nil { - return err - } - } else if fileInfo != nil { - o, err := f.newObjectWithInfo(ctx, remote, fileInfo) - if err != nil { - return err - } - if o.(*Object).exportType.listable() { - err = list.Add(o) - if err != nil { - return err - } - } - } - } - if !res.HasMore { - break - } - } - return list.Flush() + return f.listDir(ctx, f.srv, root, dir, "", list) } // Put the object @@ -1868,8 +1945,9 @@ func (o *Object) export(ctx context.Context) (in io.ReadCloser, err error) { arg := files.ExportArg{Path: o.id, ExportFormat: string(o.exportAPIFormat)} var exportResult *files.ExportResult + srv := o.fileSrv() err = o.fs.pacer.Call(func() (bool, error) { - exportResult, in, err = o.fs.srv.Export(&arg) + exportResult, in, err = srv.Export(&arg) return shouldRetry(ctx, err) }) if err != nil { @@ -1910,8 +1988,9 @@ func (o *Object) Open(ctx context.Context, options ...fs.OpenOption) (in io.Read Path: o.id, ExtraHeaders: headers, } + srv := o.fileSrv() err = o.fs.pacer.Call(func() (bool, error) { - _, in, err = o.fs.srv.Download(&arg) + _, in, err = srv.Download(&arg) return shouldRetry(ctx, err) }) diff --git a/backend/dropbox/dropbox_internal_test.go b/backend/dropbox/dropbox_internal_test.go index 431660ab3..32dda4505 100644 --- a/backend/dropbox/dropbox_internal_test.go +++ b/backend/dropbox/dropbox_internal_test.go @@ -5,9 +5,13 @@ import ( "io" "strings" "testing" + "time" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/async" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/sharing" + "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fstest/fstests" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -95,8 +99,137 @@ func (f *Fs) InternalTestPaperExport(t *testing.T) { require.Contains(t, text, excerpt) } } +func (f *Fs) InternalTestSharedFolderList(t *testing.T) { + ctx := context.Background() + + // Create a subfolder and a file inside it using the normal API. + // The test Fs root is something like /rclone-test-xxxxx so the + // shared folder will be the test root directory itself. + testDir := f.slashRoot + testFileName := "shared-folder-test-file.txt" + testFilePath := testDir + "/" + testFileName + + // Upload a small test file + content := "hello shared folder" + uploadArg := files.NewUploadArg(testFilePath) + uploadArg.Mode = &files.WriteMode{Tagged: dropbox.Tagged{Tag: files.WriteModeOverwrite}} + var err error + err = f.pacer.Call(func() (bool, error) { + _, err = f.srv.Upload(uploadArg, strings.NewReader(content)) + return shouldRetry(ctx, err) + }) + require.NoError(t, err) + defer func() { + // Clean up test file + _, _ = f.srv.DeleteV2(files.NewDeleteArg(testFilePath)) + }() + + // Share the test root folder + folderName := f.opt.Enc.ToStandardName(strings.TrimPrefix(testDir, "/")) + shareArg := sharing.NewShareFolderArg(testDir) + var sharedFolderID string + var shareLaunch *sharing.ShareFolderLaunch + err = f.pacer.Call(func() (bool, error) { + shareLaunch, err = f.sharing.ShareFolder(shareArg) + return shouldRetry(ctx, err) + }) + if err != nil { + // If the folder is already shared, find its ID + sharedFolderID, err = f.findSharedFolder(ctx, folderName) + require.NoError(t, err, "ShareFolder failed and couldn't find existing share") + } else { + switch shareLaunch.Tag { + case sharing.ShareFolderLaunchComplete: + sharedFolderID = shareLaunch.Complete.SharedFolderId + case sharing.ShareFolderLaunchAsyncJobId: + // Poll for completion + pollArg := async.PollArg{AsyncJobId: shareLaunch.AsyncJobId} + for range 30 { + time.Sleep(time.Second) + var status *sharing.ShareFolderJobStatus + err = f.pacer.Call(func() (bool, error) { + status, err = f.sharing.CheckShareJobStatus(&pollArg) + return shouldRetry(ctx, err) + }) + require.NoError(t, err) + if status.Tag == sharing.ShareFolderJobStatusComplete { + sharedFolderID = status.Complete.SharedFolderId + break + } + } + require.NotEmpty(t, sharedFolderID, "share folder job did not complete") + } + } + require.NotEmpty(t, sharedFolderID) + + defer func() { + // Unshare the folder when done + unshareArg := sharing.NewUnshareFolderArg(sharedFolderID) + unshareArg.LeaveACopy = true + _ = f.pacer.Call(func() (bool, error) { + _, err = f.sharing.UnshareFolder(unshareArg) + return shouldRetry(ctx, err) + }) + }() + + // Now test listing with shared_folders mode. + // Create a copy of the Fs with SharedFolders enabled. + sharedFs := *f + sharedFs.opt.SharedFolders = true + + // Test 1: listing root should include the shared folder + var rootEntries fs.DirEntries + err = sharedFs.ListP(ctx, "", func(entries fs.DirEntries) error { + rootEntries = append(rootEntries, entries...) + return nil + }) + require.NoError(t, err) + + found := false + for _, entry := range rootEntries { + if entry.Remote() == folderName { + found = true + break + } + } + assert.True(t, found, "shared folder %q not found in root listing (have %v)", folderName, rootEntries) + + // Test 2: listing the shared folder should show its contents + var dirEntries fs.DirEntries + err = sharedFs.ListP(ctx, folderName, func(entries fs.DirEntries) error { + dirEntries = append(dirEntries, entries...) + return nil + }) + require.NoError(t, err) + + foundFile := false + expectedRemote := folderName + "/" + testFileName + for _, entry := range dirEntries { + if entry.Remote() == expectedRemote { + foundFile = true + break + } + } + assert.True(t, foundFile, "file %q not found in shared folder listing (have %v)", expectedRemote, dirEntries) + + // Test 3: NewObject should find a file inside a shared folder + obj, err := sharedFs.NewObject(ctx, expectedRemote) + require.NoError(t, err, "NewObject failed for shared folder file") + assert.Equal(t, expectedRemote, obj.Remote()) + assert.Equal(t, int64(len(content)), obj.Size()) + + // Test 4: Open should be able to read the file contents + rc, err := obj.Open(ctx) + require.NoError(t, err, "Open failed for shared folder file") + buf, err := io.ReadAll(rc) + require.NoError(t, rc.Close()) + require.NoError(t, err) + assert.Equal(t, content, string(buf)) +} + func (f *Fs) InternalTest(t *testing.T) { t.Run("PaperExport", f.InternalTestPaperExport) + t.Run("SharedFolderList", f.InternalTestSharedFolderList) } var _ fstests.InternalTester = (*Fs)(nil)