diff --git a/cmd/serve/s3/backend.go b/cmd/serve/s3/backend.go index a1e86ad32..c8ac6b74c 100644 --- a/cmd/serve/s3/backend.go +++ b/cmd/serve/s3/backend.go @@ -257,9 +257,12 @@ func (t *tenant) storeDirMarker(fp string) { t.meta.Add(fp, objectMeta{dirMarker: true}) } -// hasDirMarker reports whether a directory marker was put for the -// directory at fp. +// hasDirMarker reports whether the directory at fp has a directory +// marker: one was put for it, or --directory-markers is set. func (t *tenant) hasDirMarker(fp string) bool { + if t.b.s.opt.DirectoryMarkers { + return true + } om, ok := t.meta.Get(fp) return ok && om.dirMarker } diff --git a/cmd/serve/s3/list_test.go b/cmd/serve/s3/list_test.go index 3d0b25c9b..a5839b5ef 100644 --- a/cmd/serve/s3/list_test.go +++ b/cmd/serve/s3/list_test.go @@ -24,6 +24,11 @@ import ( // newListBackend serves a temporary directory populated with the given files, // whose paths are relative to the serve root and so start with a bucket name. func newListBackend(t *testing.T, files ...string) *s3Backend { + return newListBackendOpt(t, Opt, files...) +} + +// newListBackendOpt is newListBackend with the options opt. +func newListBackendOpt(t *testing.T, opt Options, files ...string) *s3Backend { fstest.Initialise() ctx := context.Background() root := t.TempDir() @@ -36,7 +41,6 @@ func newListBackend(t *testing.T, files ...string) *s3Backend { f, err := fs.NewFs(ctx, root) require.NoError(t, err) - opt := Opt opt.HTTP.ListenAddr = []string{endpoint} w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt) require.NoError(t, err) @@ -422,3 +426,29 @@ func TestListStorageClass(t *testing.T) { "plain": gofakes3.StorageStandard, }, got) } + +// TestListDirectoryMarkers checks every directory is listed as a +// directory marker with --directory-markers. +func TestListDirectoryMarkers(t *testing.T) { + opt := Opt + opt.DirectoryMarkers = true + b := newListBackendOpt(t, opt, "bucket/implicit/file", "bucket/a/b/c") + ctx := context.Background() + all := func() []string { + return listPages(t, b, listPrefix("", ""), 1000) + } + + want := []string{"a/", "a/b/", "a/b/c", "implicit/", "implicit/file"} + assert.Equal(t, want, all()) + assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1)) + keys, _ := listOnce(t, b, listPrefix("a/", "/"), gofakes3.ListBucketPage{}) + assert.Equal(t, []string{"a/", "a/b/"}, keys) + keys, _ = listOnce(t, b, listPrefix("", "b"), gofakes3.ListBucketPage{}) + assert.Equal(t, []string{"a/", "a/b", "implicit/", "implicit/file"}, keys) + + // The directories are kept when the keys below them are deleted + require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/c")) + assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all()) + require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/")) + assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all()) +} diff --git a/cmd/serve/s3/s3.go b/cmd/serve/s3/s3.go index 47a65a04b..c69812ee5 100644 --- a/cmd/serve/s3/s3.go +++ b/cmd/serve/s3/s3.go @@ -57,6 +57,10 @@ var OptionsInfo = fs.Options{{ Name: "metadata_max_objects", Default: 100000, Help: "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited", +}, { + Name: "directory_markers", + Default: false, + Help: "List every directory as a directory marker object", }, { Name: "multipart_expiry", Default: fs.Duration(24 * time.Hour), @@ -78,6 +82,7 @@ type Options struct { MultipartMaxUploads int `config:"multipart_max_uploads"` MetadataMaxObjects int `config:"metadata_max_objects"` MultipartExpiry fs.Duration `config:"multipart_expiry"` + DirectoryMarkers bool `config:"directory_markers"` Auth httplib.AuthConfig HTTP httplib.Config } diff --git a/cmd/serve/s3/serve_s3.md b/cmd/serve/s3/serve_s3.md index 6711745b0..c0155aa44 100644 --- a/cmd/serve/s3/serve_s3.md +++ b/cmd/serve/s3/serve_s3.md @@ -502,6 +502,13 @@ the marker of a directory holding other keys is no longer listed. While a marker is remembered its directory isn't removed when the last key below it is deleted. +With `--directory-markers` every directory is listed as a directory +marker, so the markers are listed after a restart too, as needed by +clients which rely on them, such as rclone's s3 backend with +`--s3-directory-markers`. As every directory has a marker, a directory +isn't removed when the last key below it is deleted, and deleting the +marker of a directory which holds other keys has no effect. + As keys are file paths, a key can't be both an object and a directory holding other objects, for example `a/b` and `a/b/c`. Writing an object which would need this fails with `400 Bad Request` @@ -528,7 +535,7 @@ ignored. | Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. | | Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. | | Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. | -| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). | +| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory, unless `--directory-markers` is set (see [Object names](#object-names)). | | Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. | | `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. | | Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |