serve s3: add --directory-markers to list every directory as a marker

Clients which rely on directory markers, such as rclone's s3 backend
with --s3-directory-markers, need the markers of directories holding
other keys listed, which serve s3 only does while it remembers the
marker was put.

With --directory-markers every directory is listed as a directory
marker, and isn't removed when the last key below it is deleted.

See #10082

cmd/serve/s3/backend.go

@@ -257,9 +257,12 @@ func (t *tenant) storeDirMarker(fp string) {
 	t.meta.Add(fp, objectMeta{dirMarker: true})
 }
 
-// hasDirMarker reports whether a directory marker was put for the
-// directory at fp.
+// hasDirMarker reports whether the directory at fp has a directory
+// marker: one was put for it, or --directory-markers is set.
 func (t *tenant) hasDirMarker(fp string) bool {
+	if t.b.s.opt.DirectoryMarkers {
+		return true
+	}
 	om, ok := t.meta.Get(fp)
 	return ok && om.dirMarker
 }

cmd/serve/s3/list_test.go

@@ -24,6 +24,11 @@ import (
 // newListBackend serves a temporary directory populated with the given files,
 // whose paths are relative to the serve root and so start with a bucket name.
 func newListBackend(t *testing.T, files ...string) *s3Backend {
+	return newListBackendOpt(t, Opt, files...)
+}
+
+// newListBackendOpt is newListBackend with the options opt.
+func newListBackendOpt(t *testing.T, opt Options, files ...string) *s3Backend {
 	fstest.Initialise()
 	ctx := context.Background()
 	root := t.TempDir()
@@ -36,7 +41,6 @@ func newListBackend(t *testing.T, files ...string) *s3Backend {
 	f, err := fs.NewFs(ctx, root)
 	require.NoError(t, err)
 
-	opt := Opt
 	opt.HTTP.ListenAddr = []string{endpoint}
 	w, err := newServer(ctx, f, &opt, &vfscommon.Opt, &proxy.Opt)
 	require.NoError(t, err)
@@ -422,3 +426,29 @@ func TestListStorageClass(t *testing.T) {
 		"plain":    gofakes3.StorageStandard,
 	}, got)
 }
+
+// TestListDirectoryMarkers checks every directory is listed as a
+// directory marker with --directory-markers.
+func TestListDirectoryMarkers(t *testing.T) {
+	opt := Opt
+	opt.DirectoryMarkers = true
+	b := newListBackendOpt(t, opt, "bucket/implicit/file", "bucket/a/b/c")
+	ctx := context.Background()
+	all := func() []string {
+		return listPages(t, b, listPrefix("", ""), 1000)
+	}
+
+	want := []string{"a/", "a/b/", "a/b/c", "implicit/", "implicit/file"}
+	assert.Equal(t, want, all())
+	assert.Equal(t, want, listPages(t, b, listPrefix("", ""), 1))
+	keys, _ := listOnce(t, b, listPrefix("a/", "/"), gofakes3.ListBucketPage{})
+	assert.Equal(t, []string{"a/", "a/b/"}, keys)
+	keys, _ = listOnce(t, b, listPrefix("", "b"), gofakes3.ListBucketPage{})
+	assert.Equal(t, []string{"a/", "a/b", "implicit/", "implicit/file"}, keys)
+
+	// The directories are kept when the keys below them are deleted
+	require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/c"))
+	assert.Equal(t, []string{"a/", "a/b/", "implicit/", "implicit/file"}, all())
+	require.NoError(t, b.deleteObject(ctx, "bucket", "a/b/"))
+	assert.Equal(t, []string{"a/", "implicit/", "implicit/file"}, all())
+}

cmd/serve/s3/s3.go

@@ -57,6 +57,10 @@ var OptionsInfo = fs.Options{{
 	Name:    "metadata_max_objects",
 	Default: 100000,
 	Help:    "Maximum number of objects of a user whose metadata is kept in memory, 0 for unlimited",
+}, {
+	Name:    "directory_markers",
+	Default: false,
+	Help:    "List every directory as a directory marker object",
 }, {
 	Name:    "multipart_expiry",
 	Default: fs.Duration(24 * time.Hour),
@@ -78,6 +82,7 @@ type Options struct {
 	MultipartMaxUploads           int           `config:"multipart_max_uploads"`
 	MetadataMaxObjects            int           `config:"metadata_max_objects"`
 	MultipartExpiry               fs.Duration   `config:"multipart_expiry"`
+	DirectoryMarkers              bool          `config:"directory_markers"`
 	Auth                          httplib.AuthConfig
 	HTTP                          httplib.Config
 }

cmd/serve/s3/serve_s3.md

@@ -502,6 +502,13 @@ the marker of a directory holding other keys is no longer listed.
 While a marker is remembered its directory isn't removed when the
 last key below it is deleted.
 
+With `--directory-markers` every directory is listed as a directory
+marker, so the markers are listed after a restart too, as needed by
+clients which rely on them, such as rclone's s3 backend with
+`--s3-directory-markers`. As every directory has a marker, a directory
+isn't removed when the last key below it is deleted, and deleting the
+marker of a directory which holds other keys has no effect.
+
 As keys are file paths, a key can't be both an object and a directory
 holding other objects, for example `a/b` and `a/b/c`. Writing an
 object which would need this fails with `400 Bad Request`
@@ -528,7 +535,7 @@ ignored.
 | Authentication | Partial | AWS Signature Version 4, in headers, presigned URLs and browser form (`POST`) uploads. Signature Version 2 isn't supported. |
 | Anonymous access | Partial | All requests are allowed without `--auth-key` or `--auth-proxy`, and none are with them, as buckets and objects can't be made public. |
 | Metadata | Partial | `x-amz-meta-*`, the storage class and the standard headers such as `Content-Type` are kept in memory only (see [Limitations](#limitations)), except the modification time. The storage class is only recorded, so doesn't change how the object is stored. |
-| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory (see [Object names](#object-names)). |
+| Directory markers | Partial | Keys ending in `/` are stored as directories, so can't hold data. A marker put for a directory holding other keys is only listed while it is remembered in memory, unless `--directory-markers` is set (see [Object names](#object-names)). |
 | Checksums | Partial | The `x-amz-checksum-*` of a `PutObject` (CRC32, CRC32C, CRC64NVME, SHA1 or SHA256) is checked and returned with the metadata. The checksums of multipart uploads aren't supported. |
 | `GetObjectAttributes` | Partial | Everything but the checksums of multipart uploads. |
 | Object tagging | Partial | `GetObjectTagging`, `PutObjectTagging` and `DeleteObjectTagging` work, but tags are kept in memory with the metadata. |