From 87d0b59a512e421a7456956dc705ecf41500f9ac Mon Sep 17 00:00:00 2001 From: Leon Brocard Date: Sat, 2 May 2026 12:28:30 +0100 Subject: [PATCH] cmd/serve/s3: return object listings in key order The S3 ListObjects response from `rclone serve s3` was sorting object contents by modification time instead of object key. This made the listing order incompatible with S3 clients which expect lexicographic key ordering. In particular, `aws s3 sync` assumes both source and destination iterators are ordered by key. With the old modtime ordering it could misidentify files as missing or outdated and re-download objects that were already up to date. Change the pager to sort returned objects by key and add a regression test which uses keys and modtimes arranged so the old behaviour would fail. Fixes #9002 --- cmd/serve/s3/pager.go | 4 ++-- cmd/serve/s3/pager_test.go | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 cmd/serve/s3/pager_test.go diff --git a/cmd/serve/s3/pager.go b/cmd/serve/s3/pager.go index 953f947d2..94c8e3cba 100644 --- a/cmd/serve/s3/pager.go +++ b/cmd/serve/s3/pager.go @@ -13,9 +13,9 @@ func (db *s3Backend) pager(list *gofakes3.ObjectList, page gofakes3.ListBucketPa sort.Slice(list.CommonPrefixes, func(i, j int) bool { return list.CommonPrefixes[i].Prefix < list.CommonPrefixes[j].Prefix }) - // sort by modtime + // sort by key name sort.Slice(list.Contents, func(i, j int) bool { - return list.Contents[i].LastModified.Before(list.Contents[j].LastModified.Time) + return list.Contents[i].Key < list.Contents[j].Key }) tokens := page.MaxKeys if tokens == 0 { diff --git a/cmd/serve/s3/pager_test.go b/cmd/serve/s3/pager_test.go new file mode 100644 index 000000000..e7166b1aa --- /dev/null +++ b/cmd/serve/s3/pager_test.go @@ -0,0 +1,32 @@ +package s3 + +import ( + "testing" + "time" + + "github.com/rclone/gofakes3" +) + +func TestPagerSortsContentsByKey(t *testing.T) { + list := gofakes3.NewObjectList() + list.Add(&gofakes3.Content{ + Key: "b.txt", + LastModified: gofakes3.NewContentTime(time.Unix(100, 0)), + }) + list.Add(&gofakes3.Content{ + Key: "a.txt", + LastModified: gofakes3.NewContentTime(time.Unix(200, 0)), + }) + + got, err := (&s3Backend{}).pager(list, gofakes3.ListBucketPage{MaxKeys: 2}) + if err != nil { + t.Fatal(err) + } + if len(got.Contents) != 2 { + t.Fatalf("expected 2 contents, got %d", len(got.Contents)) + } + + if got.Contents[0].Key != "a.txt" || got.Contents[1].Key != "b.txt" { + t.Fatalf("expected lexicographic key order [a.txt b.txt], got [%s %s]", got.Contents[0].Key, got.Contents[1].Key) + } +}