From db5ac0a4e2d5e8ce7f6523aab28636c167c054af Mon Sep 17 00:00:00 2001 From: Michael Barz Date: Thu, 10 Jul 2025 09:20:39 +0200 Subject: [PATCH] change: set better decomposedS3 defaults for multipart upload (#1200) --- services/storage-users/pkg/config/config.go | 2 +- services/storage-users/pkg/config/defaults/defaultconfig.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/services/storage-users/pkg/config/config.go b/services/storage-users/pkg/config/config.go index 570045f745..adb5308b78 100644 --- a/services/storage-users/pkg/config/config.go +++ b/services/storage-users/pkg/config/config.go @@ -155,7 +155,7 @@ type DecomposedS3Driver struct { DisableContentSha256 bool `yaml:"put_object_disable_content_sha254" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_DISABLE_CONTENT_SHA256" desc:"Disable sending content sha256 when copying objects to S3." introductionVersion:"1.0.0"` DisableMultipart bool `yaml:"put_object_disable_multipart" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_DISABLE_MULTIPART" desc:"Disable multipart uploads when copying objects to S3" introductionVersion:"1.0.0"` SendContentMd5 bool `yaml:"put_object_send_content_md5" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_SEND_CONTENT_MD5" desc:"Send a Content-MD5 header when copying objects to S3." introductionVersion:"1.0.0"` - ConcurrentStreamParts bool `yaml:"put_object_concurrent_stream_parts" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_CONCURRENT_STREAM_PARTS" desc:"Always precreate parts when copying objects to S3." introductionVersion:"1.0.0"` + ConcurrentStreamParts bool `yaml:"put_object_concurrent_stream_parts" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_CONCURRENT_STREAM_PARTS" desc:"Always precreate parts when copying objects to S3. This is not recommended. It uses a memory buffer. If true, PartSize needs to be set." introductionVersion:"1.0.0"` NumThreads uint `yaml:"put_object_num_threads" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_NUM_THREADS" desc:"Number of concurrent uploads to use when copying objects to S3." introductionVersion:"1.0.0"` PartSize uint64 `yaml:"put_object_part_size" env:"STORAGE_USERS_DECOMPOSEDS3_PUT_OBJECT_PART_SIZE" desc:"Part size for concurrent uploads to S3. If no value or 0 is set, the library's default value of 16MB is used. The value range is min 5MB and max 5GB." introductionVersion:"1.0.0"` // PersonalSpaceAliasTemplate contains the template used to construct diff --git a/services/storage-users/pkg/config/defaults/defaultconfig.go b/services/storage-users/pkg/config/defaults/defaultconfig.go index 4bf3c80b1a..3d79a21022 100644 --- a/services/storage-users/pkg/config/defaults/defaultconfig.go +++ b/services/storage-users/pkg/config/defaults/defaultconfig.go @@ -112,7 +112,7 @@ func DefaultConfig() *config.Config { UserLayout: "{{.Id.OpaqueId}}", Region: "default", SendContentMd5: true, - ConcurrentStreamParts: true, + ConcurrentStreamParts: false, NumThreads: 4, PersonalSpaceAliasTemplate: "{{.SpaceType}}/{{.User.Username | lower}}", PersonalSpacePathTemplate: "", @@ -123,6 +123,7 @@ func DefaultConfig() *config.Config { MaxConcurrency: 5, LockCycleDurationFactor: 30, DisableMultipart: true, + PartSize: 16777216, AsyncUploads: true, }, Decomposed: config.DecomposedDriver{