blob: remove s3Storage.needMD5 (#917)

This commit is contained in:
Julio López
2021-03-30 06:44:59 -07:00
committed by GitHub
parent 5ff2c85c80
commit c9cc16936c
2 changed files with 1 additions and 77 deletions

View File

@@ -20,15 +20,12 @@
"github.com/kopia/kopia/repo/blob"
"github.com/kopia/kopia/repo/blob/retrying"
"github.com/kopia/kopia/repo/logging"
)
const (
s3storageType = "s3"
)
var log = logging.GetContextLoggerFunc(s3storageType)
type s3Storage struct {
sendMD5 int32
Options
@@ -220,24 +217,6 @@ func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) {
return customTransport
}
// returns whether put blob requires sending the content's MD5.
func needMD5(ctx context.Context, cli *minio.Client, bucketName string) bool {
var er minio.ErrorResponse
mode, _, _, err := cli.GetBucketObjectLockConfig(ctx, bucketName)
if err == nil {
return mode != nil && mode.IsValid()
}
if !errors.As(err, &er) || er.Code != "ObjectLockConfigurationNotFoundError" {
// Not all S3 stores implement the S3 API for bucket object-locking
// configuration and return an invalid response here.
log(ctx).Debugf("Could not get object-locking configuration, assuming MD5 is not needed for put blob: %v", err)
}
return false
}
// New creates new S3-backed storage with specified options:
//
// - the 'BucketName' field is required and all other parameters are optional.
@@ -273,16 +252,10 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) {
return nil, errors.Errorf("bucket %q does not exist", opt.BucketName)
}
var sendMD5 int32
if needMD5(ctx, cli, opt.BucketName) {
sendMD5 = 1
}
return retrying.NewWrapper(&s3Storage{
Options: *opt,
cli: cli,
sendMD5: sendMD5,
sendMD5: 0,
downloadThrottler: downloadThrottler,
uploadThrottler: uploadThrottler,
}), nil

View File

@@ -337,10 +337,6 @@ func TestNeedMD5AWS(t *testing.T) {
err := cli.SetBucketObjectLockConfig(ctx, options.BucketName, &lockingMode, &unit, &days)
noError(t, err, "could not set object lock config")
if !needMD5(ctx, cli, options.BucketName) {
t.Fatal("expected bucket to require MD5 for PUT blob, but got not required")
}
options.Prefix = uuid.NewString() + "/"
s, err := New(ctx, options)
@@ -356,51 +352,6 @@ func TestNeedMD5AWS(t *testing.T) {
})
}
func TestNoNeedMD5Minio(t *testing.T) {
t.Parallel()
ctx := testlogging.Context(t)
minioEndpoint := startDockerMinioOrSkip(t)
var cli *minio.Client
options := &Options{
Endpoint: minioEndpoint,
AccessKeyID: minioAccessKeyID,
SecretAccessKey: minioSecretAccessKey,
BucketName: minioBucketName,
Region: minioRegion,
DoNotUseTLS: !minioUseSSL,
}
testutil.Retry(t, func(t *testutil.RetriableT) {
cli = createClient(t, options)
makeBucket(t, cli, options, false)
if needMD5(ctx, cli, minioBucketName) {
t.Fatal("expected bucket to not require MD5 for PUT blob, but got required")
}
})
}
func TestNoNeedMD5Providers(t *testing.T) {
t.Parallel()
for k, env := range providerCreds {
env := env
t.Run(k, func(t *testing.T) {
ctx := testlogging.Context(t)
options := getProviderOptions(t, env)
cli := createClient(t, options)
if needMD5(ctx, cli, options.BucketName) {
t.Fatal("expected bucket to not require MD5 for PUT blob, but got required")
}
})
}
}
func testStorage(t *testutil.RetriableT, options *Options) {
ctx := testlogging.Context(t)