diff --git a/cli/storage_s3.go b/cli/storage_s3.go index f70d84911..b509f1608 100644 --- a/cli/storage_s3.go +++ b/cli/storage_s3.go @@ -24,6 +24,7 @@ func init() { cmd.Flag("session-token", "Session token (overrides AWS_SESSION_TOKEN environment variable)").Envar("AWS_SESSION_TOKEN").StringVar(&s3options.SessionToken) cmd.Flag("prefix", "Prefix to use for objects in the bucket").StringVar(&s3options.Prefix) cmd.Flag("disable-tls", "Disable TLS security (HTTPS)").BoolVar(&s3options.DoNotUseTLS) + cmd.Flag("disable-tls-verification", "Disable TLS (HTTPS) certificate verification").BoolVar(&s3options.DoNotVerifyTLS) cmd.Flag("max-download-speed", "Limit the download speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&s3options.MaxDownloadSpeedBytesPerSecond) cmd.Flag("max-upload-speed", "Limit the upload speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&s3options.MaxUploadSpeedBytesPerSecond) }, diff --git a/repo/blob/s3/s3_options.go b/repo/blob/s3/s3_options.go index c8ca266d0..55c029352 100644 --- a/repo/blob/s3/s3_options.go +++ b/repo/blob/s3/s3_options.go @@ -8,8 +8,9 @@ type Options struct { // Prefix specifies additional string to prepend to all objects. Prefix string `json:"prefix,omitempty"` - Endpoint string `json:"endpoint"` - DoNotUseTLS bool `json:"doNotUseTLS,omitempty"` + Endpoint string `json:"endpoint"` + DoNotUseTLS bool `json:"doNotUseTLS,omitempty"` + DoNotVerifyTLS bool `json:"doNotVerifyTLS,omitempty"` AccessKeyID string `json:"accessKeyID"` SecretAccessKey string `json:"secretAccessKey" kopia:"sensitive"` diff --git a/repo/blob/s3/s3_storage.go b/repo/blob/s3/s3_storage.go index 320fca098..2a1a21f2e 100644 --- a/repo/blob/s3/s3_storage.go +++ b/repo/blob/s3/s3_storage.go @@ -4,6 +4,7 @@ package s3 import ( "bytes" "context" + "crypto/tls" "fmt" "io" "io/ioutil" @@ -225,6 +226,12 @@ func toBandwidth(bytesPerSecond int) iothrottler.Bandwidth { return iothrottler.Bandwidth(bytesPerSecond) * iothrottler.BytesPerSecond } +func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) { + // nolint:gosec + customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}} + return customTransport +} + // New creates new S3-backed storage with specified options: // // - the 'BucketName' field is required and all other parameters are optional. @@ -238,6 +245,10 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) { return nil, errors.Wrap(err, "unable to create client") } + if opt.DoNotVerifyTLS { + cli.SetCustomTransport(getCustomTransport(true)) + } + downloadThrottler := iothrottler.NewIOThrottlerPool(toBandwidth(opt.MaxDownloadSpeedBytesPerSecond)) uploadThrottler := iothrottler.NewIOThrottlerPool(toBandwidth(opt.MaxUploadSpeedBytesPerSecond)) diff --git a/repo/blob/s3/s3_storage_test.go b/repo/blob/s3/s3_storage_test.go index a374b8ba5..74c8537ca 100644 --- a/repo/blob/s3/s3_storage_test.go +++ b/repo/blob/s3/s3_storage_test.go @@ -7,6 +7,7 @@ import ( "fmt" "log" "net" + "net/http" "os" "testing" "time" @@ -50,6 +51,11 @@ const ( testSTSAccessKeyIDEnv = "KOPIA_S3_TEST_STS_ACCESS_KEY_ID" testSTSSecretAccessKeyEnv = "KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY" testSessionTokenEnv = "KOPIA_S3_TEST_SESSION_TOKEN" + + expiredBadSSL = "https://expired.badssl.com/" + selfSignedBadSSL = "https://self-signed.badssl.com/" + untrustedRootBadSSL = "https://untrusted-root.badssl.com/" + wrongHostBadSSL = "https://wrong.host.badssl.com/" ) var minioBucketName = getBucketName() @@ -153,55 +159,64 @@ func TestS3StorageAWSSTS(t *testing.T) { func TestS3StorageMinio(t *testing.T) { t.Parallel() - testutil.Retry(t, func(t *testutil.RetriableT) { - options := &Options{ - Endpoint: minioEndpoint, - AccessKeyID: minioAccessKeyID, - SecretAccessKey: minioSecretAccessKey, - BucketName: minioBucketName, - Region: minioRegion, - DoNotUseTLS: !minioUseSSL, - } + for _, disableTLSVerify := range []bool{true, false} { + disableTLSVerify := disableTLSVerify - if !endpointReachable(options.Endpoint) { - t.Skip("endpoint not reachable") - } + testutil.Retry(t, func(t *testutil.RetriableT) { + options := &Options{ + Endpoint: minioEndpoint, + AccessKeyID: minioAccessKeyID, + SecretAccessKey: minioSecretAccessKey, + BucketName: minioBucketName, + Region: minioRegion, + DoNotUseTLS: !minioUseSSL, + DoNotVerifyTLS: disableTLSVerify, + } - createBucket(t, options) - testStorage(t, options) - }) + if !endpointReachable(options.Endpoint) { + t.Skip("endpoint not reachable") + } + + createBucket(t, options) + testStorage(t, options) + }) + } } func TestS3StorageMinioSTS(t *testing.T) { t.Parallel() - testutil.Retry(t, func(t *testutil.RetriableT) { + for _, disableTLSVerify := range []bool{true, false} { + disableTLSVerify := disableTLSVerify - // create kopia user and session token - kopiaUserName := generateName("kopiauser") - kopiaUserPasswd := generateName("kopiapassword") + testutil.Retry(t, func(t *testutil.RetriableT) { + // create kopia user and session token + kopiaUserName := generateName("kopiauser") + kopiaUserPasswd := generateName("kopiapassword") - createMinioUser(t, kopiaUserName, kopiaUserPasswd) - defer deleteMinioUser(t, kopiaUserName) - kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, kopiaUserName, kopiaUserPasswd, minioBucketName) + createMinioUser(t, kopiaUserName, kopiaUserPasswd) + defer deleteMinioUser(t, kopiaUserName) + kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, kopiaUserName, kopiaUserPasswd, minioBucketName) - options := &Options{ - Endpoint: minioEndpoint, - AccessKeyID: kopiaAccessKeyID, - SecretAccessKey: kopiaSecretKey, - SessionToken: kopiaSessionToken, - BucketName: minioBucketName, - Region: minioRegion, - DoNotUseTLS: !minioUseSSL, - } + options := &Options{ + Endpoint: minioEndpoint, + AccessKeyID: kopiaAccessKeyID, + SecretAccessKey: kopiaSecretKey, + SessionToken: kopiaSessionToken, + BucketName: minioBucketName, + Region: minioRegion, + DoNotUseTLS: !minioUseSSL, + DoNotVerifyTLS: disableTLSVerify, + } - if !endpointReachable(options.Endpoint) { - t.Skip("endpoint not reachable") - } + if !endpointReachable(options.Endpoint) { + t.Skip("endpoint not reachable") + } - createBucket(t, options) - testStorage(t, options) - }) + createBucket(t, options) + testStorage(t, options) + }) + } } func testStorage(t *testutil.RetriableT, options *Options) { @@ -219,7 +234,7 @@ func testStorage(t *testutil.RetriableT, options *Options) { v, err := retry.WithExponentialBackoff(ctx, "New() S3 storage", attempt, func(err error) bool { return err != nil }) if err != nil { - t.Fatalf("err: %v", err) + t.Fatalf("err: %v, options:%v", err, options) } st := v.(blob.Storage) @@ -231,6 +246,38 @@ func testStorage(t *testutil.RetriableT, options *Options) { } } +func TestCustomTransportNoSSLVerify(t *testing.T) { + testURL(expiredBadSSL, t) + testURL(selfSignedBadSSL, t) + testURL(untrustedRootBadSSL, t) + testURL(wrongHostBadSSL, t) +} + +func getURL(url string, insecureSkipVerify bool) error { + client := &http.Client{Transport: getCustomTransport(insecureSkipVerify)} + resp, err := client.Get(url) + + if err != nil { + return err + } + + defer resp.Body.Close() + + return nil +} + +func testURL(url string, t *testing.T) { + err := getURL(url, true) + if err != nil { + t.Fatalf("could not get url:%s, error:%v", url, err) + } + + err = getURL(url, false) + if err == nil { + t.Fatalf("expected a TLS issue, but none found for url:%s", url) + } +} + func createBucket(t *testutil.RetriableT, opt *Options) { minioClient, err := minio.New(opt.Endpoint, opt.AccessKeyID, opt.SecretAccessKey, !opt.DoNotUseTLS) if err != nil {