Add CLI option for disabling tls verification while connecting to s3 (#370)

* Add CLI option for disabling tls verification while connecting to s3

cli/storage_s3.go

@@ -24,6 +24,7 @@ func init() {
 			cmd.Flag("session-token", "Session token (overrides AWS_SESSION_TOKEN environment variable)").Envar("AWS_SESSION_TOKEN").StringVar(&s3options.SessionToken)
 			cmd.Flag("prefix", "Prefix to use for objects in the bucket").StringVar(&s3options.Prefix)
 			cmd.Flag("disable-tls", "Disable TLS security (HTTPS)").BoolVar(&s3options.DoNotUseTLS)
+			cmd.Flag("disable-tls-verification", "Disable TLS (HTTPS) certificate verification").BoolVar(&s3options.DoNotVerifyTLS)
 			cmd.Flag("max-download-speed", "Limit the download speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&s3options.MaxDownloadSpeedBytesPerSecond)
 			cmd.Flag("max-upload-speed", "Limit the upload speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&s3options.MaxUploadSpeedBytesPerSecond)
 		},

repo/blob/s3/s3_options.go

@@ -8,8 +8,9 @@ type Options struct {
 	// Prefix specifies additional string to prepend to all objects.
 	Prefix string `json:"prefix,omitempty"`
 
-	Endpoint    string `json:"endpoint"`
-	DoNotUseTLS bool   `json:"doNotUseTLS,omitempty"`
+	Endpoint       string `json:"endpoint"`
+	DoNotUseTLS    bool   `json:"doNotUseTLS,omitempty"`
+	DoNotVerifyTLS bool   `json:"doNotVerifyTLS,omitempty"`
 
 	AccessKeyID     string `json:"accessKeyID"`
 	SecretAccessKey string `json:"secretAccessKey" kopia:"sensitive"`

repo/blob/s3/s3_storage.go

@@ -4,6 +4,7 @@ package s3
 import (
 	"bytes"
 	"context"
+	"crypto/tls"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -225,6 +226,12 @@ func toBandwidth(bytesPerSecond int) iothrottler.Bandwidth {
 	return iothrottler.Bandwidth(bytesPerSecond) * iothrottler.BytesPerSecond
 }
 
+func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) {
+	// nolint:gosec
+	customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}}
+	return customTransport
+}
+
 // New creates new S3-backed storage with specified options:
 //
 // - the 'BucketName' field is required and all other parameters are optional.
@@ -238,6 +245,10 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) {
 		return nil, errors.Wrap(err, "unable to create client")
 	}
 
+	if opt.DoNotVerifyTLS {
+		cli.SetCustomTransport(getCustomTransport(true))
+	}
+
 	downloadThrottler := iothrottler.NewIOThrottlerPool(toBandwidth(opt.MaxDownloadSpeedBytesPerSecond))
 	uploadThrottler := iothrottler.NewIOThrottlerPool(toBandwidth(opt.MaxUploadSpeedBytesPerSecond))
 

repo/blob/s3/s3_storage_test.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"net/http"
 	"os"
 	"testing"
 	"time"
@@ -50,6 +51,11 @@ const (
 	testSTSAccessKeyIDEnv     = "KOPIA_S3_TEST_STS_ACCESS_KEY_ID"
 	testSTSSecretAccessKeyEnv = "KOPIA_S3_TEST_STS_SECRET_ACCESS_KEY"
 	testSessionTokenEnv       = "KOPIA_S3_TEST_SESSION_TOKEN"
+
+	expiredBadSSL       = "https://expired.badssl.com/"
+	selfSignedBadSSL    = "https://self-signed.badssl.com/"
+	untrustedRootBadSSL = "https://untrusted-root.badssl.com/"
+	wrongHostBadSSL     = "https://wrong.host.badssl.com/"
 )
 
 var minioBucketName = getBucketName()
@@ -153,55 +159,64 @@ func TestS3StorageAWSSTS(t *testing.T) {
 func TestS3StorageMinio(t *testing.T) {
 	t.Parallel()
 
-	testutil.Retry(t, func(t *testutil.RetriableT) {
-		options := &Options{
-			Endpoint:        minioEndpoint,
-			AccessKeyID:     minioAccessKeyID,
-			SecretAccessKey: minioSecretAccessKey,
-			BucketName:      minioBucketName,
-			Region:          minioRegion,
-			DoNotUseTLS:     !minioUseSSL,
-		}
+	for _, disableTLSVerify := range []bool{true, false} {
+		disableTLSVerify := disableTLSVerify
 
-		if !endpointReachable(options.Endpoint) {
-			t.Skip("endpoint not reachable")
-		}
+		testutil.Retry(t, func(t *testutil.RetriableT) {
+			options := &Options{
+				Endpoint:        minioEndpoint,
+				AccessKeyID:     minioAccessKeyID,
+				SecretAccessKey: minioSecretAccessKey,
+				BucketName:      minioBucketName,
+				Region:          minioRegion,
+				DoNotUseTLS:     !minioUseSSL,
+				DoNotVerifyTLS:  disableTLSVerify,
+			}
 
-		createBucket(t, options)
-		testStorage(t, options)
-	})
+			if !endpointReachable(options.Endpoint) {
+				t.Skip("endpoint not reachable")
+			}
+
+			createBucket(t, options)
+			testStorage(t, options)
+		})
+	}
 }
 
 func TestS3StorageMinioSTS(t *testing.T) {
 	t.Parallel()
 
-	testutil.Retry(t, func(t *testutil.RetriableT) {
+	for _, disableTLSVerify := range []bool{true, false} {
+		disableTLSVerify := disableTLSVerify
 
-		// create kopia user and session token
-		kopiaUserName := generateName("kopiauser")
-		kopiaUserPasswd := generateName("kopiapassword")
+		testutil.Retry(t, func(t *testutil.RetriableT) {
+			// create kopia user and session token
+			kopiaUserName := generateName("kopiauser")
+			kopiaUserPasswd := generateName("kopiapassword")
 
-		createMinioUser(t, kopiaUserName, kopiaUserPasswd)
-		defer deleteMinioUser(t, kopiaUserName)
-		kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, kopiaUserName, kopiaUserPasswd, minioBucketName)
+			createMinioUser(t, kopiaUserName, kopiaUserPasswd)
+			defer deleteMinioUser(t, kopiaUserName)
+			kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, kopiaUserName, kopiaUserPasswd, minioBucketName)
 
-		options := &Options{
-			Endpoint:        minioEndpoint,
-			AccessKeyID:     kopiaAccessKeyID,
-			SecretAccessKey: kopiaSecretKey,
-			SessionToken:    kopiaSessionToken,
-			BucketName:      minioBucketName,
-			Region:          minioRegion,
-			DoNotUseTLS:     !minioUseSSL,
-		}
+			options := &Options{
+				Endpoint:        minioEndpoint,
+				AccessKeyID:     kopiaAccessKeyID,
+				SecretAccessKey: kopiaSecretKey,
+				SessionToken:    kopiaSessionToken,
+				BucketName:      minioBucketName,
+				Region:          minioRegion,
+				DoNotUseTLS:     !minioUseSSL,
+				DoNotVerifyTLS:  disableTLSVerify,
+			}
 
-		if !endpointReachable(options.Endpoint) {
-			t.Skip("endpoint not reachable")
-		}
+			if !endpointReachable(options.Endpoint) {
+				t.Skip("endpoint not reachable")
+			}
 
-		createBucket(t, options)
-		testStorage(t, options)
-	})
+			createBucket(t, options)
+			testStorage(t, options)
+		})
+	}
 }
 
 func testStorage(t *testutil.RetriableT, options *Options) {
@@ -219,7 +234,7 @@ func testStorage(t *testutil.RetriableT, options *Options) {
 
 	v, err := retry.WithExponentialBackoff(ctx, "New() S3 storage", attempt, func(err error) bool { return err != nil })
 	if err != nil {
-		t.Fatalf("err: %v", err)
+		t.Fatalf("err: %v, options:%v", err, options)
 	}
 
 	st := v.(blob.Storage)
@@ -231,6 +246,38 @@ func testStorage(t *testutil.RetriableT, options *Options) {
 	}
 }
 
+func TestCustomTransportNoSSLVerify(t *testing.T) {
+	testURL(expiredBadSSL, t)
+	testURL(selfSignedBadSSL, t)
+	testURL(untrustedRootBadSSL, t)
+	testURL(wrongHostBadSSL, t)
+}
+
+func getURL(url string, insecureSkipVerify bool) error {
+	client := &http.Client{Transport: getCustomTransport(insecureSkipVerify)}
+	resp, err := client.Get(url)
+
+	if err != nil {
+		return err
+	}
+
+	defer resp.Body.Close()
+
+	return nil
+}
+
+func testURL(url string, t *testing.T) {
+	err := getURL(url, true)
+	if err != nil {
+		t.Fatalf("could not get url:%s, error:%v", url, err)
+	}
+
+	err = getURL(url, false)
+	if err == nil {
+		t.Fatalf("expected a TLS issue, but none found for url:%s", url)
+	}
+}
+
 func createBucket(t *testutil.RetriableT, opt *Options) {
 	minioClient, err := minio.New(opt.Endpoint, opt.AccessKeyID, opt.SecretAccessKey, !opt.DoNotUseTLS)
 	if err != nil {