feat(repository): Add Custom Root CA option for S3 (#2845)

* Add Custom Root CA option

example:

```bash
kopia repository connect s3 \
  --access-key the-access-key \
  --secret-access-key the-secret-key \
  --bucket my-backup-bucket \
  --endpoint localhost:9000 \
  --rootca=$(cat public.crt|base64 -w0)
```

* Handle CA as file too
* Lower case JSON option
* Handle file as well as inline
* Add env variable
* Standardize options names

cli/storage_s3.go

@@ -2,6 +2,8 @@
 
 import (
 	"context"
+	"encoding/base64"
+	"os"
 	"time"
 
 	"github.com/alecthomas/kingpin/v2"
@@ -12,7 +14,9 @@
 )
 
 type storageS3Flags struct {
-	s3options s3.Options
+	s3options       s3.Options
+	rootCaPemBase64 string
+	rootCaPemPath   string
 }
 
 func (c *storageS3Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClause) {
@@ -44,6 +48,35 @@ func (c *storageS3Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClau
 	}
 
 	cmd.Flag("point-in-time", "Use a point-in-time view of the storage repository when supported").PlaceHolder(time.RFC3339).PreAction(pitPreAction).StringVar(&pointInTimeStr)
+
+	cmd.Flag("root-ca-pem-base64", "Certficate authority in-line (base64 enc.)").Envar(svc.EnvName("ROOT_CA_PEM_BASE64")).PreAction(c.preActionLoadPEMBase64).StringVar(&c.rootCaPemBase64)
+	cmd.Flag("root-ca-pem-path", "Certficate authority file path").PreAction(c.preActionLoadPEMPath).StringVar(&c.rootCaPemPath)
+}
+
+func (c *storageS3Flags) preActionLoadPEMPath(pc *kingpin.ParseContext) error {
+	if len(c.s3options.RootCA) > 0 {
+		return errors.Errorf("root-ca-pem-base64 and root-ca-pem-path are mutually exclusive")
+	}
+
+	data, err := os.ReadFile(c.rootCaPemPath) //#nosec
+	if err != nil {
+		return errors.Wrapf(err, "error opening root-ca-pem-path %v", c.rootCaPemPath)
+	}
+
+	c.s3options.RootCA = data
+
+	return nil
+}
+
+func (c *storageS3Flags) preActionLoadPEMBase64(pc *kingpin.ParseContext) error {
+	caContent, err := base64.StdEncoding.DecodeString(c.rootCaPemBase64)
+	if err != nil {
+		return errors.Wrap(err, "unable to decode CA")
+	}
+
+	c.s3options.RootCA = caContent
+
+	return nil
 }
 
 func (c *storageS3Flags) Connect(ctx context.Context, isCreate bool, formatVersion int) (blob.Storage, error) {

cli/storage_s3_test.go

@@ -0,0 +1,64 @@
+package cli
+
+import (
+	"encoding/base64"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/kopia/kopia/internal/testutil"
+)
+
+var (
+	fakeCertContent         = []byte("fake certificate content")
+	fakeCertContentAsBase64 = base64.StdEncoding.EncodeToString(fakeCertContent)
+)
+
+func TestLoadPEMBase64(t *testing.T) {
+	var s3flags storageS3Flags
+
+	s3flags = storageS3Flags{rootCaPemBase64: ""}
+	require.NoError(t, s3flags.preActionLoadPEMBase64(nil))
+
+	s3flags = storageS3Flags{rootCaPemBase64: "AA=="}
+	require.NoError(t, s3flags.preActionLoadPEMBase64(nil))
+
+	s3flags = storageS3Flags{rootCaPemBase64: fakeCertContentAsBase64}
+	require.NoError(t, s3flags.preActionLoadPEMBase64(nil))
+	require.Equal(t, fakeCertContent, s3flags.s3options.RootCA, "content of RootCA should be %v", fakeCertContent)
+
+	s3flags = storageS3Flags{rootCaPemBase64: "!"}
+	err := s3flags.preActionLoadPEMBase64(nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "illegal base64 data")
+}
+
+func TestLoadPEMPath(t *testing.T) {
+	var s3flags storageS3Flags
+
+	tempdir := testutil.TempDirectory(t)
+	certpath := filepath.Join(tempdir, "certificate-filename")
+
+	require.NoError(t, os.WriteFile(certpath, fakeCertContent, 0o644))
+
+	// Test regular file
+	s3flags = storageS3Flags{rootCaPemPath: certpath}
+	require.NoError(t, s3flags.preActionLoadPEMPath(nil))
+	require.Equal(t, fakeCertContent, s3flags.s3options.RootCA, "content of RootCA should be %v", fakeCertContent)
+
+	// Test inexistent file
+	s3flags = storageS3Flags{rootCaPemPath: "/does-not-exists"}
+	err := s3flags.preActionLoadPEMPath(nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "error opening root-ca-pem-path")
+}
+
+func TestLoadPEMBoth(t *testing.T) {
+	s3flags := storageS3Flags{rootCaPemBase64: "AA==", rootCaPemPath: "/tmp/blah"}
+	require.NoError(t, s3flags.preActionLoadPEMBase64(nil))
+	err := s3flags.preActionLoadPEMPath(nil)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "mutually exclusive")
+}

repo/blob/s3/s3_options.go

@@ -17,6 +17,7 @@ type Options struct {
 	Endpoint       string `json:"endpoint"`
 	DoNotUseTLS    bool   `json:"doNotUseTLS,omitempty"`
 	DoNotVerifyTLS bool   `json:"doNotVerifyTLS,omitempty"`
+	RootCA         []byte `json:"rootCA,omitempty"`
 
 	AccessKeyID     string `json:"accessKeyID"`
 	SecretAccessKey string `json:"secretAccessKey" kopia:"sensitive"`

repo/blob/s3/s3_storage.go

@@ -5,6 +5,7 @@
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
 	"net/http"
@@ -292,10 +293,25 @@ func (s *s3Storage) FlushCaches(ctx context.Context) error {
 	return nil
 }
 
-func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) {
-	//nolint:gosec
-	customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}}
-	return customTransport
+func getCustomTransport(opt *Options) (*http.Transport, error) {
+	if opt.DoNotVerifyTLS {
+		//nolint:gosec
+		return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, nil
+	}
+
+	transport := http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert
+
+	if len(opt.RootCA) != 0 {
+		rootcas := x509.NewCertPool()
+
+		if ok := rootcas.AppendCertsFromPEM(opt.RootCA); !ok {
+			return nil, errors.Errorf("cannot parse provided CA")
+		}
+
+		transport.TLSClientConfig.RootCAs = rootcas
+	}
+
+	return transport, nil
 }
 
 // New creates new S3-backed storage with specified options:
@@ -349,8 +365,12 @@ func newStorageWithCredentials(ctx context.Context, creds *credentials.Credentia
 		Region: opt.Region,
 	}
 
-	if opt.DoNotVerifyTLS {
-		minioOpts.Transport = getCustomTransport(true)
+	var err error
+
+	minioOpts.Transport, err = getCustomTransport(opt)
+
+	if err != nil {
+		return nil, err
 	}
 
 	cli, err := minio.New(opt.Endpoint, minioOpts)

repo/blob/s3/s3_storage_test.go

@@ -2,7 +2,6 @@
 
 import (
 	"context"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -401,6 +400,48 @@ func TestS3StorageMinioSelfSignedCert(t *testing.T) {
 	testStorage(t, options, true, blob.PutOptions{})
 }
 
+func TestS3StorageMinioSelfSignedCertWithProvidedCA(t *testing.T) {
+	t.Parallel()
+	testutil.ProviderTest(t)
+
+	ctx := testlogging.Context(t)
+	minioConfigDir := testutil.TempDirectory(t)
+	certsDir := filepath.Join(minioConfigDir, "certs")
+	require.NoError(t, os.MkdirAll(certsDir, 0o755))
+
+	cert, key, err := tlsutil.GenerateServerCertificate(
+		ctx,
+		2048,
+		24*time.Hour,
+		[]string{"localhost"})
+
+	require.NoError(t, err)
+
+	certificatePath := filepath.Join(certsDir, "public.crt")
+
+	require.NoError(t, tlsutil.WriteCertificateToFile(certificatePath, cert))
+	require.NoError(t, tlsutil.WritePrivateKeyToFile(filepath.Join(certsDir, "private.key"), key))
+
+	minioEndpoint := startDockerMinioOrSkip(t, minioConfigDir)
+
+	data, err := os.ReadFile(certificatePath)
+
+	require.NoError(t, err)
+
+	options := &Options{
+		Endpoint:        minioEndpoint,
+		AccessKeyID:     minioRootAccessKeyID,
+		SecretAccessKey: minioRootSecretAccessKey,
+		BucketName:      minioBucketName,
+		Region:          minioRegion,
+		DoNotVerifyTLS:  false,
+		RootCA:          data,
+	}
+
+	createBucket(t, options)
+	testStorage(t, options, true, blob.PutOptions{})
+}
+
 func TestInvalidCredsFailsFast(t *testing.T) {
 	t.Parallel()
 	testutil.ProviderTest(t)
@@ -571,7 +612,12 @@ func TestCustomTransportNoSSLVerify(t *testing.T) {
 }
 
 func getURL(url string, insecureSkipVerify bool) error {
-	client := &http.Client{Transport: getCustomTransport(insecureSkipVerify)}
+	transport, err := getCustomTransport(&Options{DoNotVerifyTLS: insecureSkipVerify})
+	if err != nil {
+		return err
+	}
+
+	client := &http.Client{Transport: transport}
 
 	resp, err := client.Get(url) //nolint:noctx
 	if err != nil {
@@ -602,8 +648,12 @@ func createClient(tb testing.TB, opt *Options) *minio.Client {
 
 	var transport http.RoundTripper
 
-	if opt.DoNotVerifyTLS {
-		transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
+	var err error
+
+	transport, err = getCustomTransport(opt)
+
+	if err != nil {
+		tb.Fatalf("unable to get proper transport: %v", err)
 	}
 
 	minioClient, err := minio.New(opt.Endpoint,