From f17daee9ea1d89aa21392a169ab800525fbf127b Mon Sep 17 00:00:00 2001 From: Damien Degois Date: Fri, 31 Mar 2023 23:32:34 +0200 Subject: [PATCH] feat(repository): Add Custom Root CA option for S3 (#2845) * Add Custom Root CA option example: ```bash kopia repository connect s3 \ --access-key the-access-key \ --secret-access-key the-secret-key \ --bucket my-backup-bucket \ --endpoint localhost:9000 \ --rootca=$(cat public.crt|base64 -w0) ``` * Handle CA as file too * Lower case JSON option * Handle file as well as inline * Add env variable * Standardize options names --- cli/storage_s3.go | 35 +++++++++++++++++- cli/storage_s3_test.go | 64 +++++++++++++++++++++++++++++++++ repo/blob/s3/s3_options.go | 1 + repo/blob/s3/s3_storage.go | 32 +++++++++++++---- repo/blob/s3/s3_storage_test.go | 58 +++++++++++++++++++++++++++--- 5 files changed, 179 insertions(+), 11 deletions(-) create mode 100644 cli/storage_s3_test.go diff --git a/cli/storage_s3.go b/cli/storage_s3.go index 444ef3ff4..1b237ad30 100644 --- a/cli/storage_s3.go +++ b/cli/storage_s3.go @@ -2,6 +2,8 @@ import ( "context" + "encoding/base64" + "os" "time" "github.com/alecthomas/kingpin/v2" @@ -12,7 +14,9 @@ ) type storageS3Flags struct { - s3options s3.Options + s3options s3.Options + rootCaPemBase64 string + rootCaPemPath string } func (c *storageS3Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClause) { @@ -44,6 +48,35 @@ func (c *storageS3Flags) Setup(svc StorageProviderServices, cmd *kingpin.CmdClau } cmd.Flag("point-in-time", "Use a point-in-time view of the storage repository when supported").PlaceHolder(time.RFC3339).PreAction(pitPreAction).StringVar(&pointInTimeStr) + + cmd.Flag("root-ca-pem-base64", "Certficate authority in-line (base64 enc.)").Envar(svc.EnvName("ROOT_CA_PEM_BASE64")).PreAction(c.preActionLoadPEMBase64).StringVar(&c.rootCaPemBase64) + cmd.Flag("root-ca-pem-path", "Certficate authority file path").PreAction(c.preActionLoadPEMPath).StringVar(&c.rootCaPemPath) +} + +func (c *storageS3Flags) preActionLoadPEMPath(pc *kingpin.ParseContext) error { + if len(c.s3options.RootCA) > 0 { + return errors.Errorf("root-ca-pem-base64 and root-ca-pem-path are mutually exclusive") + } + + data, err := os.ReadFile(c.rootCaPemPath) //#nosec + if err != nil { + return errors.Wrapf(err, "error opening root-ca-pem-path %v", c.rootCaPemPath) + } + + c.s3options.RootCA = data + + return nil +} + +func (c *storageS3Flags) preActionLoadPEMBase64(pc *kingpin.ParseContext) error { + caContent, err := base64.StdEncoding.DecodeString(c.rootCaPemBase64) + if err != nil { + return errors.Wrap(err, "unable to decode CA") + } + + c.s3options.RootCA = caContent + + return nil } func (c *storageS3Flags) Connect(ctx context.Context, isCreate bool, formatVersion int) (blob.Storage, error) { diff --git a/cli/storage_s3_test.go b/cli/storage_s3_test.go new file mode 100644 index 000000000..ae4a146b3 --- /dev/null +++ b/cli/storage_s3_test.go @@ -0,0 +1,64 @@ +package cli + +import ( + "encoding/base64" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/kopia/kopia/internal/testutil" +) + +var ( + fakeCertContent = []byte("fake certificate content") + fakeCertContentAsBase64 = base64.StdEncoding.EncodeToString(fakeCertContent) +) + +func TestLoadPEMBase64(t *testing.T) { + var s3flags storageS3Flags + + s3flags = storageS3Flags{rootCaPemBase64: ""} + require.NoError(t, s3flags.preActionLoadPEMBase64(nil)) + + s3flags = storageS3Flags{rootCaPemBase64: "AA=="} + require.NoError(t, s3flags.preActionLoadPEMBase64(nil)) + + s3flags = storageS3Flags{rootCaPemBase64: fakeCertContentAsBase64} + require.NoError(t, s3flags.preActionLoadPEMBase64(nil)) + require.Equal(t, fakeCertContent, s3flags.s3options.RootCA, "content of RootCA should be %v", fakeCertContent) + + s3flags = storageS3Flags{rootCaPemBase64: "!"} + err := s3flags.preActionLoadPEMBase64(nil) + require.Error(t, err) + require.Contains(t, err.Error(), "illegal base64 data") +} + +func TestLoadPEMPath(t *testing.T) { + var s3flags storageS3Flags + + tempdir := testutil.TempDirectory(t) + certpath := filepath.Join(tempdir, "certificate-filename") + + require.NoError(t, os.WriteFile(certpath, fakeCertContent, 0o644)) + + // Test regular file + s3flags = storageS3Flags{rootCaPemPath: certpath} + require.NoError(t, s3flags.preActionLoadPEMPath(nil)) + require.Equal(t, fakeCertContent, s3flags.s3options.RootCA, "content of RootCA should be %v", fakeCertContent) + + // Test inexistent file + s3flags = storageS3Flags{rootCaPemPath: "/does-not-exists"} + err := s3flags.preActionLoadPEMPath(nil) + require.Error(t, err) + require.Contains(t, err.Error(), "error opening root-ca-pem-path") +} + +func TestLoadPEMBoth(t *testing.T) { + s3flags := storageS3Flags{rootCaPemBase64: "AA==", rootCaPemPath: "/tmp/blah"} + require.NoError(t, s3flags.preActionLoadPEMBase64(nil)) + err := s3flags.preActionLoadPEMPath(nil) + require.Error(t, err) + require.Contains(t, err.Error(), "mutually exclusive") +} diff --git a/repo/blob/s3/s3_options.go b/repo/blob/s3/s3_options.go index 1b20ce490..ab17ef34b 100644 --- a/repo/blob/s3/s3_options.go +++ b/repo/blob/s3/s3_options.go @@ -17,6 +17,7 @@ type Options struct { Endpoint string `json:"endpoint"` DoNotUseTLS bool `json:"doNotUseTLS,omitempty"` DoNotVerifyTLS bool `json:"doNotVerifyTLS,omitempty"` + RootCA []byte `json:"rootCA,omitempty"` AccessKeyID string `json:"accessKeyID"` SecretAccessKey string `json:"secretAccessKey" kopia:"sensitive"` diff --git a/repo/blob/s3/s3_storage.go b/repo/blob/s3/s3_storage.go index 0f6f8b79d..e67eb0589 100644 --- a/repo/blob/s3/s3_storage.go +++ b/repo/blob/s3/s3_storage.go @@ -5,6 +5,7 @@ "bytes" "context" "crypto/tls" + "crypto/x509" "fmt" "io" "net/http" @@ -292,10 +293,25 @@ func (s *s3Storage) FlushCaches(ctx context.Context) error { return nil } -func getCustomTransport(insecureSkipVerify bool) (transport *http.Transport) { - //nolint:gosec - customTransport := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: insecureSkipVerify}} - return customTransport +func getCustomTransport(opt *Options) (*http.Transport, error) { + if opt.DoNotVerifyTLS { + //nolint:gosec + return &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, nil + } + + transport := http.DefaultTransport.(*http.Transport).Clone() //nolint:forcetypeassert + + if len(opt.RootCA) != 0 { + rootcas := x509.NewCertPool() + + if ok := rootcas.AppendCertsFromPEM(opt.RootCA); !ok { + return nil, errors.Errorf("cannot parse provided CA") + } + + transport.TLSClientConfig.RootCAs = rootcas + } + + return transport, nil } // New creates new S3-backed storage with specified options: @@ -349,8 +365,12 @@ func newStorageWithCredentials(ctx context.Context, creds *credentials.Credentia Region: opt.Region, } - if opt.DoNotVerifyTLS { - minioOpts.Transport = getCustomTransport(true) + var err error + + minioOpts.Transport, err = getCustomTransport(opt) + + if err != nil { + return nil, err } cli, err := minio.New(opt.Endpoint, minioOpts) diff --git a/repo/blob/s3/s3_storage_test.go b/repo/blob/s3/s3_storage_test.go index 2a23257e9..f02d75877 100644 --- a/repo/blob/s3/s3_storage_test.go +++ b/repo/blob/s3/s3_storage_test.go @@ -2,7 +2,6 @@ import ( "context" - "crypto/tls" "encoding/json" "errors" "fmt" @@ -401,6 +400,48 @@ func TestS3StorageMinioSelfSignedCert(t *testing.T) { testStorage(t, options, true, blob.PutOptions{}) } +func TestS3StorageMinioSelfSignedCertWithProvidedCA(t *testing.T) { + t.Parallel() + testutil.ProviderTest(t) + + ctx := testlogging.Context(t) + minioConfigDir := testutil.TempDirectory(t) + certsDir := filepath.Join(minioConfigDir, "certs") + require.NoError(t, os.MkdirAll(certsDir, 0o755)) + + cert, key, err := tlsutil.GenerateServerCertificate( + ctx, + 2048, + 24*time.Hour, + []string{"localhost"}) + + require.NoError(t, err) + + certificatePath := filepath.Join(certsDir, "public.crt") + + require.NoError(t, tlsutil.WriteCertificateToFile(certificatePath, cert)) + require.NoError(t, tlsutil.WritePrivateKeyToFile(filepath.Join(certsDir, "private.key"), key)) + + minioEndpoint := startDockerMinioOrSkip(t, minioConfigDir) + + data, err := os.ReadFile(certificatePath) + + require.NoError(t, err) + + options := &Options{ + Endpoint: minioEndpoint, + AccessKeyID: minioRootAccessKeyID, + SecretAccessKey: minioRootSecretAccessKey, + BucketName: minioBucketName, + Region: minioRegion, + DoNotVerifyTLS: false, + RootCA: data, + } + + createBucket(t, options) + testStorage(t, options, true, blob.PutOptions{}) +} + func TestInvalidCredsFailsFast(t *testing.T) { t.Parallel() testutil.ProviderTest(t) @@ -571,7 +612,12 @@ func TestCustomTransportNoSSLVerify(t *testing.T) { } func getURL(url string, insecureSkipVerify bool) error { - client := &http.Client{Transport: getCustomTransport(insecureSkipVerify)} + transport, err := getCustomTransport(&Options{DoNotVerifyTLS: insecureSkipVerify}) + if err != nil { + return err + } + + client := &http.Client{Transport: transport} resp, err := client.Get(url) //nolint:noctx if err != nil { @@ -602,8 +648,12 @@ func createClient(tb testing.TB, opt *Options) *minio.Client { var transport http.RoundTripper - if opt.DoNotVerifyTLS { - transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}} + var err error + + transport, err = getCustomTransport(opt) + + if err != nil { + tb.Fatalf("unable to get proper transport: %v", err) } minioClient, err := minio.New(opt.Endpoint,