From f962d93159aab9c32488b1affb4ea634f233efeb Mon Sep 17 00:00:00 2001 From: Julio Lopez <1953782+julio-lopez@users.noreply.github.com> Date: Sun, 16 Jul 2023 09:51:40 -0700 Subject: [PATCH] refactor(providers): use minio SDK credentials package in S3 STS test (#3148) * refactor: return credentials.Value in createMinioSessionToken helper * refactor: use minio SDK credentials package in STS test * go mod tidy -compat=1.20 => removes dependency on AWS SDK v1 packages * cleanup: unalias use of minio credentials package --- go.mod | 2 - go.sum | 13 ----- repo/blob/s3/s3_storage_test.go | 87 +++++++++++++-------------------- 3 files changed, 35 insertions(+), 67 deletions(-) diff --git a/go.mod b/go.mod index 92bf27e75..0c0b12bc1 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 github.com/alecthomas/kingpin/v2 v2.3.2 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 - github.com/aws/aws-sdk-go v1.44.298 github.com/chmduquesne/rollinghash v4.0.0+incompatible github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9 github.com/chromedp/chromedp v0.9.1 @@ -99,7 +98,6 @@ require ( github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect diff --git a/go.sum b/go.sum index a0b713fe9..eecd620a6 100644 --- a/go.sum +++ b/go.sum @@ -45,8 +45,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0= github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= -github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g= -github.com/aws/aws-sdk-go v1.44.298/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= @@ -180,10 +178,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A= github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs= github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= -github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= -github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -198,8 +192,6 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= -github.com/kopia/htmluibuild v0.0.0-20230605144737-e386b860759d h1:qvV3TN5X/RsgmckkxsKh9P7Vtf9GYy6vOPzQY1SY4qM= -github.com/kopia/htmluibuild v0.0.0-20230605144737-e386b860759d/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw= github.com/kopia/htmluibuild v0.0.0-20230714030926-ff7ed652d883 h1:LX3CQvH12mNYXt+rBUqxwbwAieb9QmDrMXjKVAuMxvQ= github.com/kopia/htmluibuild v0.0.0-20230714030926-ff7ed652d883/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw= github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= @@ -362,7 +354,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -393,14 +384,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -409,7 +398,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -474,7 +462,6 @@ gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMv gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/repo/blob/s3/s3_storage_test.go b/repo/blob/s3/s3_storage_test.go index f02d75877..1636e2acc 100644 --- a/repo/blob/s3/s3_storage_test.go +++ b/repo/blob/s3/s3_storage_test.go @@ -13,13 +13,9 @@ "testing" "time" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/credentials" - "github.com/aws/aws-sdk-go/aws/session" - "github.com/aws/aws-sdk-go/service/sts" "github.com/google/uuid" "github.com/minio/minio-go/v7" - miniocreds "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/credentials" "github.com/stretchr/testify/require" "github.com/kopia/kopia/internal/blobtesting" @@ -477,7 +473,7 @@ func TestS3StorageMinioSTS(t *testing.T) { time.Sleep(2 * time.Second) - kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, minioEndpoint, minioRootAccessKeyID, minioRootSecretAccessKey, minioBucketName) + kopiaCreds := createMinioSessionToken(t, minioEndpoint, minioRootAccessKeyID, minioRootSecretAccessKey, minioBucketName) createBucket(t, &Options{ Endpoint: minioEndpoint, @@ -488,14 +484,15 @@ func TestS3StorageMinioSTS(t *testing.T) { DoNotUseTLS: true, }) - require.NotEqual(t, kopiaAccessKeyID, minioRootAccessKeyID) - require.NotEqual(t, kopiaSecretKey, minioRootSecretAccessKey) + require.NotEqual(t, kopiaCreds.AccessKeyID, minioRootAccessKeyID) + require.NotEqual(t, kopiaCreds.SecretAccessKey, minioRootSecretAccessKey) + require.NotEmpty(t, kopiaCreds.SessionToken) testStorage(t, &Options{ Endpoint: minioEndpoint, - AccessKeyID: kopiaAccessKeyID, - SecretAccessKey: kopiaSecretKey, - SessionToken: kopiaSessionToken, + AccessKeyID: kopiaCreds.AccessKeyID, + SecretAccessKey: kopiaCreds.SecretAccessKey, + SessionToken: kopiaCreds.SessionToken, BucketName: minioBucketName, Region: minioRegion, DoNotUseTLS: true, @@ -658,7 +655,7 @@ func createClient(tb testing.TB, opt *Options) *minio.Client { minioClient, err := minio.New(opt.Endpoint, &minio.Options{ - Creds: miniocreds.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, ""), + Creds: credentials.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, ""), Secure: !opt.DoNotUseTLS, Region: opt.Region, Transport: transport, @@ -723,28 +720,14 @@ func makeBucket(tb testing.TB, cli *minio.Client, opt *Options, objectLocking bo } } -func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUserPasswd, bucketName string) (accessID, secretKey, sessionToken string) { +func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUserPasswd, bucketName string) credentials.Value { t.Helper() - // Configure to use MinIO Server - awsConfig := &aws.Config{ - Credentials: credentials.NewStaticCredentials(kopiaUserName, kopiaUserPasswd, ""), - Endpoint: aws.String(minioEndpoint), - Region: aws.String(minioRegion), - S3ForcePathStyle: aws.Bool(true), - DisableSSL: aws.Bool(true), - } - - awsSession, err := session.NewSession(awsConfig) - if err != nil { - t.Fatalf("failed to create aws session: %v", err) - } - - svc := sts.New(awsSession) - - input := &sts.AssumeRoleInput{ - // give access to only S3 bucket with name bucketName - Policy: aws.String(fmt.Sprintf(`{ + stsOpts := credentials.STSAssumeRoleOptions{ + AccessKey: kopiaUserName, + SecretKey: kopiaUserPasswd, + DurationSeconds: 900, + Policy: fmt.Sprintf(`{ "Version":"2012-10-17", "Statement":[ { @@ -759,25 +742,25 @@ func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUs "Action": "s3:*", "Resource": "arn:aws:s3:::%v/*" } - ]}`, bucketName, bucketName)), + ]}`, bucketName, bucketName), // RoleArn and RoleSessionName are not meaningful for MinIO and can be set to any value - RoleArn: aws.String("arn:xxx:xxx:xxx:xxxx"), - RoleSessionName: aws.String("kopiaTestSession"), - DurationSeconds: aws.Int64(900), // in seconds + RoleARN: "arn:xxx:xxx:xxx:xxxx", + RoleSessionName: "kopiaTestSession", } - result, err := svc.AssumeRole(input) - if err != nil { - t.Fatalf("failed to create session with aws assume role: %v", err) + if !strings.HasPrefix(minioEndpoint, "http") { + minioEndpoint = "http://" + minioEndpoint } - if result.Credentials == nil { - t.Fatalf("couldn't find aws creds in aws assume role response") - } + // Get STS credentials from MinIO server + roleCreds, err := credentials.NewSTSAssumeRole(minioEndpoint, stsOpts) + require.NoError(t, err, "during STSAssumeRole:", minioEndpoint) + require.NotNil(t, roleCreds) - t.Logf("created session token with assume role: expiration: %s", result.Credentials.Expiration) + credsValue, err := roleCreds.Get() + require.NoError(t, err) - return *result.Credentials.AccessKeyId, *result.Credentials.SecretAccessKey, *result.Credentials.SessionToken + return credsValue } // customProvider is a custom provider based on minio's STSAssumeRole struct @@ -789,7 +772,7 @@ func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUs // the next call to Retrieve to return expired credentials. type customProvider struct { forceExpired atomic.Bool - stsProvider miniocreds.STSAssumeRole + stsProvider credentials.STSAssumeRole } const expiredSessionToken = "IQoJb3JpZ2luX2VjEBMaCXVzLXdlc3QtMiJIM" + @@ -806,13 +789,13 @@ type customProvider struct { "RCcVWcntllxyL/sUZ7VbMr7xZxWWbilu8pVtQqTwwBxZO0rth8XftMzGQ5oyd" + "82CdcwRB+t7K1LEmRErltbteGtM=" -func (cp *customProvider) Retrieve() (miniocreds.Value, error) { +func (cp *customProvider) Retrieve() (credentials.Value, error) { if cp.forceExpired.Load() { - return miniocreds.Value{ + return credentials.Value{ AccessKeyID: "ASIAQREAKNKDBR4F5F2I", SecretAccessKey: "EF82nKmZbnFETa96xxx1C3k20hG4Nw+2v+FBNjp3", SessionToken: expiredSessionToken, - SignerType: miniocreds.SignatureV2, + SignerType: credentials.SignatureV2, }, nil } @@ -825,8 +808,8 @@ func (cp *customProvider) IsExpired() bool { // customCredentialsAndProvider creates a custom provider and returns credentials // using this provider. -func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) (*miniocreds.Credentials, *customProvider) { - opts := miniocreds.STSAssumeRoleOptions{ +func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) (*credentials.Credentials, *customProvider) { + opts := credentials.STSAssumeRoleOptions{ AccessKey: accessKey, SecretKey: secretKey, Location: region, @@ -835,7 +818,7 @@ func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) } stsEndpoint := awsStsEndpointUSWest2 cp := &customProvider{ - stsProvider: miniocreds.STSAssumeRole{ + stsProvider: credentials.STSAssumeRole{ Client: &http.Client{ Transport: http.DefaultTransport, }, @@ -846,5 +829,5 @@ func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) // Initialize expired to false cp.forceExpired.Store(false) - return miniocreds.New(cp), cp + return credentials.New(cp), cp }