mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-03-05 06:47:12 -05:00
Code Issues Packages Projects Releases Wiki Activity

refactor(providers): use minio SDK credentials package in S3 STS test (#3148)

Browse Source 
* refactor: return credentials.Value in createMinioSessionToken helper
* refactor: use minio SDK credentials package in STS test
* go mod tidy -compat=1.20 => removes dependency on AWS SDK v1 packages
* cleanup: unalias use of minio credentials package
This commit is contained in:
Julio Lopez
2023-07-16 09:51:40 -07:00
committed by GitHub
parent d8a33bdd44
commit f962d93159
3 changed files with 35 additions and 67 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@@ -9,7 +9,6 @@ require (
github.com/Azure/azure-storage-blob-go v0.15.0
github.com/alecthomas/kingpin/v2 v2.3.2
github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
github.com/aws/aws-sdk-go v1.44.298
github.com/chmduquesne/rollinghash v4.0.0+incompatible
github.com/chromedp/cdproto v0.0.0-20230220211738-2b1ec77315c9
github.com/chromedp/chromedp v0.9.1
@@ -99,7 +98,6 @@ require (
github.com/google/s2a-go v0.1.4 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.11.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/json-iterator/go v1.1.12 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect

13
go.sum
View File

@@ -45,8 +45,6 @@ github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8V
github.com/alessio/shellescape v1.4.1 h1:V7yhSDDn8LP4lc4jS8pFkt0zCnzVJlG5JXy9BVKJUX0=
github.com/alessio/shellescape v1.4.1/go.mod h1:PZAiSCk0LJaZkiCSkPv8qIobYglO3FPpyFjDCtHLS30=
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/aws/aws-sdk-go v1.44.298 h1:5qTxdubgV7PptZJmp/2qDwD2JL187ePL7VOxsSh1i3g=
github.com/aws/aws-sdk-go v1.44.298/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
@@ -180,10 +178,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFb
github.com/hanwen/go-fuse/v2 v2.3.0 h1:t5ivNIH2PK+zw4OBul/iJjsoG9K6kXo4nMDoBpciC8A=
github.com/hanwen/go-fuse/v2 v2.3.0/go.mod h1:xKwi1cF7nXAOBCXujD5ie0ZKsxc8GGSA1rlMJc+8IJs=
github.com/ianlancetaylor/demangle v0.0.0-20210905161508-09a460cdf81d/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -198,8 +192,6 @@ github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU
github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY=
github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A=
github.com/kopia/htmluibuild v0.0.0-20230605144737-e386b860759d h1:qvV3TN5X/RsgmckkxsKh9P7Vtf9GYy6vOPzQY1SY4qM=
github.com/kopia/htmluibuild v0.0.0-20230605144737-e386b860759d/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
github.com/kopia/htmluibuild v0.0.0-20230714030926-ff7ed652d883 h1:LX3CQvH12mNYXt+rBUqxwbwAieb9QmDrMXjKVAuMxvQ=
github.com/kopia/htmluibuild v0.0.0-20230714030926-ff7ed652d883/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw=
github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8=
@@ -362,7 +354,6 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50=
golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -393,14 +384,12 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc
golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA=
golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c=
golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
@@ -409,7 +398,6 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4=
golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -474,7 +462,6 @@ gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216 h1:2TSTkQ8PMv
gopkg.in/kothar/go-backblaze.v0 v0.0.0-20210124194846-35409b867216/go.mod h1:zJ2QpyDCYo1KvLXlmdnFlQAyF/Qfth0fB8239Qg7BIE=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

87
repo/blob/s3/s3_storage_test.go
View File

@@ -13,13 +13,9 @@
"testing"
"time"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/sts"
"github.com/google/uuid"
"github.com/minio/minio-go/v7"
miniocreds "github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/stretchr/testify/require"
"github.com/kopia/kopia/internal/blobtesting"
@@ -477,7 +473,7 @@ func TestS3StorageMinioSTS(t *testing.T) {
time.Sleep(2 * time.Second)
kopiaAccessKeyID, kopiaSecretKey, kopiaSessionToken := createMinioSessionToken(t, minioEndpoint, minioRootAccessKeyID, minioRootSecretAccessKey, minioBucketName)
kopiaCreds := createMinioSessionToken(t, minioEndpoint, minioRootAccessKeyID, minioRootSecretAccessKey, minioBucketName)
createBucket(t, &Options{
Endpoint: minioEndpoint,
@@ -488,14 +484,15 @@ func TestS3StorageMinioSTS(t *testing.T) {
DoNotUseTLS: true,
})
require.NotEqual(t, kopiaAccessKeyID, minioRootAccessKeyID)
require.NotEqual(t, kopiaSecretKey, minioRootSecretAccessKey)
require.NotEqual(t, kopiaCreds.AccessKeyID, minioRootAccessKeyID)
require.NotEqual(t, kopiaCreds.SecretAccessKey, minioRootSecretAccessKey)
require.NotEmpty(t, kopiaCreds.SessionToken)
testStorage(t, &Options{
Endpoint: minioEndpoint,
AccessKeyID: kopiaAccessKeyID,
SecretAccessKey: kopiaSecretKey,
SessionToken: kopiaSessionToken,
AccessKeyID: kopiaCreds.AccessKeyID,
SecretAccessKey: kopiaCreds.SecretAccessKey,
SessionToken: kopiaCreds.SessionToken,
BucketName: minioBucketName,
Region: minioRegion,
DoNotUseTLS: true,
@@ -658,7 +655,7 @@ func createClient(tb testing.TB, opt *Options) *minio.Client {
minioClient, err := minio.New(opt.Endpoint,
&minio.Options{
Creds: miniocreds.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, ""),
Creds: credentials.NewStaticV4(opt.AccessKeyID, opt.SecretAccessKey, ""),
Secure: !opt.DoNotUseTLS,
Region: opt.Region,
Transport: transport,
@@ -723,28 +720,14 @@ func makeBucket(tb testing.TB, cli *minio.Client, opt *Options, objectLocking bo
}
}
func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUserPasswd, bucketName string) (accessID, secretKey, sessionToken string) {
func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUserPasswd, bucketName string) credentials.Value {
t.Helper()
// Configure to use MinIO Server
awsConfig := &aws.Config{
Credentials: credentials.NewStaticCredentials(kopiaUserName, kopiaUserPasswd, ""),
Endpoint: aws.String(minioEndpoint),
Region: aws.String(minioRegion),
S3ForcePathStyle: aws.Bool(true),
DisableSSL: aws.Bool(true),
}
awsSession, err := session.NewSession(awsConfig)
if err != nil {
t.Fatalf("failed to create aws session: %v", err)
}
svc := sts.New(awsSession)
input := &sts.AssumeRoleInput{
// give access to only S3 bucket with name bucketName
Policy: aws.String(fmt.Sprintf(`{
stsOpts := credentials.STSAssumeRoleOptions{
AccessKey: kopiaUserName,
SecretKey: kopiaUserPasswd,
DurationSeconds: 900,
Policy: fmt.Sprintf(`{
"Version":"2012-10-17",
"Statement":[
{
@@ -759,25 +742,25 @@ func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUs
"Action": "s3:*",
"Resource": "arn:aws:s3:::%v/*"
}
]}`, bucketName, bucketName)),
]}`, bucketName, bucketName),
// RoleArn and RoleSessionName are not meaningful for MinIO and can be set to any value
RoleArn: aws.String("arn:xxx:xxx:xxx:xxxx"),
RoleSessionName: aws.String("kopiaTestSession"),
DurationSeconds: aws.Int64(900), // in seconds
RoleARN: "arn:xxx:xxx:xxx:xxxx",
RoleSessionName: "kopiaTestSession",
}
result, err := svc.AssumeRole(input)
if err != nil {
t.Fatalf("failed to create session with aws assume role: %v", err)
if !strings.HasPrefix(minioEndpoint, "http") {
minioEndpoint = "http://" + minioEndpoint
}
if result.Credentials == nil {
t.Fatalf("couldn't find aws creds in aws assume role response")
}
// Get STS credentials from MinIO server
roleCreds, err := credentials.NewSTSAssumeRole(minioEndpoint, stsOpts)
require.NoError(t, err, "during STSAssumeRole:", minioEndpoint)
require.NotNil(t, roleCreds)
t.Logf("created session token with assume role: expiration: %s", result.Credentials.Expiration)
credsValue, err := roleCreds.Get()
require.NoError(t, err)
return *result.Credentials.AccessKeyId, *result.Credentials.SecretAccessKey, *result.Credentials.SessionToken
return credsValue
}
// customProvider is a custom provider based on minio's STSAssumeRole struct
@@ -789,7 +772,7 @@ func createMinioSessionToken(t *testing.T, minioEndpoint, kopiaUserName, kopiaUs
// the next call to Retrieve to return expired credentials.
type customProvider struct {
forceExpired atomic.Bool
stsProvider miniocreds.STSAssumeRole
stsProvider credentials.STSAssumeRole
}
const expiredSessionToken = "IQoJb3JpZ2luX2VjEBMaCXVzLXdlc3QtMiJIM" +
@@ -806,13 +789,13 @@ type customProvider struct {
"RCcVWcntllxyL/sUZ7VbMr7xZxWWbilu8pVtQqTwwBxZO0rth8XftMzGQ5oyd" +
"82CdcwRB+t7K1LEmRErltbteGtM="
func (cp *customProvider) Retrieve() (miniocreds.Value, error) {
func (cp *customProvider) Retrieve() (credentials.Value, error) {
if cp.forceExpired.Load() {
return miniocreds.Value{
return credentials.Value{
AccessKeyID: "ASIAQREAKNKDBR4F5F2I",
SecretAccessKey: "EF82nKmZbnFETa96xxx1C3k20hG4Nw+2v+FBNjp3",
SessionToken: expiredSessionToken,
SignerType: miniocreds.SignatureV2,
SignerType: credentials.SignatureV2,
}, nil
}
@@ -825,8 +808,8 @@ func (cp *customProvider) IsExpired() bool {
// customCredentialsAndProvider creates a custom provider and returns credentials
// using this provider.
func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) (*miniocreds.Credentials, *customProvider) {
opts := miniocreds.STSAssumeRoleOptions{
func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string) (*credentials.Credentials, *customProvider) {
opts := credentials.STSAssumeRoleOptions{
AccessKey: accessKey,
SecretKey: secretKey,
Location: region,
@@ -835,7 +818,7 @@ func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string)
}
stsEndpoint := awsStsEndpointUSWest2
cp := &customProvider{
stsProvider: miniocreds.STSAssumeRole{
stsProvider: credentials.STSAssumeRole{
Client: &http.Client{
Transport: http.DefaultTransport,
},
@@ -846,5 +829,5 @@ func customCredentialsAndProvider(accessKey, secretKey, roleARN, region string)
// Initialize expired to false
cp.forceExpired.Store(false)
return miniocreds.New(cp), cp
return credentials.New(cp), cp
}