From 97b0c02e365c729c68c20d36d804d1c0ab683bc2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Julio=20L=C3=B3pez?= <1953782+julio-lopez@users.noreply.github.com> Date: Thu, 26 Sep 2024 22:22:10 -0700 Subject: [PATCH] fix(providers): GCS tests (#4136) * `getCredJSONFromEnv` helper * use `getCredJSONFromEnv` * `getEnvVarOrSkip` helper * skip GCS immutable tests if bucket name is not provided --- repo/blob/gcs/gcs_immu_test.go | 16 ++++----- repo/blob/gcs/gcs_storage_test.go | 50 +++++++++++++++++++++-------- repo/blob/gcs/gcs_versioned_test.go | 32 ++++++++++-------- 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/repo/blob/gcs/gcs_immu_test.go b/repo/blob/gcs/gcs_immu_test.go index a545260fb..bd0163898 100644 --- a/repo/blob/gcs/gcs_immu_test.go +++ b/repo/blob/gcs/gcs_immu_test.go @@ -27,8 +27,8 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) { opts := bucketOpts{ projectID: os.Getenv(testBucketProjectID), - bucket: os.Getenv(testImmutableBucketEnv), - credentialsFile: os.Getenv(testBucketCredentialsFile), + bucket: getImmutableBucketNameOrSkip(t), + credentialsJSON: getCredJSONFromEnv(t), isLockedBucket: true, } createBucket(t, opts) @@ -43,9 +43,9 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) { newctx, cancel := context.WithCancel(ctx) prefix := fmt.Sprintf("test-%v-%x/", clock.Now().Unix(), data) st, err := gcs.New(newctx, &gcs.Options{ - BucketName: opts.bucket, - ServiceAccountCredentialsFile: opts.credentialsFile, - Prefix: prefix, + BucketName: opts.bucket, + ServiceAccountCredentialJSON: opts.credentialsJSON, + Prefix: prefix, }, false) cancel() @@ -67,7 +67,7 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) { } err = st.PutBlob(ctx, dummyBlob, gather.FromSlice([]byte("x")), putOpts) require.NoError(t, err) - cli := getGoogleCLI(t, opts.credentialsFile) + cli := getGoogleCLI(t, opts.credentialsJSON) count := getBlobCount(ctx, t, st, dummyBlob[:1]) require.Equal(t, 1, count) @@ -112,11 +112,11 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) { } // getGoogleCLI returns a separate client to verify things the Storage interface doesn't support. -func getGoogleCLI(t *testing.T, credentialsFile string) *gcsclient.Client { +func getGoogleCLI(t *testing.T, credentialsJSON []byte) *gcsclient.Client { t.Helper() ctx := context.Background() - cli, err := gcsclient.NewClient(ctx, option.WithCredentialsFile(credentialsFile)) + cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(credentialsJSON)) if err != nil { t.Fatalf("unable to create GCS client: %v", err) } diff --git a/repo/blob/gcs/gcs_storage_test.go b/repo/blob/gcs/gcs_storage_test.go index d56b3cd9a..28959ceb0 100644 --- a/repo/blob/gcs/gcs_storage_test.go +++ b/repo/blob/gcs/gcs_storage_test.go @@ -33,7 +33,7 @@ type bucketOpts struct { bucket string - credentialsFile string + credentialsJSON []byte projectID string isLockedBucket bool } @@ -42,7 +42,7 @@ func createBucket(t *testing.T, opts bucketOpts) { t.Helper() ctx := context.Background() - cli, err := gcsclient.NewClient(ctx, option.WithCredentialsFile(opts.credentialsFile)) + cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON)) if err != nil { t.Fatalf("unable to create GCS client: %v", err) } @@ -75,7 +75,7 @@ func validateBucket(t *testing.T, opts bucketOpts) { t.Helper() ctx := context.Background() - cli, err := gcsclient.NewClient(ctx, option.WithCredentialsFile(opts.credentialsFile)) + cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON)) if err != nil { t.Fatalf("unable to create GCS client: %v", err) } @@ -151,6 +151,38 @@ func gunzip(d []byte) ([]byte, error) { return io.ReadAll(z) } +func getEnvVarOrSkip(t *testing.T, envVarName string) string { + t.Helper() + + v := os.Getenv(envVarName) + if v == "" { + t.Skipf("%q is not set", envVarName) + } + + return v +} + +func getCredJSONFromEnv(t *testing.T) []byte { + t.Helper() + + b64Data := os.Getenv(testBucketCredentialsJSONGzip) + if b64Data == "" { + t.Skip(testBucketCredentialsJSONGzip + "is not set") + } + + credDataGZ, err := base64.StdEncoding.DecodeString(b64Data) + if err != nil { + t.Skip("skipping test because GCS credentials file can't be decoded") + } + + credJSON, err := gunzip(credDataGZ) + if err != nil { + t.Skip("skipping test because GCS credentials file can't be unzipped") + } + + return credJSON +} + func mustGetOptionsOrSkip(t *testing.T, prefix string) *gcs.Options { t.Helper() @@ -159,19 +191,9 @@ func mustGetOptionsOrSkip(t *testing.T, prefix string) *gcs.Options { t.Skip("KOPIA_GCS_TEST_BUCKET not provided") } - credDataGZ, err := base64.StdEncoding.DecodeString(os.Getenv(testBucketCredentialsJSONGzip)) - if err != nil { - t.Skip("skipping test because GCS credentials file can't be decoded") - } - - credData, err := gunzip(credDataGZ) - if err != nil { - t.Skip("skipping test because GCS credentials file can't be unzipped") - } - return &gcs.Options{ BucketName: bucket, - ServiceAccountCredentialJSON: credData, + ServiceAccountCredentialJSON: getCredJSONFromEnv(t), Prefix: prefix, } } diff --git a/repo/blob/gcs/gcs_versioned_test.go b/repo/blob/gcs/gcs_versioned_test.go index 5b7be2d15..e68b5284e 100644 --- a/repo/blob/gcs/gcs_versioned_test.go +++ b/repo/blob/gcs/gcs_versioned_test.go @@ -35,9 +35,9 @@ func TestGetBlobVersionsFailsWhenVersioningDisabled(t *testing.T) { prefix := fmt.Sprintf("test-%v-%x/", clock.Now().Unix(), data) opts := &gcs.Options{ - BucketName: bucket, - ServiceAccountCredentialsFile: os.Getenv(testBucketCredentialsFile), - Prefix: prefix, + BucketName: bucket, + ServiceAccountCredentialJSON: getCredJSONFromEnv(t), + Prefix: prefix, } st, err := gcs.New(newctx, opts, false) require.NoError(t, err) @@ -59,8 +59,8 @@ func TestGetBlobVersions(t *testing.T) { // must be with Versioning enabled. bOpts := bucketOpts{ projectID: os.Getenv(testBucketProjectID), - bucket: os.Getenv(testImmutableBucketEnv), - credentialsFile: os.Getenv(testBucketCredentialsFile), + bucket: getImmutableBucketNameOrSkip(t), + credentialsJSON: getCredJSONFromEnv(t), isLockedBucket: true, } @@ -76,9 +76,9 @@ func TestGetBlobVersions(t *testing.T) { prefix := fmt.Sprintf("test-%v-%x/", clock.Now().Unix(), data) opts := &gcs.Options{ - BucketName: bOpts.bucket, - ServiceAccountCredentialsFile: bOpts.credentialsFile, - Prefix: prefix, + BucketName: bOpts.bucket, + ServiceAccountCredentialJSON: bOpts.credentialsJSON, + Prefix: prefix, } st, err := gcs.New(newctx, opts, false) require.NoError(t, err) @@ -166,8 +166,8 @@ func TestGetBlobVersionsWithDeletion(t *testing.T) { // must be with Versioning enabled. bOpts := bucketOpts{ projectID: os.Getenv(testBucketProjectID), - bucket: os.Getenv(testImmutableBucketEnv), - credentialsFile: os.Getenv(testBucketCredentialsFile), + bucket: getImmutableBucketNameOrSkip(t), + credentialsJSON: getCredJSONFromEnv(t), isLockedBucket: true, } @@ -183,9 +183,9 @@ func TestGetBlobVersionsWithDeletion(t *testing.T) { prefix := fmt.Sprintf("test-%v-%x/", clock.Now().Unix(), data) opts := &gcs.Options{ - BucketName: bOpts.bucket, - ServiceAccountCredentialsFile: bOpts.credentialsFile, - Prefix: prefix, + BucketName: bOpts.bucket, + ServiceAccountCredentialJSON: bOpts.credentialsJSON, + Prefix: prefix, } st, err := gcs.New(newctx, opts, false) require.NoError(t, err) @@ -256,3 +256,9 @@ func putBlobs(ctx context.Context, cli blob.Storage, blobID blob.ID, blobs []str return putTimes, nil } + +func getImmutableBucketNameOrSkip(t *testing.T) string { + t.Helper() + + return getEnvVarOrSkip(t, testImmutableBucketEnv) +}