mirror/kopia
1
0
Fork 0
mirror of https://github.com/kopia/kopia.git synced 2026-04-23 07:29:15 -04:00
Code Issues Packages Projects Releases Wiki Activity

azure: added support for using SAS Tokens instead of storage keys (#1093)

Browse Source 
Fixes #1071
This commit is contained in:
Jarek Kowalski
2021-05-22 12:43:55 -07:00
committed by GitHub
parent 76490dc361
commit e15a79474c
8 changed files with 99 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.github/workflows/provider-tests.yml vendored
View File

@@ -21,6 +21,7 @@ jobs:
KOPIA_AZURE_TEST_CONTAINER: ${{ secrets.KOPIA_AZURE_TEST_CONTAINER }}
KOPIA_AZURE_TEST_STORAGE_ACCOUNT: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_ACCOUNT }}
KOPIA_AZURE_TEST_STORAGE_KEY: ${{ secrets.KOPIA_AZURE_TEST_STORAGE_KEY }}
KOPIA_AZURE_TEST_SAS_TOKEN: ${{ secrets.KOPIA_AZURE_TEST_SAS_TOKEN }}
# used in B2 tests
KOPIA_B2_TEST_BUCKET: ${{ secrets.KOPIA_B2_TEST_BUCKET }}

6
cli/storage_azure.go
View File

@@ -15,8 +15,10 @@ type storageAzureFlags struct {
func (c *storageAzureFlags) setup(_ storageProviderServices, cmd *kingpin.CmdClause) {
cmd.Flag("container", "Name of the Azure blob container").Required().StringVar(&c.azOptions.Container)
cmd.Flag("storage-account", "Azure storage account name(overrides AZURE_STORAGE_ACCOUNT environment variable)").Required().Envar("AZURE_STORAGE_ACCOUNT").StringVar(&c.azOptions.StorageAccount)
cmd.Flag("storage-key", "Azure storage account key(overrides AZURE_STORAGE_KEY environment variable)").Required().Envar("AZURE_STORAGE_KEY").StringVar(&c.azOptions.StorageKey)
cmd.Flag("storage-account", "Azure storage account name (overrides AZURE_STORAGE_ACCOUNT environment variable)").Required().Envar("AZURE_STORAGE_ACCOUNT").StringVar(&c.azOptions.StorageAccount)
cmd.Flag("storage-key", "Azure storage account key (overrides AZURE_STORAGE_KEY environment variable)").Envar("AZURE_STORAGE_KEY").StringVar(&c.azOptions.StorageKey)
cmd.Flag("storage-domain", "Azure storage domain").Envar("AZURE_STORAGE_DOMAIN").StringVar(&c.azOptions.StorageDomain)
cmd.Flag("sas-token", "Azure SAS Token").Envar("AZURE_STORAGE_SAS_TOKEN").StringVar(&c.azOptions.SASToken)
cmd.Flag("prefix", "Prefix to use for objects in the bucket").StringVar(&c.azOptions.Prefix)
cmd.Flag("max-download-speed", "Limit the download speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&c.azOptions.MaxDownloadSpeedBytesPerSecond)
cmd.Flag("max-upload-speed", "Limit the upload speed.").PlaceHolder("BYTES_PER_SEC").IntVar(&c.azOptions.MaxUploadSpeedBytesPerSecond)

1
go.mod
View File

@@ -5,6 +5,7 @@ go 1.16
require (
cloud.google.com/go/storage v1.15.0
contrib.go.opencensus.io/exporter/prometheus v0.3.0
github.com/Azure/azure-pipeline-go v0.2.3 // indirect
github.com/Azure/azure-storage-blob-go v0.13.0
github.com/GehirnInc/crypt v0.0.0-20200316065508-bb7000b8a962 // indirect
github.com/StackExchange/wmi v0.0.0-20210224194228-fe8f1750fd46 // indirect

12
htmlui/src/SetupAzure.js
View File

@@ -13,18 +13,22 @@ export class SetupAzure extends Component {
}
validate() {
return validateRequiredFields(this, ["container", "storageAccount", "storageKey"])
return validateRequiredFields(this, ["container", "storageAccount"])
}
render() {
return <>
<Form.Row>
{RequiredField(this, "Container", "container", { autoFocus: true, placeholder: "enter container name" })}
{OptionalField(this, "Object Name Prefix", "prefix", { placeholder: "enter object name prefix or leave empty", type: "password" })}
{OptionalField(this, "Object Name Prefix", "prefix", { placeholder: "enter object name prefix or leave empty" })}
</Form.Row>
<Form.Row>
{RequiredField(this, "Access Key ID", "storageAccount", { placeholder: "enter access key ID" })}
{RequiredField(this, "Secret Access Key", "storageKey", { placeholder: "enter secret access key", type: "password" })}
{RequiredField(this, "Storage Account", "storageAccount", { placeholder: "enter access key ID" })}
{OptionalField(this, "Access Key", "storageKey", { placeholder: "enter secret access key", type: "password" })}
</Form.Row>
<Form.Row>
{OptionalField(this, "Azure Storage Domain", "storageDomain", { placeholder: "enter storage domain or leave empty for default 'blob.core.windows.net'" })}
{OptionalField(this, "SAS Token", "sasToken", { placeholder: "enter secret SAS Token", type: "password" })}
</Form.Row>
</>;
}

6
htmlui/src/tests/SetupAzure.test.js
View File

@@ -11,9 +11,11 @@ it('can set fields', async () => {
// required
changeControlValue(getByTestId("control-container"), "some-container");
changeControlValue(getByTestId("control-storageAccount"), "some-storageAccount");
changeControlValue(getByTestId("control-storageKey"), "some-storageKey");
expect(ref.current.validate()).toBe(true);
// optional
changeControlValue(getByTestId("control-storageKey"), "some-storageKey");
changeControlValue(getByTestId("control-sasToken"), "some-sas-token");
changeControlValue(getByTestId("control-storageDomain"), "some-storage-domain");
changeControlValue(getByTestId("control-prefix"), "some-prefix");
expect(ref.current.validate()).toBe(true);
@@ -22,5 +24,7 @@ it('can set fields', async () => {
"container": "some-container",
"prefix": "some-prefix",
"storageKey": "some-storageKey",
"sasToken": "some-sas-token",
"storageDomain": "some-storage-domain",
});
});

5
repo/blob/azure/azure_options.go
View File

@@ -12,6 +12,11 @@ type Options struct {
StorageAccount string `json:"storageAccount"`
StorageKey string `json:"storageKey" kopia:"sensitive"`
// Alternatively provide SAS Token
SASToken string `json:"sasToken" kopia:"sensitive"`
StorageDomain string `json:"storageDomain,omitempty"`
MaxUploadSpeedBytesPerSecond int `json:"maxUploadSpeedBytesPerSecond,omitempty"`
MaxDownloadSpeedBytesPerSecond int `json:"maxDownloadSpeedBytesPerSecond,omitempty"`
}

34
repo/blob/azure/azure_storage.go
View File

@@ -9,6 +9,7 @@
"net/http"
"time"
"github.com/Azure/azure-pipeline-go/pipeline"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/efarrer/iothrottler"
"github.com/pkg/errors"
@@ -218,17 +219,36 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) {
return nil, errors.New("container name must be specified")
}
// create a credentials object.
credential, err := azureblob.NewCredential(azureblob.AccountName(opt.StorageAccount), azureblob.AccountKey(opt.StorageKey))
if err != nil {
return nil, errors.Wrap(err, "unable to initialize credentials")
var (
abo azureblob.Options
pl pipeline.Pipeline
pipelineOpts azblob.PipelineOptions
account = azureblob.AccountName(opt.StorageAccount)
)
if opt.SASToken != "" {
abo.SASToken = azureblob.SASToken(opt.SASToken)
// don't set abo.Credential
pl = azureblob.NewPipeline(azblob.NewAnonymousCredential(), pipelineOpts)
} else {
if opt.StorageKey == "" {
return nil, errors.Errorf("either storage key or SAS token must be provided")
}
// create a credentials object.
cred, err := azureblob.NewCredential(account, azureblob.AccountKey(opt.StorageKey))
if err != nil {
return nil, errors.Wrap(err, "unable to initialize credentials")
}
abo.Credential = cred
pl = azureblob.NewPipeline(cred, pipelineOpts)
}
// create a Pipeline with credentials.
pipeline := azureblob.NewPipeline(credential, azblob.PipelineOptions{})
abo.StorageDomain = azureblob.StorageDomain(opt.StorageDomain)
// create a *blob.Bucket.
bucket, err := azureblob.OpenBucket(ctx, pipeline, azureblob.AccountName(opt.StorageAccount), opt.Container, &azureblob.Options{Credential: credential})
bucket, err := azureblob.OpenBucket(ctx, pl, account, opt.Container, &abo)
if err != nil {
return nil, errors.Wrap(err, "unable to open bucket")
}

51
repo/blob/azure/azure_storage_test.go
View File

@@ -20,9 +20,10 @@
)
const (
testContainerEnv = "KOPIA_AZURE_TEST_CONTAINER"
testStorageAccountEnv = "KOPIA_AZURE_TEST_STORAGE_ACCOUNT"
testStorageKeyEnv = "KOPIA_AZURE_TEST_STORAGE_KEY"
testContainerEnv = "KOPIA_AZURE_TEST_CONTAINER"
testStorageAccountEnv = "KOPIA_AZURE_TEST_STORAGE_ACCOUNT"
testStorageKeyEnv = "KOPIA_AZURE_TEST_STORAGE_KEY"
testStorageSASTokenEnv = "KOPIA_AZURE_TEST_SAS_TOKEN"
)
func getEnvOrSkip(t *testing.T, name string) string {
@@ -117,6 +118,50 @@ func TestAzureStorage(t *testing.T) {
}
}
func TestAzureStorageSASToken(t *testing.T) {
t.Parallel()
testutil.ProviderTest(t)
container := getEnvOrSkip(t, testContainerEnv)
storageAccount := getEnvOrSkip(t, testStorageAccountEnv)
sasToken := getEnvOrSkip(t, testStorageSASTokenEnv)
data := make([]byte, 8)
rand.Read(data)
ctx := testlogging.Context(t)
st, err := azure.New(ctx, &azure.Options{
Container: container,
StorageAccount: storageAccount,
SASToken: sasToken,
Prefix: fmt.Sprintf("sastest-%v-%x-", clock.Now().Unix(), data),
})
if err != nil {
t.Fatalf("unable to connect to Azure: %v", err)
}
if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error {
return st.DeleteBlob(ctx, bm.BlobID)
}); err != nil {
t.Fatalf("unable to clear Azure blob container: %v", err)
}
blobtesting.VerifyStorage(ctx, t, st)
blobtesting.AssertConnectionInfoRoundTrips(ctx, t, st)
// delete everything again
if err := st.ListBlobs(ctx, "", func(bm blob.Metadata) error {
return st.DeleteBlob(ctx, bm.BlobID)
}); err != nil {
t.Fatalf("unable to clear Azure blob container: %v", err)
}
if err := st.Close(ctx); err != nil {
t.Fatalf("err: %v", err)
}
}
func TestAzureStorageInvalidBlob(t *testing.T) {
testutil.ProviderTest(t)