From 7b1e652c1b5722944f72a587ed3ced48cfd92e60 Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Fri, 11 Feb 2022 17:29:24 -0800 Subject: [PATCH] build(deps): update storage/azblob to v0.3.0 (#1743) This helped remove ugly workaround we had in place due to upstream bug. --- go.mod | 6 ++-- go.sum | 14 ++++----- repo/blob/azure/azure_storage.go | 50 ++++++++++---------------------- 3 files changed, 24 insertions(+), 46 deletions(-) diff --git a/go.mod b/go.mod index 2f55eec30..cfe70452e 100644 --- a/go.mod +++ b/go.mod @@ -70,9 +70,9 @@ require ( require ( cloud.google.com/go v0.100.2 // indirect - github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 // indirect - github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 // indirect - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 github.com/Azure/go-autorest/autorest/adal v0.9.16 // indirect github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect github.com/beorn7/perks v1.0.1 // indirect diff --git a/go.sum b/go.sum index 4aec19634..8b79e41b1 100644 --- a/go.sum +++ b/go.sum @@ -58,12 +58,12 @@ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7 dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0 h1:KQgdWmEOmaJKxaUUZwHAYh12t+b+ZJf8q3friycK1kA= -github.com/Azure/azure-sdk-for-go/sdk/azcore v0.20.0/go.mod h1:ZPW/Z0kLCTdDZaDbYTetxc9Cxl/2lNqxYHYNOF2bti0= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1 h1:BUYIbDf/mMZ8945v3QkG3OuqGVyS4Iek0AOLwdRAYoc= -github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.1/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0 h1:62Ew5xXg5UCGIXDOM7+y4IL5/6mQJq1nenhBCJAeGX8= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.2.0/go.mod h1:eHWhQKXc1Gv1DvWH//UzgWjWFEo0Pp4pH2vBzjBw8Fc= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1 h1:qoVeMsc9/fh/yhxVaA0obYjVH/oI/ihrOoMwsLS9KSA= +github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3 h1:E+m3SkZCN0Bf5q7YdTs5lSm2CYY3CK4spn5OmUIiQtk= +github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0 h1:Px2UA+2RvSSvv+RvJNuUB6n7rs5Wsel4dXLe90Um2n4= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/azure-storage-blob-go v0.14.0 h1:1BCg74AmVdYwO3dlKwtFU1V0wU2PZdREkXvAmZJRUlM= github.com/Azure/azure-storage-blob-go v0.14.0/go.mod h1:SMqIBi+SuiQH32bvyjngEewEeXoPfKMgWlBDaYf6fck= github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= @@ -400,8 +400,6 @@ github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kopia/htmluibuild v0.0.0-20220131042136-3e252f22c951 h1:5Ax9T6D+EFzIP0uTabNDTVQQPCOk4QlD/VnL+6cN8E4= -github.com/kopia/htmluibuild v0.0.0-20220131042136-3e252f22c951/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw= github.com/kopia/htmluibuild v0.0.0-20220206023946-bd8fce6ec20d h1:bE9j+g8iiICIJjYr6K56OrnxRFKSfrnLCpOhD1vOBTc= github.com/kopia/htmluibuild v0.0.0-20220206023946-bd8fce6ec20d/go.mod h1:eWer4rx9P8lJo2eKc+Q7AZ1dE1x1hJNdkbDFPzMu1Hw= github.com/kopia/htmluibuild v0.0.0-20220208062920-a1d47d1e3096 h1:rbrPWsO4OTT2ce4q2rSsh/LlApYu+Sr8EDZEa5jV3zg= diff --git a/repo/blob/azure/azure_storage.go b/repo/blob/azure/azure_storage.go index b4f52b9e3..d656c152a 100644 --- a/repo/blob/azure/azure_storage.go +++ b/repo/blob/azure/azure_storage.go @@ -3,9 +3,7 @@ package azure import ( "context" - "encoding/xml" "fmt" - "time" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/pkg/errors" @@ -54,7 +52,7 @@ func (az *azStorage) GetBlob(ctx context.Context, b blob.ID, offset, length int6 return translateError(err) } - body := resp.Body(azblob.RetryReaderOptions{}) + body := resp.Body(nil) defer body.Close() // nolint:errcheck if length == 0 { @@ -170,45 +168,19 @@ func (az *azStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback fun for pager.NextPage(ctx) { resp := pager.PageResponse() - // workaround for the XML parsing bug reported upstream - // https://github.com/Azure/azure-sdk-for-go/issues/16679 - var enumerationResults struct { - Blobs struct { - Blob []struct { - Name string - Properties struct { - ContentLength int64 `xml:"Content-Length"` - LastModified string `xml:"Last-Modified"` - } - Metadata struct { - Kopiamtime string - } - } - } - } + for _, it := range resp.Segment.BlobItems { + n := *it.Name - dec := xml.NewDecoder(resp.RawResponse.Body) - if err := dec.Decode(&enumerationResults); err != nil { - return errors.Wrap(err, "unable to decode response") - } - - for _, it := range enumerationResults.Blobs.Blob { bm := blob.Metadata{ - BlobID: blob.ID(it.Name[len(az.Prefix):]), - Length: it.Properties.ContentLength, + BlobID: blob.ID(n[len(az.Prefix):]), + Length: *it.Properties.ContentLength, } // see if we have 'Kopiamtime' metadata, if so - trust it. - if t, ok := timestampmeta.FromValue(it.Metadata.Kopiamtime); ok { + if t, ok := timestampmeta.FromValue(stringDefault(it.Metadata["kopiamtime"], "")); ok { bm.Timestamp = t } else { - // fall back to using last modified time. - t, err := time.Parse(time.RFC1123, it.Properties.LastModified) - if err != nil { - return errors.Wrapf(err, "invalid timestamp for BLOB '%v': %q", bm.BlobID, it.Properties.LastModified) - } - - bm.Timestamp = t + bm.Timestamp = *it.Properties.LastModified } if err := callback(bm); err != nil { @@ -220,6 +192,14 @@ func (az *azStorage) ListBlobs(ctx context.Context, prefix blob.ID, callback fun return translateError(pager.Err()) } +func stringDefault(s *string, def string) string { + if s == nil { + return def + } + + return *s +} + func (az *azStorage) ConnectionInfo() blob.ConnectionInfo { return blob.ConnectionInfo{ Type: azStorageType,