repo: added Repository.Upgrade() API

This commit is contained in:
Jarek Kowalski
2018-12-31 17:08:41 -08:00
parent b4c80348f8
commit 24bd5bbe1f
8 changed files with 112 additions and 40 deletions

24
go.mod
View File

@@ -1,22 +1,22 @@
module github.com/kopia/repo
require (
cloud.google.com/go v0.32.0
cloud.google.com/go v0.34.0
github.com/efarrer/iothrottler v0.0.0-20141121142253-60e7e547c7fe
github.com/go-ini/ini v1.39.0 // indirect
github.com/googleapis/gax-go v2.0.0+incompatible // indirect
github.com/minio/minio-go v6.0.9+incompatible
github.com/go-ini/ini v1.40.0 // indirect
github.com/googleapis/gax-go v2.0.2+incompatible // indirect
github.com/minio/minio-go v6.0.11+incompatible
github.com/mitchellh/go-homedir v1.0.0 // indirect
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6
github.com/studio-b12/gowebdav v0.0.0-20181024110551-cba565a9dcfc
github.com/studio-b12/gowebdav v0.0.0-20181230112802-6c32839dbdfc
go.opencensus.io v0.18.0 // indirect
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16
golang.org/x/exp v0.0.0-20181022080537-42ba7d4b6eb2
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc
golang.org/x/oauth2 v0.0.0-20181102170140-232e45548389
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc // indirect
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9
golang.org/x/exp v0.0.0-20181221233300-b68661188fbf
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 // indirect
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb // indirect
google.golang.org/api v0.0.0-20181229000844-f26a60c56f14
google.golang.org/genproto v0.0.0-20181101192439-c830210a61df // indirect
google.golang.org/grpc v1.16.0 // indirect
google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f // indirect
google.golang.org/grpc v1.17.0 // indirect
)

49
go.sum
View File

@@ -1,27 +1,27 @@
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.32.0 h1:DSt59WoyNcfAInilEpfvm2ugq8zvNyaHAm9MkzOwRQ4=
cloud.google.com/go v0.32.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/efarrer/iothrottler v0.0.0-20141121142253-60e7e547c7fe h1:WAx1vRufH0I2pTWldQkXPzpc+jndCOi2FH334LFQ1PI=
github.com/efarrer/iothrottler v0.0.0-20141121142253-60e7e547c7fe/go.mod h1:zjXkUoNEq44qYz/1TlzBhN2W21rDU3HvDBiJWQAZTq8=
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
github.com/go-ini/ini v1.39.0 h1:/CyW/jTlZLjuzy52jc1XnhJm6IUKEuunpJFpecywNeI=
github.com/go-ini/ini v1.39.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/go-ini/ini v1.40.0 h1:/pbZah2UXAjMCtUlVRASCb6nX+0A8aCXjmYouBEXu0c=
github.com/go-ini/ini v1.40.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E=
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
github.com/googleapis/gax-go v2.0.0+incompatible h1:j0GKcs05QVmm7yesiZq2+9cxHkNK9YM6zKx4D2qucQU=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww=
github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/minio/minio-go v6.0.9+incompatible h1:1GBagCy3VtWteFBwjjNyajSf0JJ/iT0hYVlK8xipsds=
github.com/minio/minio-go v6.0.9+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/minio/minio-go v6.0.11+incompatible h1:ue0S9ZVNhy88iS+GM4y99k3oSSeKIF+OKEe6HRMWLRw=
github.com/minio/minio-go v6.0.11+incompatible/go.mod h1:7guKYtitv8dktvNUGrhzmNlA5wrAABTQXCoesZdFQO8=
github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnGC8aR0=
github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
@@ -33,27 +33,29 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q
github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6 h1:31fhvQj+O9qDqMxUgQDOCQA5RV1iIFMzYPhBUyzg2p0=
github.com/silvasur/buzhash v0.0.0-20160816060738-9bdec3dec7c6/go.mod h1:jk5gVE20+MCoyJ2TFiiMrbWPyaH4t9T5F3HwVdthB2w=
github.com/studio-b12/gowebdav v0.0.0-20181024110551-cba565a9dcfc h1:p1iYuFAxSsQ5JDzBOpBEsqFpjgKRyGrnjQpvvq2AK5A=
github.com/studio-b12/gowebdav v0.0.0-20181024110551-cba565a9dcfc/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
github.com/studio-b12/gowebdav v0.0.0-20181230112802-6c32839dbdfc h1:iuD/gqAYTn1N3KSn8LqhdNqKVOrrCXHETAM/42M6x58=
github.com/studio-b12/gowebdav v0.0.0-20181230112802-6c32839dbdfc/go.mod h1:gCcfDlA1Y7GqOaeEKw5l9dOGx1VLdc/HuQSlQAaZ30s=
go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938=
go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16 h1:y6ce7gCWtnH+m3dCjzQ1PCuwl28DDIc3VNnvY29DlIA=
golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/exp v0.0.0-20181022080537-42ba7d4b6eb2 h1:lpkPb6P4ObnPRN3VbEzv/6CUtwaEDtx0cvCg4eWQuBk=
golang.org/x/exp v0.0.0-20181022080537-42ba7d4b6eb2/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9 h1:mKdxBk7AujPs8kU4m80U72y/zjbZ3UcXC7dClwKbUI0=
golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
golang.org/x/exp v0.0.0-20181221233300-b68661188fbf h1:rK+9ETFkUX8OofKNLhik8qEGY/3gnM5x4eVNWJf1z/8=
golang.org/x/exp v0.0.0-20181221233300-b68661188fbf/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc h1:ZMCWScCvS2fUVFw8LOpxyUUW5qiviqr4Dg5NdjLeiLU=
golang.org/x/net v0.0.0-20181102091132-c10e9556a7bc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3 h1:eH6Eip3UpmR+yM/qI9Ijluzb1bNv/cAU/n+6l8tRSis=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181102170140-232e45548389 h1:NSr16yuMknNO4kjJ2yNMJBdS55sdwZiWrXbt3fbM3pI=
golang.org/x/oauth2 v0.0.0-20181102170140-232e45548389/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc h1:SdCq5U4J+PpbSDIl9bM0V1e1Ug1jsnBkAFvTs1htn7U=
golang.org/x/sys v0.0.0-20181031143558-9b800f95dbbc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb h1:pf3XwC90UUdNPYWZdFjhGBE7DUFuK3Ct1zWmZ65QN30=
golang.org/x/sys v0.0.0-20181228144115-9a3f9b0469bb/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -63,11 +65,12 @@ google.golang.org/api v0.0.0-20181229000844-f26a60c56f14/go.mod h1:4mhQ8q/RsB7i+
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181101192439-c830210a61df h1:Ri2mROsxIxitlzRQ0pYoP8/dsqeLEolHrhh29dltSI4=
google.golang.org/genproto v0.0.0-20181101192439-c830210a61df/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f h1:eT3B0O2ghdSPzjAOznr3oOLyN1HFeYUncYl7FRwg4VI=
google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg=
google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw=
google.golang.org/grpc v1.16.0 h1:dz5IJGuC2BB7qXR5AyHNwAUBhZscK2xVez7mznh72sY=
google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio=
google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk=
google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

View File

@@ -26,9 +26,9 @@
// NewRepositoryOptions specifies options that apply to newly created repositories.
// All fields are optional, when not provided, reasonable defaults will be used.
type NewRepositoryOptions struct {
UniqueID []byte // force the use of particular unique ID for metadata manager
MetadataEncryptionAlgorithm string // identifier of encryption algorithm
KeyDerivationAlgorithm string // identifier of key derivation algorithm
UniqueID []byte // force the use of particular unique ID
FormatEncryptionAlgorithm string // identifier of encryption algorithm
KeyDerivationAlgorithm string // identifier of key derivation algorithm
BlockFormat block.FormattingOptions
DisableHMAC bool
@@ -75,7 +75,7 @@ func formatBlockFromOptions(opt *NewRepositoryOptions) *formatBlock {
KeyDerivationAlgorithm: applyDefaultString(opt.KeyDerivationAlgorithm, DefaultKeyDerivationAlgorithm),
UniqueID: applyDefaultRandomBytes(opt.UniqueID, 32),
Version: "1",
EncryptionAlgorithm: applyDefaultString(opt.MetadataEncryptionAlgorithm, DefaultEncryptionAlgorithm),
EncryptionAlgorithm: applyDefaultString(opt.FormatEncryptionAlgorithm, DefaultEncryptionAlgorithm),
}
}

View File

@@ -52,7 +52,7 @@ func (e *Environment) Setup(t *testing.T, opts ...func(*repo.NewRepositoryOption
Splitter: "FIXED",
MaxBlockSize: 400,
},
MetadataEncryptionAlgorithm: "NONE",
FormatEncryptionAlgorithm: "NONE",
}
for _, mod := range opts {

View File

@@ -122,6 +122,9 @@ func OpenWithConfig(ctx context.Context, st storage.Storage, lc *LocalConfig, pa
Manifests: manifests,
CacheDirectory: caching.CacheDirectory,
UniqueID: f.UniqueID,
formatBlock: f,
masterKey: masterKey,
}, nil
}

View File

@@ -21,6 +21,9 @@ type Repository struct {
ConfigFile string
CacheDirectory string
formatBlock *formatBlock
masterKey []byte
}
// Close closes the repository and releases all resources.

View File

@@ -156,6 +156,20 @@ func TestHMAC(t *testing.T) {
}
}
func TestUpgrade(t *testing.T) {
var env repotesting.Environment
defer env.Setup(t).Close(t)
ctx := context.Background()
if err := env.Repository.Upgrade(ctx); err != nil {
t.Errorf("upgrade error: %v", err)
}
if err := env.Repository.Upgrade(ctx); err != nil {
t.Errorf("2nd upgrade error: %v", err)
}
}
func TestReaderStoredBlockNotFound(t *testing.T) {
var env repotesting.Environment
defer env.Setup(t).Close(t)

49
upgrade.go Normal file
View File

@@ -0,0 +1,49 @@
package repo
import (
"context"
"fmt"
)
// Upgrade upgrades repository data structures to the latest version.
func (r *Repository) Upgrade(ctx context.Context) error {
f := r.formatBlock
log.Debug("decrypting format...")
repoConfig, err := f.decryptFormatBytes(r.masterKey)
if err != nil {
return fmt.Errorf("unable to decrypt repository config: %v", err)
}
var migrated bool
if repoConfig.FormattingOptions.LegacyBlockFormat != "" {
log.Infof("upgrading from legacy block format to explicit hash/encryption spec")
switch repoConfig.FormattingOptions.LegacyBlockFormat {
case "UNENCRYPTED_HMAC_SHA256":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256"
repoConfig.FormattingOptions.Encryption = "NONE"
case "UNENCRYPTED_HMAC_SHA256_128":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256-128"
repoConfig.FormattingOptions.Encryption = "NONE"
case "ENCRYPTED_HMAC_SHA256_AES256_SIV":
repoConfig.FormattingOptions.Hash = "HMAC-SHA256-128"
repoConfig.FormattingOptions.Encryption = "AES-256-CTR"
}
repoConfig.FormattingOptions.LegacyBlockFormat = ""
migrated = true
}
if !migrated {
log.Infof("nothing to do")
return nil
}
log.Debug("encrypting format...")
if err := encryptFormatBytes(f, repoConfig, r.masterKey, f.UniqueID); err != nil {
return fmt.Errorf("unable to encrypt format bytes")
}
log.Infof("writing updated format block...")
return writeFormatBlock(ctx, r.Storage, f)
}