diff --git a/storage/caching/cache_entry.go b/blob/caching/cache_entry.go similarity index 100% rename from storage/caching/cache_entry.go rename to blob/caching/cache_entry.go diff --git a/storage/caching/caching_storage.go b/blob/caching/caching_storage.go similarity index 88% rename from storage/caching/caching_storage.go rename to blob/caching/caching_storage.go index 038360eb4..d1fb60867 100644 --- a/storage/caching/caching_storage.go +++ b/blob/caching/caching_storage.go @@ -9,8 +9,8 @@ "time" "github.com/boltdb/bolt" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/filesystem" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/filesystem" ) var ( @@ -23,8 +23,8 @@ ) type cachingStorage struct { - master storage.Storage - cache storage.Storage + master blob.Storage + cache blob.Storage db *bolt.DB sizeBytes int64 @@ -102,7 +102,7 @@ func (c *cachingStorage) BlockSize(id string) (int64, error) { return entry.size, nil } - return 0, storage.ErrBlockNotFound + return 0, blob.ErrBlockNotFound } c.Lock(id) @@ -110,7 +110,7 @@ func (c *cachingStorage) BlockSize(id string) (int64, error) { l, err := c.master.BlockSize(id) if err != nil { - if err == storage.ErrBlockNotFound { + if err == blob.ErrBlockNotFound { c.setCacheEntrySize(id, sizeDoesNotExists) } return 0, err @@ -141,7 +141,7 @@ func (c *cachingStorage) GetBlock(id string) ([]byte, error) { if blockCacheEntry, ok := c.getCacheEntry(id); ok { if !blockCacheEntry.exists() { - return nil, storage.ErrBlockNotFound + return nil, blob.ErrBlockNotFound } v, err := c.cache.GetBlock(id) @@ -155,16 +155,16 @@ func (c *cachingStorage) GetBlock(id string) ([]byte, error) { if err == nil { l := int64(len(b)) - c.cache.PutBlock(id, b, storage.PutOptionsOverwrite) + c.cache.PutBlock(id, b, blob.PutOptionsOverwrite) c.setCacheEntrySize(id, l) - } else if err == storage.ErrBlockNotFound { + } else if err == blob.ErrBlockNotFound { c.setCacheEntrySize(id, sizeDoesNotExists) } return b, err } -func (c *cachingStorage) PutBlock(id string, data []byte, options storage.PutOptions) error { +func (c *cachingStorage) PutBlock(id string, data []byte, options blob.PutOptions) error { c.Lock(id) defer c.Unlock(id) @@ -175,7 +175,7 @@ func (c *cachingStorage) PutBlock(id string, data []byte, options storage.PutOpt return c.master.PutBlock(id, data, options) } -func (c *cachingStorage) ListBlocks(prefix string) chan storage.BlockMetadata { +func (c *cachingStorage) ListBlocks(prefix string) chan blob.BlockMetadata { return c.master.ListBlocks(prefix) } @@ -204,7 +204,7 @@ type Options struct { } // NewWrapper creates new caching storage wrapper. -func NewWrapper(master storage.Storage, options *Options) (storage.Storage, error) { +func NewWrapper(master blob.Storage, options *Options) (blob.Storage, error) { if options.CacheDir == "" { return nil, fmt.Errorf("Cache directory must be specified") } @@ -242,4 +242,4 @@ func NewWrapper(master storage.Storage, options *Options) (storage.Storage, erro return s, nil } -var _ storage.Storage = &cachingStorage{} +var _ blob.Storage = &cachingStorage{} diff --git a/storage/caching/caching_storage_test.go b/blob/caching/caching_storage_test.go similarity index 94% rename from storage/caching/caching_storage_test.go rename to blob/caching/caching_storage_test.go index 9ed91592a..74b135b3d 100644 --- a/storage/caching/caching_storage_test.go +++ b/blob/caching/caching_storage_test.go @@ -7,9 +7,9 @@ "os" "strings" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/internal/storagetesting" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/logging" "testing" ) @@ -127,7 +127,7 @@ func TestCache(t *testing.T) { storagetesting.AssertGetBlockNotFound(t, cache, "z") tr.assertNoActivity(t) - cache.PutBlock("z", data1, storage.PutOptionsDefault) + cache.PutBlock("z", data1, blob.PutOptionsDefault) tr.assertActivityAndClear(t, "PutBlock") storagetesting.AssertBlockExists(t, cache, "z", true) @@ -136,7 +136,7 @@ func TestCache(t *testing.T) { storagetesting.AssertGetBlock(t, cache, "z", data1) tr.assertActivityAndClear(t, "GetBlock") - cache.PutBlock("x2", data1, storage.PutOptionsDefault) + cache.PutBlock("x2", data1, blob.PutOptionsDefault) tr.assertActivityAndClear(t, "PutBlock") storagetesting.AssertListResults(t, cache, "", "x", "x2", "z") diff --git a/storage/caching/lock_map.go b/blob/caching/lock_map.go similarity index 100% rename from storage/caching/lock_map.go rename to blob/caching/lock_map.go diff --git a/storage/caching/lock_map_test.go b/blob/caching/lock_map_test.go similarity index 100% rename from storage/caching/lock_map_test.go rename to blob/caching/lock_map_test.go diff --git a/storage/config.go b/blob/config.go similarity index 98% rename from storage/config.go rename to blob/config.go index 5f7bec86c..2ae39932a 100644 --- a/storage/config.go +++ b/blob/config.go @@ -1,4 +1,4 @@ -package storage +package blob import ( "encoding/json" diff --git a/blob/doc.go b/blob/doc.go new file mode 100644 index 000000000..6716e1290 --- /dev/null +++ b/blob/doc.go @@ -0,0 +1,2 @@ +// Package blob implements simple storage of immutable, unstructured binary large objects (BLOBs). +package blob diff --git a/storage/errors.go b/blob/errors.go similarity index 95% rename from storage/errors.go rename to blob/errors.go index f95525ad1..85957dd4c 100644 --- a/storage/errors.go +++ b/blob/errors.go @@ -1,4 +1,4 @@ -package storage +package blob import "errors" diff --git a/storage/filesystem/filesystem_options.go b/blob/filesystem/filesystem_options.go similarity index 100% rename from storage/filesystem/filesystem_options.go rename to blob/filesystem/filesystem_options.go diff --git a/storage/filesystem/filesystem_storage.go b/blob/filesystem/filesystem_storage.go similarity index 90% rename from storage/filesystem/filesystem_storage.go rename to blob/filesystem/filesystem_storage.go index e2ad459ac..74610e506 100644 --- a/storage/filesystem/filesystem_storage.go +++ b/blob/filesystem/filesystem_storage.go @@ -10,7 +10,7 @@ "strconv" "strings" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) const ( @@ -36,7 +36,7 @@ func (fs *fsStorage) BlockSize(blockID string) (int64, error) { } if os.IsNotExist(err) { - return 0, storage.ErrBlockNotFound + return 0, blob.ErrBlockNotFound } return 0, err @@ -50,7 +50,7 @@ func (fs *fsStorage) GetBlock(blockID string) ([]byte, error) { } if os.IsNotExist(err) { - return nil, storage.ErrBlockNotFound + return nil, blob.ErrBlockNotFound } return nil, err @@ -68,8 +68,8 @@ func makeFileName(blockID string) string { return string(blockID) + fsStorageChunkSuffix } -func (fs *fsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { - result := make(chan (storage.BlockMetadata)) +func (fs *fsStorage) ListBlocks(prefix string) chan (blob.BlockMetadata) { + result := make(chan (blob.BlockMetadata)) prefixString := string(prefix) @@ -95,7 +95,7 @@ func (fs *fsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { } } else if fullID, ok := getstringFromFileName(currentPrefix + e.Name()); ok { if strings.HasPrefix(string(fullID), prefixString) { - result <- storage.BlockMetadata{ + result <- blob.BlockMetadata{ BlockID: fullID, Length: e.Size(), TimeStamp: e.ModTime(), @@ -115,7 +115,7 @@ func (fs *fsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { return result } -func (fs *fsStorage) PutBlock(blockID string, data []byte, options storage.PutOptions) error { +func (fs *fsStorage) PutBlock(blockID string, data []byte, options blob.PutOptions) error { shardPath, path := fs.getShardedPathAndFilePath(blockID) // Open temporary file, create dir if required. @@ -196,8 +196,8 @@ func parseShardString(shardString string) ([]int, error) { return result, nil } -func (fs *fsStorage) ConnectionInfo() storage.ConnectionInfo { - return storage.ConnectionInfo{ +func (fs *fsStorage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ Type: fsStorageType, Config: &fs.Options, } @@ -208,7 +208,7 @@ func (fs *fsStorage) Close() error { } // New creates new filesystem-backed storage in a specified directory. -func New(options *Options) (storage.Storage, error) { +func New(options *Options) (blob.Storage, error) { var err error if _, err = os.Stat(options.Path); err != nil { @@ -223,10 +223,10 @@ func New(options *Options) (storage.Storage, error) { } func init() { - storage.AddSupportedStorage( + blob.AddSupportedStorage( fsStorageType, func() interface{} { return &Options{} }, - func(o interface{}) (storage.Storage, error) { + func(o interface{}) (blob.Storage, error) { return New(o.(*Options)) }) } diff --git a/storage/filesystem/filesystem_storage_test.go b/blob/filesystem/filesystem_storage_test.go similarity index 100% rename from storage/filesystem/filesystem_storage_test.go rename to blob/filesystem/filesystem_storage_test.go diff --git a/storage/gcs/errors.go b/blob/gcs/errors.go similarity index 100% rename from storage/gcs/errors.go rename to blob/gcs/errors.go diff --git a/storage/gcs/gcs_options.go b/blob/gcs/gcs_options.go similarity index 100% rename from storage/gcs/gcs_options.go rename to blob/gcs/gcs_options.go diff --git a/storage/gcs/gcs_storage.go b/blob/gcs/gcs_storage.go similarity index 92% rename from storage/gcs/gcs_storage.go rename to blob/gcs/gcs_storage.go index fe1b97c92..6fdeddb17 100644 --- a/storage/gcs/gcs_storage.go +++ b/blob/gcs/gcs_storage.go @@ -16,7 +16,7 @@ "github.com/skratchdot/open-golang/open" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" "golang.org/x/net/context" "golang.org/x/oauth2" @@ -48,7 +48,7 @@ func() (interface{}, error) { }) if isGoogleAPIError(err, http.StatusNotFound) { - return 0, storage.ErrBlockNotFound + return 0, blob.ErrBlockNotFound } return int64(v.(*gcsclient.Object).Size), nil @@ -62,7 +62,7 @@ func() (interface{}, error) { return call.Download() }) if isGoogleAPIError(err, http.StatusNotFound) { - return nil, storage.ErrBlockNotFound + return nil, blob.ErrBlockNotFound } if err != nil { @@ -76,7 +76,7 @@ func() (interface{}, error) { return ioutil.ReadAll(dl.Body) } -func (gcs *gcsStorage) PutBlock(b string, data []byte, options storage.PutOptions) error { +func (gcs *gcsStorage) PutBlock(b string, data []byte, options blob.PutOptions) error { object := gcsclient.Object{ Name: gcs.getObjectNameString(b), } @@ -87,7 +87,7 @@ func (gcs *gcsStorage) PutBlock(b string, data []byte, options storage.PutOption // Specify exact chunk size to ensure data is uploaded in one shot or not at all. googleapi.ChunkSize(len(data)), ) - if options&storage.PutOptionsOverwrite == 0 { + if options&blob.PutOptionsOverwrite == 0 { // To avoid the race, check this server-side. call = call.IfGenerationMatch(0) } @@ -124,8 +124,8 @@ func (gcs *gcsStorage) getObjectNameString(b string) string { return gcs.Prefix + string(b) } -func (gcs *gcsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { - ch := make(chan storage.BlockMetadata, 100) +func (gcs *gcsStorage) ListBlocks(prefix string) chan (blob.BlockMetadata) { + ch := make(chan blob.BlockMetadata, 100) go func() { ps := gcs.getObjectNameString(prefix) @@ -138,7 +138,7 @@ func() (interface{}, error) { for { if err != nil { - ch <- storage.BlockMetadata{Error: err} + ch <- blob.BlockMetadata{Error: err} break } @@ -149,11 +149,11 @@ func() (interface{}, error) { for _, o := range objects.Items { t, e := time.Parse(time.RFC3339, o.TimeCreated) if e != nil { - ch <- storage.BlockMetadata{ + ch <- blob.BlockMetadata{ Error: e, } } else { - ch <- storage.BlockMetadata{ + ch <- blob.BlockMetadata{ BlockID: string(o.Name)[len(gcs.Prefix):], Length: int64(o.Size), TimeStamp: t, @@ -179,8 +179,8 @@ func() (interface{}, error) { return ch } -func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo { - return storage.ConnectionInfo{ +func (gcs *gcsStorage) ConnectionInfo() blob.ConnectionInfo { + return blob.ConnectionInfo{ Type: gcsStorageType, Config: &gcs.Options, } @@ -222,7 +222,7 @@ func saveToken(file string, token *oauth2.Token) { // // By default the connection reuses credentials managed by (https://cloud.google.com/sdk/), // but this can be disabled by setting IgnoreDefaultCredentials to true. -func New(options *Options) (storage.Storage, error) { +func New(options *Options) (blob.Storage, error) { ctx := context.TODO() //ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{}) @@ -408,14 +408,14 @@ func tokenFromWebManual(ctx context.Context, config *oauth2.Config) (*oauth2.Tok } func init() { - storage.AddSupportedStorage( + blob.AddSupportedStorage( gcsStorageType, func() interface{} { return &Options{} }, - func(o interface{}) (storage.Storage, error) { + func(o interface{}) (blob.Storage, error) { return New(o.(*Options)) }) } -var _ storage.ConnectionInfoProvider = &gcsStorage{} +var _ blob.ConnectionInfoProvider = &gcsStorage{} diff --git a/storage/logging/logging_storage.go b/blob/logging/logging_storage.go similarity index 90% rename from storage/logging/logging_storage.go rename to blob/logging/logging_storage.go index 94d08d4d7..accf986ad 100644 --- a/storage/logging/logging_storage.go +++ b/blob/logging/logging_storage.go @@ -5,11 +5,11 @@ "log" "time" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) type loggingStorage struct { - base storage.Storage + base blob.Storage printf func(string, ...interface{}) prefix string } @@ -34,7 +34,7 @@ func (s *loggingStorage) GetBlock(id string) ([]byte, error) { return result, err } -func (s *loggingStorage) PutBlock(id string, data []byte, options storage.PutOptions) error { +func (s *loggingStorage) PutBlock(id string, data []byte, options blob.PutOptions) error { t0 := time.Now() err := s.base.PutBlock(id, data, options) dt := time.Since(t0) @@ -50,7 +50,7 @@ func (s *loggingStorage) DeleteBlock(id string) error { return err } -func (s *loggingStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { +func (s *loggingStorage) ListBlocks(prefix string) chan (blob.BlockMetadata) { t0 := time.Now() ch := s.base.ListBlocks(prefix) dt := time.Since(t0) @@ -70,7 +70,7 @@ func (s *loggingStorage) Close() error { type Option func(s *loggingStorage) // NewWrapper returns a Storage wrapper that logs all storage commands. -func NewWrapper(wrapped storage.Storage, options ...Option) storage.Storage { +func NewWrapper(wrapped blob.Storage, options ...Option) blob.Storage { s := &loggingStorage{base: wrapped, printf: log.Printf} for _, o := range options { o(s) diff --git a/storage/logging/logging_storage_test.go b/blob/logging/logging_storage_test.go similarity index 100% rename from storage/logging/logging_storage_test.go rename to blob/logging/logging_storage_test.go diff --git a/storage/registry.go b/blob/registry.go similarity index 98% rename from storage/registry.go rename to blob/registry.go index a7e3820a9..6817e761c 100644 --- a/storage/registry.go +++ b/blob/registry.go @@ -1,4 +1,4 @@ -package storage +package blob import "fmt" diff --git a/storage/storage.go b/blob/storage.go similarity index 98% rename from storage/storage.go rename to blob/storage.go index cb1b64c3b..fd4ba1a5d 100644 --- a/storage/storage.go +++ b/blob/storage.go @@ -1,4 +1,4 @@ -package storage +package blob import ( "io" diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index ff0eab569..de3f4be35 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -7,8 +7,8 @@ "gopkg.in/alecthomas/kingpin.v2" + "github.com/kopia/kopia/blob" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" "github.com/kopia/kopia/vault" ) @@ -59,7 +59,7 @@ func repositoryFormat() (*repo.Format, error) { return f, nil } -func openStorageAndEnsureEmpty(url string) (storage.Storage, error) { +func openStorageAndEnsureEmpty(url string) (blob.Storage, error) { s, err := newStorageFromURL(url) if err != nil { return nil, err diff --git a/cmd/kopia/config.go b/cmd/kopia/config.go index 66bdfdba2..918432694 100644 --- a/cmd/kopia/config.go +++ b/cmd/kopia/config.go @@ -13,12 +13,12 @@ "github.com/bgentry/speakeasy" "github.com/kopia/kopia" + "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/fs" "github.com/kopia/kopia/fs/localfs" "github.com/kopia/kopia/fs/loggingfs" "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage/logging" "github.com/kopia/kopia/vault" ) diff --git a/cmd/kopia/urls.go b/cmd/kopia/urls.go index 0eb498821..92f584a6c 100644 --- a/cmd/kopia/urls.go +++ b/cmd/kopia/urls.go @@ -7,12 +7,12 @@ "strconv" "strings" - "github.com/kopia/kopia/storage" - fsstorage "github.com/kopia/kopia/storage/filesystem" - gcsstorage "github.com/kopia/kopia/storage/gcs" + "github.com/kopia/kopia/blob" + fsstorage "github.com/kopia/kopia/blob/filesystem" + gcsstorage "github.com/kopia/kopia/blob/gcs" ) -func newStorageFromURL(urlString string) (storage.Storage, error) { +func newStorageFromURL(urlString string) (blob.Storage, error) { if strings.HasPrefix(urlString, "/") { urlString = "file://" + urlString } diff --git a/connection.go b/connection.go index 740c7a92d..7c37ebd77 100644 --- a/connection.go +++ b/connection.go @@ -4,11 +4,11 @@ "errors" "fmt" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/caching" + "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/internal/config" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/caching" - "github.com/kopia/kopia/storage/logging" "github.com/kopia/kopia/vault" ) @@ -62,7 +62,7 @@ func Open(configFile string, options *ConnectionOptions) (*Connection, error) { return nil, fmt.Errorf("invalid vault credentials: %v", err) } - rawVaultStorage, err := storage.NewStorage(lc.VaultConnection.ConnectionInfo) + rawVaultStorage, err := blob.NewStorage(lc.VaultConnection.ConnectionInfo) if err != nil { return nil, fmt.Errorf("cannot open vault storage: %v", err) } @@ -80,12 +80,12 @@ func Open(configFile string, options *ConnectionOptions) (*Connection, error) { return nil, fmt.Errorf("unable to open vault: %v", err) } - var repositoryStorage storage.Storage + var repositoryStorage blob.Storage if lc.RepoConnection == nil { repositoryStorage = rawVaultStorage } else { - repositoryStorage, err = storage.NewStorage(*lc.RepoConnection) + repositoryStorage, err = blob.NewStorage(*lc.RepoConnection) if err != nil { vaultStorage.Close() return nil, err diff --git a/fs/repofs/upload_test.go b/fs/repofs/upload_test.go index af309cad4..8a2065271 100644 --- a/fs/repofs/upload_test.go +++ b/fs/repofs/upload_test.go @@ -7,10 +7,10 @@ "os" "reflect" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/filesystem" "github.com/kopia/kopia/internal/mockfs" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/filesystem" "testing" ) @@ -19,7 +19,7 @@ type uploadTestHarness struct { sourceDir *mockfs.Directory repoDir string repo *repo.Repository - storage storage.Storage + storage blob.Storage uploader *Uploader } diff --git a/internal/config/local_config.go b/internal/config/local_config.go index b543802c8..2e188812d 100644 --- a/internal/config/local_config.go +++ b/internal/config/local_config.go @@ -6,17 +6,17 @@ "io" "os" - "github.com/kopia/kopia/storage/caching" + "github.com/kopia/kopia/blob/caching" "github.com/kopia/kopia/vault" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) // LocalConfig is a configuration of Kopia. type LocalConfig struct { - VaultConnection *vault.Config `json:"vault,omitempty"` - RepoConnection *storage.ConnectionInfo `json:"repository,omitempty"` // can be nil indicating the same connection as for the vault - Caching *caching.Options `json:"caching,omitempty"` + VaultConnection *vault.Config `json:"vault,omitempty"` + RepoConnection *blob.ConnectionInfo `json:"repository,omitempty"` // can be nil indicating the same connection as for the vault + Caching *caching.Options `json:"caching,omitempty"` } // Load reads local configuration from the specified reader. diff --git a/internal/storagetesting/asserts.go b/internal/storagetesting/asserts.go index 8d286c9c5..3edb01c0a 100644 --- a/internal/storagetesting/asserts.go +++ b/internal/storagetesting/asserts.go @@ -8,11 +8,11 @@ "runtime" "testing" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) // AssertGetBlock asserts that the specified storage block has correct content. -func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []byte) { +func AssertGetBlock(t *testing.T, s blob.Storage, block string, expected []byte) { b, err := s.GetBlock(block) if err != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned error %v, expected data: %v", block, err, expected) @@ -25,20 +25,20 @@ func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []by } // AssertGetBlockNotFound asserts that GetBlock() for specified storage block returns ErrBlockNotFound. -func AssertGetBlockNotFound(t *testing.T, s storage.Storage, block string) { +func AssertGetBlockNotFound(t *testing.T, s blob.Storage, block string) { b, err := s.GetBlock(block) - if err != storage.ErrBlockNotFound || b != nil { + if err != blob.ErrBlockNotFound || b != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned %v, %v but expected ErrBlockNotFound", block, b, err) } } // AssertBlockExists asserts that BlockExists() the specified storage block returns the correct value. -func AssertBlockExists(t *testing.T, s storage.Storage, block string, expected bool) { +func AssertBlockExists(t *testing.T, s blob.Storage, block string, expected bool) { _, err := s.BlockSize(block) var exists bool if err == nil { exists = true - } else if err == storage.ErrBlockNotFound { + } else if err == blob.ErrBlockNotFound { exists = false } else { t.Errorf(errorPrefix()+"BlockSize(%v) returned error: %v", block, err) @@ -51,7 +51,7 @@ func AssertBlockExists(t *testing.T, s storage.Storage, block string, expected b } // AssertListResults asserts that the list results with given prefix return the specified list of names in order. -func AssertListResults(t *testing.T, s storage.Storage, prefix string, expected ...string) { +func AssertListResults(t *testing.T, s blob.Storage, prefix string, expected ...string) { var names []string for e := range s.ListBlocks(prefix) { diff --git a/internal/storagetesting/map.go b/internal/storagetesting/map.go index 72ef77d87..64328bf0b 100644 --- a/internal/storagetesting/map.go +++ b/internal/storagetesting/map.go @@ -6,7 +6,7 @@ "sync" "time" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) type mapStorage struct { @@ -19,7 +19,7 @@ func (s *mapStorage) BlockSize(id string) (int64, error) { defer s.mutex.RUnlock() d, ok := s.data[string(id)] if !ok { - return 0, storage.ErrBlockNotFound + return 0, blob.ErrBlockNotFound } return int64(len(d)), nil @@ -34,10 +34,10 @@ func (s *mapStorage) GetBlock(id string) ([]byte, error) { return data, nil } - return nil, storage.ErrBlockNotFound + return nil, blob.ErrBlockNotFound } -func (s *mapStorage) PutBlock(id string, data []byte, options storage.PutOptions) error { +func (s *mapStorage) PutBlock(id string, data []byte, options blob.PutOptions) error { s.mutex.Lock() defer s.mutex.Unlock() @@ -57,8 +57,8 @@ func (s *mapStorage) DeleteBlock(id string) error { return nil } -func (s *mapStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { - ch := make(chan (storage.BlockMetadata)) +func (s *mapStorage) ListBlocks(prefix string) chan (blob.BlockMetadata) { + ch := make(chan (blob.BlockMetadata)) fixedTime := time.Now() go func() { s.mutex.RLock() @@ -75,7 +75,7 @@ func (s *mapStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { for _, k := range keys { v := s.data[k] - ch <- storage.BlockMetadata{ + ch <- blob.BlockMetadata{ BlockID: string(k), Length: int64(len(v)), TimeStamp: fixedTime, @@ -92,6 +92,6 @@ func (s *mapStorage) Close() error { // NewMapStorage returns an implementation of Storage backed by the contents of given map. // Used primarily for testing. -func NewMapStorage(data map[string][]byte) storage.Storage { +func NewMapStorage(data map[string][]byte) blob.Storage { return &mapStorage{data: data} } diff --git a/internal/storagetesting/verify.go b/internal/storagetesting/verify.go index 2a8c24b9a..7e0c1a58b 100644 --- a/internal/storagetesting/verify.go +++ b/internal/storagetesting/verify.go @@ -4,11 +4,11 @@ "bytes" "testing" - "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/blob" ) // VerifyStorage verifies the behavior of the specified storage. -func VerifyStorage(t *testing.T, r storage.Storage) { +func VerifyStorage(t *testing.T, r blob.Storage) { blocks := []struct { blk string contents []byte @@ -21,7 +21,7 @@ func VerifyStorage(t *testing.T, r storage.Storage) { // First verify that blocks don't exist. for _, b := range blocks { - if _, err := r.BlockSize(b.blk); err != storage.ErrBlockNotFound { + if _, err := r.BlockSize(b.blk); err != blob.ErrBlockNotFound { t.Errorf("block exists or error: %v %v", b.blk, err) } @@ -31,7 +31,7 @@ func VerifyStorage(t *testing.T, r storage.Storage) { // Now add blocks. for _, b := range blocks { - r.PutBlock(b.blk, b.contents, storage.PutOptionsDefault) + r.PutBlock(b.blk, b.contents, blob.PutOptionsDefault) AssertBlockExists(t, r, b.blk, true) AssertGetBlock(t, r, b.blk, b.contents) diff --git a/repo/repository.go b/repo/repository.go index e1df4dbe2..007d5efac 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -10,9 +10,9 @@ "sync" "sync/atomic" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/internal/jsonstream" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/logging" ) // ObjectReader allows reading, seeking, getting the length of and closing of a repository object. @@ -36,8 +36,8 @@ func (s semaphore) Unlock() { // Repository implements a content-addressable storage on top of blob storage. type Repository struct { - Stats Stats // vital statistics - storage storage.Storage // underlying blob storage + Stats Stats // vital statistics + storage blob.Storage // underlying blob storage verbose bool bufferManager *bufferManager @@ -173,7 +173,7 @@ func EnableLogging(options ...logging.Option) RepositoryOption { } // New creates a Repository with the specified storage, format and options. -func New(s storage.Storage, f *Format, options ...RepositoryOption) (*Repository, error) { +func New(s blob.Storage, f *Format, options ...RepositoryOption) (*Repository, error) { if err := f.Validate(); err != nil { return nil, err } @@ -261,7 +261,7 @@ func (r *Repository) hashEncryptAndWrite(objectID ObjectID, buffer *bytes.Buffer return objectID, nil } - if err != nil && err != storage.ErrBlockNotFound { + if err != nil && err != blob.ErrBlockNotFound { // Don't know whether block exists in storage. return NullObjectID, err } @@ -278,7 +278,7 @@ func (r *Repository) hashEncryptAndWrite(objectID ObjectID, buffer *bytes.Buffer atomic.AddInt32(&r.Stats.WrittenBlocks, int32(1)) atomic.AddInt64(&r.Stats.WrittenBytes, int64(len(data))) - if err := r.storage.PutBlock(objectID.StorageBlock, data, storage.PutOptionsDefault); err != nil { + if err := r.storage.PutBlock(objectID.StorageBlock, data, blob.PutOptionsDefault); err != nil { r.writeBackErrors.add(err) } diff --git a/repo/repository_test.go b/repo/repository_test.go index 0afbf210e..7597375c8 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -12,9 +12,9 @@ "reflect" "testing" + "github.com/kopia/kopia/blob" "github.com/kopia/kopia/internal/jsonstream" "github.com/kopia/kopia/internal/storagetesting" - "github.com/kopia/kopia/storage" ) func init() { @@ -293,7 +293,7 @@ func TestReaderStoredBlockNotFound(t *testing.T) { t.Errorf("cannot parse object ID: %v", err) } reader, err := repo.Open(objectID) - if err != storage.ErrBlockNotFound || reader != nil { + if err != blob.ErrBlockNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } } diff --git a/storage/doc.go b/storage/doc.go deleted file mode 100644 index a2c193b6f..000000000 --- a/storage/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package storage implements simple storage of immutable, unstructured binary large objects (BLOBs). -package storage diff --git a/vault/vault.go b/vault/vault.go index 1ebc50fa5..bb9f2740e 100644 --- a/vault/vault.go +++ b/vault/vault.go @@ -14,8 +14,8 @@ "strings" "sync" + "github.com/kopia/kopia/blob" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" "golang.org/x/crypto/hkdf" ) @@ -37,7 +37,7 @@ // Vault is a secure storage for secrets such as repository object identifiers. type Vault struct { - storage storage.Storage + storage blob.Storage format Format RepoConfig RepositoryConfig @@ -84,13 +84,13 @@ func (v *Vault) writeEncryptedBlock(itemID string, content []byte) error { content = cipherText } - return v.storage.PutBlock(v.itemPrefix+itemID, content, storage.PutOptionsOverwrite) + return v.storage.PutBlock(v.itemPrefix+itemID, content, blob.PutOptionsOverwrite) } func (v *Vault) readEncryptedBlock(itemID string) ([]byte, error) { content, err := v.storage.GetBlock(v.itemPrefix + itemID) if err != nil { - if err == storage.ErrBlockNotFound { + if err == blob.ErrBlockNotFound { return nil, ErrItemNotFound } return nil, fmt.Errorf("unexpected error reading %v: %v", itemID, err) @@ -227,14 +227,14 @@ func (v *Vault) Close() error { // Config represents JSON-compatible configuration of the vault connection, including vault key. type Config struct { - ConnectionInfo storage.ConnectionInfo `json:"connection"` - Key []byte `json:"key,omitempty"` + ConnectionInfo blob.ConnectionInfo `json:"connection"` + Key []byte `json:"key,omitempty"` } // Config returns a configuration of vault storage its credentials that's suitable // for storing in configuration file. func (v *Vault) Config() (*Config, error) { - cip, ok := v.storage.(storage.ConnectionInfoProvider) + cip, ok := v.storage.(blob.ConnectionInfoProvider) if !ok { return nil, errors.New("repository does not support persisting configuration") } @@ -258,10 +258,10 @@ func (v *Vault) Remove(itemID string) error { // Create initializes a Vault attached to the specified repository. func Create( - vaultStorage storage.Storage, + vaultStorage blob.Storage, vaultFormat *Format, vaultCreds Credentials, - repoStorage storage.Storage, + repoStorage blob.Storage, repoFormat *repo.Format, ) (*Vault, error) { v := Vault{ @@ -274,7 +274,7 @@ func Create( v.itemPrefix = colocatedVaultItemPrefix } - cip, ok := repoStorage.(storage.ConnectionInfoProvider) + cip, ok := repoStorage.(blob.ConnectionInfoProvider) if !ok { return nil, errors.New("repository does not support persisting configuration") } @@ -291,7 +291,7 @@ func Create( return nil, err } - if err := vaultStorage.PutBlock(v.itemPrefix+formatBlockID, formatBytes, storage.PutOptionsOverwrite); err != nil { + if err := vaultStorage.PutBlock(v.itemPrefix+formatBlockID, formatBytes, blob.PutOptionsOverwrite); err != nil { return nil, err } @@ -315,7 +315,7 @@ func Create( // CreateColocated initializes a Vault attached to a Repository sharing the same storage. func CreateColocated( - sharedStorage storage.Storage, + sharedStorage blob.Storage, vaultFormat *Format, vaultCreds Credentials, repoFormat *repo.Format, @@ -325,12 +325,12 @@ func CreateColocated( // RepositoryConfig stores the configuration of the repository associated with the vault. type RepositoryConfig struct { - Connection *storage.ConnectionInfo `json:"connection"` - Format *repo.Format `json:"format"` + Connection *blob.ConnectionInfo `json:"connection"` + Format *repo.Format `json:"format"` } // Open opens a vault. -func Open(vaultStorage storage.Storage, vaultCreds Credentials) (*Vault, error) { +func Open(vaultStorage blob.Storage, vaultCreds Credentials) (*Vault, error) { v := Vault{ storage: vaultStorage, } diff --git a/vault/vault_test.go b/vault/vault_test.go index 63974ae50..98e5db47c 100644 --- a/vault/vault_test.go +++ b/vault/vault_test.go @@ -7,9 +7,9 @@ "reflect" "strings" + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/blob/filesystem" "github.com/kopia/kopia/repo" - "github.com/kopia/kopia/storage" - "github.com/kopia/kopia/storage/filesystem" "testing" ) @@ -265,7 +265,7 @@ func assertVaultItems(t *testing.T, v *Vault, prefix string, expected []string) } } -func mustCreateFileStorage(t *testing.T, path string) storage.Storage { +func mustCreateFileStorage(t *testing.T, path string) blob.Storage { os.MkdirAll(path, 0700) s, err := filesystem.New(&filesystem.Options{ Path: path,