diff --git a/block/block_manager.go b/block/block_manager.go index a06794fa6..2b3bbda3d 100644 --- a/block/block_manager.go +++ b/block/block_manager.go @@ -12,7 +12,7 @@ "sync/atomic" "time" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) const parallelFetches = 5 @@ -46,7 +46,7 @@ type Info struct { // Manager manages storage blocks at a low level with encryption, deduplication and packaging. type Manager struct { - storage blob.Storage + storage storage.Storage stats Stats mu sync.Mutex @@ -75,7 +75,7 @@ func (bm *Manager) BlockSize(blockID string) (int64, error) { ndx := pi[blockID] if ndx == nil { - return 0, blob.ErrBlockNotFound + return 0, storage.ErrBlockNotFound } return int64(ndx.Items[blockID].size), nil @@ -644,7 +644,7 @@ func (bm *Manager) writeUnpackedBlock(data []byte, prefix string, force bool) (s return blockID, nil } - if err != nil && err != blob.ErrBlockNotFound { + if err != nil && err != storage.ErrBlockNotFound { // Don't know whether block exists in storage. return "", err } @@ -685,7 +685,7 @@ func (bm *Manager) getPendingBlockLocked(blockID string) ([]byte, error) { } } } - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound } // GetBlock gets the contents of a given block. If the block is not found returns blob.ErrBlockNotFound. @@ -721,7 +721,7 @@ func (bm *Manager) BlockInfo(blockID string) (Info, error) { func (bm *Manager) blockInfoLocked(blockID string) (Info, error) { ndx := bm.blockToIndex[blockID] if ndx == nil { - return Info{}, blob.ErrBlockNotFound + return Info{}, storage.ErrBlockNotFound } return Info{ @@ -737,7 +737,7 @@ func (bm *Manager) blockInfoLocked(blockID string) (Info, error) { func (bm *Manager) getBlockInternal(blockID string) ([]byte, error) { s, err := bm.blockInfoLocked(blockID) if err != nil { - if err != blob.ErrBlockNotFound { + if err != storage.ErrBlockNotFound { return nil, err } } @@ -788,9 +788,9 @@ func (bm *Manager) verifyChecksum(data []byte, blockID string) error { } // NewManager creates new block manager with given packing options and a formatter. -func NewManager(storage blob.Storage, maxPackedContentLength, maxPackSize int, formatter Formatter) *Manager { +func NewManager(st storage.Storage, maxPackedContentLength, maxPackSize int, formatter Formatter) *Manager { return &Manager{ - storage: storage, + storage: st, openPackGroups: make(map[string]*packInfo), timeNow: time.Now, flushPackIndexesAfter: time.Now().Add(flushPackIndexTimeout), diff --git a/block/block_manager_test.go b/block/block_manager_test.go index 828fed011..3e785f212 100644 --- a/block/block_manager_test.go +++ b/block/block_manager_test.go @@ -14,7 +14,7 @@ "testing" "time" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" "github.com/kopia/kopia/internal/storagetesting" ) @@ -146,12 +146,12 @@ func TestBlockManagerEmpty(t *testing.T) { noSuchBlockID := md5hash([]byte("foo")) b, err := bm.GetBlock(noSuchBlockID) - if err != blob.ErrBlockNotFound { + if err != storage.ErrBlockNotFound { t.Errorf("unexpected error when getting non-existent block: %v, %v", b, err) } bs, err := bm.BlockSize(noSuchBlockID) - if err != blob.ErrBlockNotFound { + if err != storage.ErrBlockNotFound { t.Errorf("unexpected error when getting non-existent block size: %v, %v", bs, err) } @@ -432,8 +432,8 @@ func verifyBlockNotFound(t *testing.T, bm *Manager, blockID string) { t.Helper() b, err := bm.GetBlock(blockID) - if err != blob.ErrBlockNotFound { - t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, blob.ErrBlockNotFound) + if err != storage.ErrBlockNotFound { + t.Errorf("unexpected response from GetBlock(%q), got %v,%v, expected %v", blockID, b, err, storage.ErrBlockNotFound) } } diff --git a/cli/command_repository_create.go b/cli/command_repository_create.go index 5fbd024e3..f1efcb579 100644 --- a/cli/command_repository_create.go +++ b/cli/command_repository_create.go @@ -6,10 +6,10 @@ kingpin "gopkg.in/alecthomas/kingpin.v2" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/block" "github.com/kopia/kopia/internal/units" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage" ) var ( @@ -49,7 +49,7 @@ func newRepositoryOptionsFromFlags() *repo.NewRepositoryOptions { } } -func openStorageAndEnsureEmpty(url string) (blob.Storage, error) { +func openStorageAndEnsureEmpty(url string) (storage.Storage, error) { s, err := newStorageFromURL(getContext(), url) if err != nil { return nil, err diff --git a/cli/command_repository_status.go b/cli/command_repository_status.go index b70af0fd0..c02727f80 100644 --- a/cli/command_repository_status.go +++ b/cli/command_repository_status.go @@ -4,8 +4,8 @@ "encoding/json" "fmt" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/internal/units" + "github.com/kopia/kopia/storage" kingpin "gopkg.in/alecthomas/kingpin.v2" ) @@ -23,7 +23,7 @@ func runStatusCommand(context *kingpin.ParseContext) error { fmt.Printf("Cache directory: %v\n", rep.CacheDirectory) fmt.Println() - if cip, ok := rep.Storage.(blob.ConnectionInfoProvider); ok { + if cip, ok := rep.Storage.(storage.ConnectionInfoProvider); ok { ci := cip.ConnectionInfo() fmt.Printf("Storage type: %v\n", ci.Type) if cjson, err := json.MarshalIndent(ci.Config, " ", " "); err == nil { diff --git a/cli/urls.go b/cli/urls.go index 27add0919..85f7330e6 100644 --- a/cli/urls.go +++ b/cli/urls.go @@ -8,14 +8,14 @@ "strconv" "strings" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" - fsstorage "github.com/kopia/kopia/blob/filesystem" - gcsstorage "github.com/kopia/kopia/blob/gcs" - "github.com/kopia/kopia/blob/webdav" + fsstorage "github.com/kopia/kopia/storage/filesystem" + gcsstorage "github.com/kopia/kopia/storage/gcs" + "github.com/kopia/kopia/storage/webdav" ) -func newStorageFromURL(ctx context.Context, urlString string) (blob.Storage, error) { +func newStorageFromURL(ctx context.Context, urlString string) (storage.Storage, error) { if strings.HasPrefix(urlString, "/") { urlString = "file://" + urlString } diff --git a/internal/config/local_config.go b/internal/config/local_config.go index 908784320..d4d206836 100644 --- a/internal/config/local_config.go +++ b/internal/config/local_config.go @@ -5,7 +5,7 @@ "io" "os" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // LocalConfig is a configuration of Kopia. @@ -30,8 +30,8 @@ type RepositoryObjectFormat struct { // RepositoryConnectionInfo represents JSON-serializable configuration of the repository connection, including master key. type RepositoryConnectionInfo struct { - ConnectionInfo blob.ConnectionInfo `json:"storage"` - Key []byte `json:"key,omitempty"` + ConnectionInfo storage.ConnectionInfo `json:"storage"` + Key []byte `json:"key,omitempty"` } // EncryptedRepositoryConfig contains the configuration of repository that's persisted in encrypted format. diff --git a/internal/storagetesting/asserts.go b/internal/storagetesting/asserts.go index 8cff84a5c..c3e0ff5db 100644 --- a/internal/storagetesting/asserts.go +++ b/internal/storagetesting/asserts.go @@ -8,11 +8,11 @@ "runtime" "testing" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // AssertGetBlock asserts that the specified storage block has correct content. -func AssertGetBlock(t *testing.T, s blob.Storage, block string, expected []byte) { +func AssertGetBlock(t *testing.T, s storage.Storage, block string, expected []byte) { b, err := s.GetBlock(block, 0, -1) if err != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned error %v, expected data: %v", block, err, expected) @@ -25,20 +25,20 @@ func AssertGetBlock(t *testing.T, s blob.Storage, block string, expected []byte) } // AssertGetBlockNotFound asserts that GetBlock() for specified storage block returns ErrBlockNotFound. -func AssertGetBlockNotFound(t *testing.T, s blob.Storage, block string) { +func AssertGetBlockNotFound(t *testing.T, s storage.Storage, block string) { b, err := s.GetBlock(block, 0, -1) - if err != blob.ErrBlockNotFound || b != nil { + if err != storage.ErrBlockNotFound || b != nil { t.Errorf(errorPrefix()+"GetBlock(%v) returned %v, %v but expected ErrBlockNotFound", block, b, err) } } // AssertBlockExists asserts that BlockExists() the specified storage block returns the correct value. -func AssertBlockExists(t *testing.T, s blob.Storage, block string, expected bool) { +func AssertBlockExists(t *testing.T, s storage.Storage, block string, expected bool) { _, err := s.BlockSize(block) var exists bool if err == nil { exists = true - } else if err == blob.ErrBlockNotFound { + } else if err == storage.ErrBlockNotFound { exists = false } else { t.Errorf(errorPrefix()+"BlockSize(%v) returned error: %v", block, err) @@ -51,7 +51,7 @@ func AssertBlockExists(t *testing.T, s blob.Storage, block string, expected bool } // AssertListResults asserts that the list results with given prefix return the specified list of names in order. -func AssertListResults(t *testing.T, s blob.Storage, prefix string, expected ...string) { +func AssertListResults(t *testing.T, s storage.Storage, prefix string, expected ...string) { var names []string blocks, cancel := s.ListBlocks(prefix) diff --git a/internal/storagetesting/map.go b/internal/storagetesting/map.go index b3cd682ed..6a7df06a3 100644 --- a/internal/storagetesting/map.go +++ b/internal/storagetesting/map.go @@ -6,7 +6,7 @@ "sync" "time" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) type mapStorage struct { @@ -19,7 +19,7 @@ func (s *mapStorage) BlockSize(id string) (int64, error) { defer s.mutex.RUnlock() d, ok := s.data[string(id)] if !ok { - return 0, blob.ErrBlockNotFound + return 0, storage.ErrBlockNotFound } return int64(len(d)), nil @@ -42,7 +42,7 @@ func (s *mapStorage) GetBlock(id string, offset, length int64) ([]byte, error) { return data[0:length], nil } - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound } func (s *mapStorage) PutBlock(id string, data []byte) error { @@ -65,8 +65,8 @@ func (s *mapStorage) DeleteBlock(id string) error { return nil } -func (s *mapStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { - ch := make(chan blob.BlockMetadata) +func (s *mapStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { + ch := make(chan storage.BlockMetadata) cancelled := make(chan bool) fixedTime := time.Now() go func() { @@ -88,7 +88,7 @@ func (s *mapStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.Ca select { case <-cancelled: return - case ch <- blob.BlockMetadata{ + case ch <- storage.BlockMetadata{ BlockID: string(k), Length: int64(len(v)), TimeStamp: fixedTime, @@ -107,6 +107,6 @@ func (s *mapStorage) Close() error { // NewMapStorage returns an implementation of Storage backed by the contents of given map. // Used primarily for testing. -func NewMapStorage(data map[string][]byte) blob.Storage { +func NewMapStorage(data map[string][]byte) storage.Storage { return &mapStorage{data: data} } diff --git a/internal/storagetesting/verify.go b/internal/storagetesting/verify.go index f168762ed..a16a5dc07 100644 --- a/internal/storagetesting/verify.go +++ b/internal/storagetesting/verify.go @@ -4,11 +4,11 @@ "bytes" "testing" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // VerifyStorage verifies the behavior of the specified storage. -func VerifyStorage(t *testing.T, r blob.Storage) { +func VerifyStorage(t *testing.T, r storage.Storage) { blocks := []struct { blk string contents []byte @@ -21,7 +21,7 @@ func VerifyStorage(t *testing.T, r blob.Storage) { // First verify that blocks don't exist. for _, b := range blocks { - if _, err := r.BlockSize(b.blk); err != blob.ErrBlockNotFound { + if _, err := r.BlockSize(b.blk); err != storage.ErrBlockNotFound { t.Errorf("block exists or error: %v %v", b.blk, err) } diff --git a/repo/connection.go b/repo/connection.go index e996f488e..aeca51b5c 100644 --- a/repo/connection.go +++ b/repo/connection.go @@ -10,14 +10,14 @@ "path/filepath" "github.com/kopia/kopia/auth" - "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/block" "github.com/kopia/kopia/internal/config" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/logging" // Register well-known blob storage providers - _ "github.com/kopia/kopia/blob/filesystem" - _ "github.com/kopia/kopia/blob/gcs" + _ "github.com/kopia/kopia/storage/filesystem" + _ "github.com/kopia/kopia/storage/gcs" ) // Options provides configuration parameters for connection to a repository. @@ -58,7 +58,7 @@ func Open(ctx context.Context, configFile string, options *Options) (*Repository return nil, fmt.Errorf("invalid credentials: %v", err) } - st, err := blob.NewStorage(ctx, lc.Connection.ConnectionInfo) + st, err := storage.NewStorage(ctx, lc.Connection.ConnectionInfo) if err != nil { return nil, fmt.Errorf("cannot open storage: %v", err) } @@ -82,7 +82,7 @@ type ConnectOptions struct { } // Connect connects to the repository in the specified storage and persists the configuration and credentials in the file provided. -func Connect(ctx context.Context, configFile string, st blob.Storage, creds auth.Credentials, opt ConnectOptions) error { +func Connect(ctx context.Context, configFile string, st storage.Storage, creds auth.Credentials, opt ConnectOptions) error { r, err := connect(ctx, st, creds, nil) if err != nil { return err @@ -112,7 +112,7 @@ func Connect(ctx context.Context, configFile string, st blob.Storage, creds auth return ioutil.WriteFile(configFile, d, 0600) } -func connect(ctx context.Context, st blob.Storage, creds auth.Credentials, options *Options) (*Repository, error) { +func connect(ctx context.Context, st storage.Storage, creds auth.Credentials, options *Options) (*Repository, error) { if options == nil { options = &Options{} } diff --git a/repo/initialize.go b/repo/initialize.go index 39b063eff..a0e363190 100644 --- a/repo/initialize.go +++ b/repo/initialize.go @@ -7,9 +7,9 @@ "io" "github.com/kopia/kopia/auth" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/block" "github.com/kopia/kopia/internal/config" + "github.com/kopia/kopia/storage" ) // NewRepositoryOptions specifies options that apply to newly created repositories. @@ -34,7 +34,7 @@ type NewRepositoryOptions struct { } // Initialize creates initial repository data structures in the specified storage with given credentials. -func Initialize(st blob.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error { +func Initialize(st storage.Storage, opt *NewRepositoryOptions, creds auth.Credentials) error { if opt == nil { opt = &NewRepositoryOptions{} } diff --git a/repo/metadata_cache.go b/repo/metadata_cache.go index a7c8de441..27bf0dd4d 100644 --- a/repo/metadata_cache.go +++ b/repo/metadata_cache.go @@ -6,7 +6,7 @@ "strings" "sync" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) const ( @@ -15,7 +15,7 @@ ) type metadataCache struct { - st blob.Storage + st storage.Storage mu sync.Mutex sortedNames []string @@ -43,7 +43,7 @@ func (mc *metadataCache) GetBlock(name string) ([]byte, error) { cid := mc.nameToCacheID[name] if cid == "" { mc.mu.Unlock() - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound } // see if the data is cached @@ -143,7 +143,7 @@ func cloneBytes(d []byte) []byte { return append([]byte(nil), d...) } -func newMetadataCache(st blob.Storage) (*metadataCache, error) { +func newMetadataCache(st storage.Storage) (*metadataCache, error) { c := &metadataCache{ st: st, nameToCacheID: make(map[string]string), diff --git a/repo/metadata_manager.go b/repo/metadata_manager.go index e34b421ea..02a9a179f 100644 --- a/repo/metadata_manager.go +++ b/repo/metadata_manager.go @@ -12,8 +12,8 @@ "sync" "github.com/kopia/kopia/auth" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/internal/config" + "github.com/kopia/kopia/storage" "golang.org/x/crypto/hkdf" ) @@ -54,7 +54,7 @@ func init() { // MetadataManager manages JSON metadata, such as snapshot manifests, policies, object format etc. // in a repository. type MetadataManager struct { - storage blob.Storage + storage storage.Storage cache *metadataCache format config.MetadataFormat repoConfig config.EncryptedRepositoryConfig @@ -102,7 +102,7 @@ func (mm *MetadataManager) writeEncryptedBlock(itemID string, content []byte) er func (mm *MetadataManager) readEncryptedBlock(itemID string) ([]byte, error) { content, err := mm.cache.GetBlock(itemID) if err != nil { - if err == blob.ErrBlockNotFound { + if err == storage.ErrBlockNotFound { return nil, ErrMetadataNotFound } return nil, fmt.Errorf("unexpected error reading %v: %v", itemID, err) @@ -222,7 +222,7 @@ func (mm *MetadataManager) ListContents(prefix string) (map[string][]byte, error // Config returns a configuration of storage its credentials that's suitable // for storing in configuration file. func (mm *MetadataManager) connectionConfiguration() (*config.RepositoryConnectionInfo, error) { - cip, ok := mm.storage.(blob.ConnectionInfoProvider) + cip, ok := mm.storage.(storage.ConnectionInfoProvider) if !ok { return nil, errors.New("repository does not support persisting configuration") } @@ -275,7 +275,7 @@ func (mm *MetadataManager) RemoveMany(itemIDs []string) error { } // newMetadataManager opens a MetadataManager for given storage and credentials. -func newMetadataManager(st blob.Storage, creds auth.Credentials) (*MetadataManager, error) { +func newMetadataManager(st storage.Storage, creds auth.Credentials) (*MetadataManager, error) { cache, err := newMetadataCache(st) if err != nil { return nil, err diff --git a/repo/metadata_manager_test.go b/repo/metadata_manager_test.go index 795a9f43f..c6e4d96b7 100644 --- a/repo/metadata_manager_test.go +++ b/repo/metadata_manager_test.go @@ -8,8 +8,8 @@ "strings" "github.com/kopia/kopia/auth" - "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/blob/filesystem" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/filesystem" "testing" ) @@ -173,7 +173,7 @@ func assertMetadataItems(t *testing.T, v *MetadataManager, prefix string, expect } } -func mustCreateFileStorage(t *testing.T, path string) blob.Storage { +func mustCreateFileStorage(t *testing.T, path string) storage.Storage { os.MkdirAll(path, 0700) s, err := filesystem.New(context.Background(), &filesystem.Options{ Path: path, diff --git a/repo/object_manager_test.go b/repo/object_manager_test.go index 76f718d48..10163e5e8 100644 --- a/repo/object_manager_test.go +++ b/repo/object_manager_test.go @@ -16,9 +16,9 @@ "github.com/kopia/kopia/auth" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/internal/jsonstream" "github.com/kopia/kopia/internal/storagetesting" + "github.com/kopia/kopia/storage" ) func setupTest(t *testing.T, mods ...func(o *NewRepositoryOptions)) (map[string][]byte, *Repository) { @@ -383,7 +383,7 @@ func TestReaderStoredBlockNotFound(t *testing.T) { t.Errorf("cannot parse object ID: %v", err) } reader, err := repo.Objects.Open(objectID) - if err != blob.ErrBlockNotFound || reader != nil { + if err != storage.ErrBlockNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } } diff --git a/repo/repository.go b/repo/repository.go index f227e899c..6d178f09b 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -4,8 +4,8 @@ "encoding/hex" "fmt" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/block" + "github.com/kopia/kopia/storage" ) // Repository represents storage where both content-addressable and user-addressable data is kept. @@ -13,7 +13,7 @@ type Repository struct { Blocks *block.Manager Objects *ObjectManager Metadata *MetadataManager - Storage blob.Storage + Storage storage.Storage ConfigFile string CacheDirectory string diff --git a/snapshot/upload_test.go b/snapshot/upload_test.go index df2f02f63..ab71bac78 100644 --- a/snapshot/upload_test.go +++ b/snapshot/upload_test.go @@ -8,10 +8,10 @@ "path/filepath" "reflect" - "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/blob/filesystem" "github.com/kopia/kopia/internal/mockfs" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/filesystem" "testing" @@ -22,7 +22,7 @@ type uploadTestHarness struct { sourceDir *mockfs.Directory repoDir string repo *repo.Repository - storage blob.Storage + storage storage.Storage } var errTest = fmt.Errorf("test error") diff --git a/blob/caching/cache_entry.go b/storage/caching/cache_entry.go similarity index 100% rename from blob/caching/cache_entry.go rename to storage/caching/cache_entry.go diff --git a/blob/caching/caching_storage.go b/storage/caching/caching_storage.go similarity index 89% rename from blob/caching/caching_storage.go rename to storage/caching/caching_storage.go index edca21d90..e169a7ce9 100644 --- a/blob/caching/caching_storage.go +++ b/storage/caching/caching_storage.go @@ -10,8 +10,8 @@ "time" "github.com/boltdb/bolt" - "github.com/kopia/kopia/blob" - "github.com/kopia/kopia/blob/filesystem" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/filesystem" ) var ( @@ -24,8 +24,8 @@ ) type cachingStorage struct { - master blob.Storage - cache blob.Storage + master storage.Storage + cache storage.Storage db *bolt.DB sizeBytes int64 @@ -103,7 +103,7 @@ func (c *cachingStorage) BlockSize(id string) (int64, error) { return entry.size, nil } - return 0, blob.ErrBlockNotFound + return 0, storage.ErrBlockNotFound } c.Lock(id) @@ -111,7 +111,7 @@ func (c *cachingStorage) BlockSize(id string) (int64, error) { l, err := c.master.BlockSize(id) if err != nil { - if err == blob.ErrBlockNotFound { + if err == storage.ErrBlockNotFound { c.setCacheEntrySize(id, sizeDoesNotExists) } return 0, err @@ -142,7 +142,7 @@ func (c *cachingStorage) GetBlock(id string, offset, length int64) ([]byte, erro if blockCacheEntry, ok := c.getCacheEntry(id); ok { if !blockCacheEntry.exists() { - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound } v, err := c.cache.GetBlock(id, offset, length) @@ -158,7 +158,7 @@ func (c *cachingStorage) GetBlock(id string, offset, length int64) ([]byte, erro l := int64(len(b)) c.cache.PutBlock(id, b) c.setCacheEntrySize(id, l) - } else if err == blob.ErrBlockNotFound { + } else if err == storage.ErrBlockNotFound { c.setCacheEntrySize(id, sizeDoesNotExists) } @@ -176,7 +176,7 @@ func (c *cachingStorage) PutBlock(id string, data []byte) error { return c.master.PutBlock(id, data) } -func (c *cachingStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { +func (c *cachingStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { return c.master.ListBlocks(prefix) } @@ -205,7 +205,7 @@ type Options struct { } // NewWrapper creates new caching storage wrapper. -func NewWrapper(ctx context.Context, master blob.Storage, options *Options) (blob.Storage, error) { +func NewWrapper(ctx context.Context, master storage.Storage, options *Options) (storage.Storage, error) { if options.CacheDir == "" { return nil, fmt.Errorf("Cache directory must be specified") } @@ -243,4 +243,4 @@ func NewWrapper(ctx context.Context, master blob.Storage, options *Options) (blo return s, nil } -var _ blob.Storage = &cachingStorage{} +var _ storage.Storage = &cachingStorage{} diff --git a/blob/caching/caching_storage_test.go b/storage/caching/caching_storage_test.go similarity index 98% rename from blob/caching/caching_storage_test.go rename to storage/caching/caching_storage_test.go index 1126cb724..0bfd9e4ca 100644 --- a/blob/caching/caching_storage_test.go +++ b/storage/caching/caching_storage_test.go @@ -8,8 +8,8 @@ "os" "strings" - "github.com/kopia/kopia/blob/logging" "github.com/kopia/kopia/internal/storagetesting" + "github.com/kopia/kopia/storage/logging" "testing" ) diff --git a/blob/caching/lock_map.go b/storage/caching/lock_map.go similarity index 100% rename from blob/caching/lock_map.go rename to storage/caching/lock_map.go diff --git a/blob/caching/lock_map_test.go b/storage/caching/lock_map_test.go similarity index 100% rename from blob/caching/lock_map_test.go rename to storage/caching/lock_map_test.go diff --git a/blob/config.go b/storage/config.go similarity index 98% rename from blob/config.go rename to storage/config.go index 2ae39932a..5f7bec86c 100644 --- a/blob/config.go +++ b/storage/config.go @@ -1,4 +1,4 @@ -package blob +package storage import ( "encoding/json" diff --git a/blob/doc.go b/storage/doc.go similarity index 86% rename from blob/doc.go rename to storage/doc.go index 6716e1290..948c45e6c 100644 --- a/blob/doc.go +++ b/storage/doc.go @@ -1,2 +1,2 @@ // Package blob implements simple storage of immutable, unstructured binary large objects (BLOBs). -package blob +package storage diff --git a/blob/errors.go b/storage/errors.go similarity index 95% rename from blob/errors.go rename to storage/errors.go index 85957dd4c..f95525ad1 100644 --- a/blob/errors.go +++ b/storage/errors.go @@ -1,4 +1,4 @@ -package blob +package storage import "errors" diff --git a/blob/filesystem/filesystem_options.go b/storage/filesystem/filesystem_options.go similarity index 100% rename from blob/filesystem/filesystem_options.go rename to storage/filesystem/filesystem_options.go diff --git a/blob/filesystem/filesystem_storage.go b/storage/filesystem/filesystem_storage.go similarity index 90% rename from blob/filesystem/filesystem_storage.go rename to storage/filesystem/filesystem_storage.go index 89d62a298..934e8165f 100644 --- a/blob/filesystem/filesystem_storage.go +++ b/storage/filesystem/filesystem_storage.go @@ -12,7 +12,7 @@ "strconv" "strings" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) const ( @@ -38,7 +38,7 @@ func (fs *fsStorage) BlockSize(blockID string) (int64, error) { } if os.IsNotExist(err) { - return 0, blob.ErrBlockNotFound + return 0, storage.ErrBlockNotFound } return 0, err @@ -49,7 +49,7 @@ func (fs *fsStorage) GetBlock(blockID string, offset, length int64) ([]byte, err f, err := os.Open(path) if os.IsNotExist(err) { - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound } if err != nil { @@ -77,8 +77,8 @@ func makeFileName(blockID string) string { return string(blockID) + fsStorageChunkSuffix } -func (fs *fsStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { - result := make(chan blob.BlockMetadata) +func (fs *fsStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { + result := make(chan storage.BlockMetadata) cancelled := make(chan bool) prefixString := string(prefix) @@ -108,7 +108,7 @@ func (fs *fsStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.Ca select { case <-cancelled: return - case result <- blob.BlockMetadata{ + case result <- storage.BlockMetadata{ BlockID: fullID, Length: e.Size(), TimeStamp: e.ModTime(), @@ -212,8 +212,8 @@ func parseShardString(shardString string) ([]int, error) { return result, nil } -func (fs *fsStorage) ConnectionInfo() blob.ConnectionInfo { - return blob.ConnectionInfo{ +func (fs *fsStorage) ConnectionInfo() storage.ConnectionInfo { + return storage.ConnectionInfo{ Type: fsStorageType, Config: &fs.Options, } @@ -224,7 +224,7 @@ func (fs *fsStorage) Close() error { } // New creates new filesystem-backed storage in a specified directory. -func New(ctx context.Context, opts *Options) (blob.Storage, error) { +func New(ctx context.Context, opts *Options) (storage.Storage, error) { var err error if _, err = os.Stat(opts.Path); err != nil { @@ -239,10 +239,10 @@ func New(ctx context.Context, opts *Options) (blob.Storage, error) { } func init() { - blob.AddSupportedStorage( + storage.AddSupportedStorage( fsStorageType, func() interface{} { return &Options{} }, - func(ctx context.Context, o interface{}) (blob.Storage, error) { + func(ctx context.Context, o interface{}) (storage.Storage, error) { return New(ctx, o.(*Options)) }) } diff --git a/blob/filesystem/filesystem_storage_test.go b/storage/filesystem/filesystem_storage_test.go similarity index 100% rename from blob/filesystem/filesystem_storage_test.go rename to storage/filesystem/filesystem_storage_test.go diff --git a/blob/gcs/gcs_options.go b/storage/gcs/gcs_options.go similarity index 100% rename from blob/gcs/gcs_options.go rename to storage/gcs/gcs_options.go diff --git a/blob/gcs/gcs_storage.go b/storage/gcs/gcs_storage.go similarity index 84% rename from blob/gcs/gcs_storage.go rename to storage/gcs/gcs_storage.go index f7cdf0064..1ce1b1f97 100644 --- a/blob/gcs/gcs_storage.go +++ b/storage/gcs/gcs_storage.go @@ -16,10 +16,10 @@ "google.golang.org/api/option" "github.com/efarrer/iothrottler" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" "golang.org/x/oauth2" - "cloud.google.com/go/storage" + gcsclient "cloud.google.com/go/storage" ) const ( @@ -30,8 +30,8 @@ type gcsStorage struct { Options ctx context.Context - storageClient *storage.Client - bucket *storage.BucketHandle + storageClient *gcsclient.Client + bucket *gcsclient.BucketHandle downloadThrottler *iothrottler.IOThrottlerPool uploadThrottler *iothrottler.IOThrottlerPool @@ -84,9 +84,9 @@ func isRetriableError(err error) bool { switch err { case nil: return false - case storage.ErrObjectNotExist: + case gcsclient.ErrObjectNotExist: return false - case storage.ErrBucketNotExist: + case gcsclient.ErrBucketNotExist: return false default: return true @@ -97,10 +97,10 @@ func translateError(err error) error { switch err { case nil: return nil - case storage.ErrObjectNotExist: - return blob.ErrBlockNotFound - case storage.ErrBucketNotExist: - return blob.ErrBlockNotFound + case gcsclient.ErrObjectNotExist: + return storage.ErrBlockNotFound + case gcsclient.ErrBucketNotExist: + return storage.ErrBlockNotFound default: return fmt.Errorf("unexpected GCS error: %v", err) } @@ -138,20 +138,20 @@ func (gcs *gcsStorage) getObjectNameString(b string) string { return gcs.Prefix + string(b) } -func (gcs *gcsStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { - ch := make(chan blob.BlockMetadata, 100) +func (gcs *gcsStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { + ch := make(chan storage.BlockMetadata, 100) cancelled := make(chan bool) go func() { defer close(ch) - lst := gcs.bucket.Objects(gcs.ctx, &storage.Query{ + lst := gcs.bucket.Objects(gcs.ctx, &gcsclient.Query{ Prefix: gcs.getObjectNameString(prefix), }) oa, err := lst.Next() for err == nil { - bm := blob.BlockMetadata{ + bm := storage.BlockMetadata{ BlockID: oa.Name[len(gcs.Prefix):], Length: oa.Size, TimeStamp: oa.Created, @@ -166,7 +166,7 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob. if err != iterator.Done { select { - case ch <- blob.BlockMetadata{Error: translateError(err)}: + case ch <- storage.BlockMetadata{Error: translateError(err)}: return case <-cancelled: return @@ -179,8 +179,8 @@ func (gcs *gcsStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob. } } -func (gcs *gcsStorage) ConnectionInfo() blob.ConnectionInfo { - return blob.ConnectionInfo{ +func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo { + return storage.ConnectionInfo{ Type: gcsStorageType, Config: &gcs.Options, } @@ -228,13 +228,13 @@ func tokenSourceFromCredentialsFile(ctx context.Context, fn string, scopes ...st // // By default the connection reuses credentials managed by (https://cloud.google.com/sdk/), // but this can be disabled by setting IgnoreDefaultCredentials to true. -func New(ctx context.Context, opt *Options) (blob.Storage, error) { +func New(ctx context.Context, opt *Options) (storage.Storage, error) { var ts oauth2.TokenSource var err error - scope := storage.ScopeReadWrite + scope := gcsclient.ScopeReadWrite if opt.ReadOnly { - scope = storage.ScopeReadOnly + scope = gcsclient.ScopeReadOnly } if sa := opt.ServiceAccountCredentials; sa != "" { @@ -253,7 +253,7 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) { hc := oauth2.NewClient(ctx, ts) hc.Transport = throttle.NewRoundTripper(hc.Transport, downloadThrottler, uploadThrottler) - cli, err := storage.NewClient(ctx, option.WithHTTPClient(hc)) + cli, err := gcsclient.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { return nil, err } @@ -273,14 +273,14 @@ func New(ctx context.Context, opt *Options) (blob.Storage, error) { } func init() { - blob.AddSupportedStorage( + storage.AddSupportedStorage( gcsStorageType, func() interface{} { return &Options{} }, - func(ctx context.Context, o interface{}) (blob.Storage, error) { + func(ctx context.Context, o interface{}) (storage.Storage, error) { return New(ctx, o.(*Options)) }) } -var _ blob.ConnectionInfoProvider = &gcsStorage{} +var _ storage.ConnectionInfoProvider = &gcsStorage{} diff --git a/blob/logging/logging_storage.go b/storage/logging/logging_storage.go similarity index 91% rename from blob/logging/logging_storage.go rename to storage/logging/logging_storage.go index a9d7ae918..f68dddd32 100644 --- a/blob/logging/logging_storage.go +++ b/storage/logging/logging_storage.go @@ -5,11 +5,11 @@ "log" "time" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) type loggingStorage struct { - base blob.Storage + base storage.Storage printf func(string, ...interface{}) prefix string } @@ -50,7 +50,7 @@ func (s *loggingStorage) DeleteBlock(id string) error { return err } -func (s *loggingStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { +func (s *loggingStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { t0 := time.Now() ch, cf := s.base.ListBlocks(prefix) s.printf(s.prefix+"ListBlocks(%q) took %v", prefix, time.Since(t0)) @@ -72,7 +72,7 @@ func (s *loggingStorage) Close() error { type Option func(s *loggingStorage) // NewWrapper returns a Storage wrapper that logs all storage commands. -func NewWrapper(wrapped blob.Storage, options ...Option) blob.Storage { +func NewWrapper(wrapped storage.Storage, options ...Option) storage.Storage { s := &loggingStorage{base: wrapped, printf: log.Printf} for _, o := range options { o(s) diff --git a/blob/logging/logging_storage_test.go b/storage/logging/logging_storage_test.go similarity index 100% rename from blob/logging/logging_storage_test.go rename to storage/logging/logging_storage_test.go diff --git a/blob/registry.go b/storage/registry.go similarity index 98% rename from blob/registry.go rename to storage/registry.go index be6526ad8..776e42cc0 100644 --- a/blob/registry.go +++ b/storage/registry.go @@ -1,4 +1,4 @@ -package blob +package storage import ( "context" diff --git a/blob/storage.go b/storage/storage.go similarity index 98% rename from blob/storage.go rename to storage/storage.go index f66a62d2d..fc852460a 100644 --- a/blob/storage.go +++ b/storage/storage.go @@ -1,4 +1,4 @@ -package blob +package storage import ( "io" diff --git a/blob/webdav/webdav_options.go b/storage/webdav/webdav_options.go similarity index 100% rename from blob/webdav/webdav_options.go rename to storage/webdav/webdav_options.go diff --git a/blob/webdav/webdav_props.go b/storage/webdav/webdav_props.go similarity index 100% rename from blob/webdav/webdav_props.go rename to storage/webdav/webdav_props.go diff --git a/blob/webdav/webdav_request.go b/storage/webdav/webdav_request.go similarity index 100% rename from blob/webdav/webdav_request.go rename to storage/webdav/webdav_request.go diff --git a/blob/webdav/webdav_storage.go b/storage/webdav/webdav_storage.go similarity index 91% rename from blob/webdav/webdav_storage.go rename to storage/webdav/webdav_storage.go index 8d1496636..045f0c1ee 100644 --- a/blob/webdav/webdav_storage.go +++ b/storage/webdav/webdav_storage.go @@ -10,7 +10,7 @@ "strconv" "strings" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) const ( @@ -49,7 +49,7 @@ func (d *davStorage) BlockSize(blockID string) (int64, error) { switch resp.StatusCode { case http.StatusNotFound: - return 0, blob.ErrBlockNotFound + return 0, storage.ErrBlockNotFound case http.StatusOK: return resp.ContentLength, nil default: @@ -78,7 +78,7 @@ func (d *davStorage) GetBlock(blockID string, offset, length int64) ([]byte, err switch resp.StatusCode { case http.StatusNotFound: - return nil, blob.ErrBlockNotFound + return nil, storage.ErrBlockNotFound case http.StatusOK, http.StatusPartialContent: return ioutil.ReadAll(resp.Body) default: @@ -98,8 +98,8 @@ func makeFileName(blockID string) string { return string(blockID) + fsStorageChunkSuffix } -func (d *davStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.CancelFunc) { - result := make(chan blob.BlockMetadata) +func (d *davStorage) ListBlocks(prefix string) (chan storage.BlockMetadata, storage.CancelFunc) { + result := make(chan storage.BlockMetadata) cancelled := make(chan bool) prefixString := string(prefix) @@ -127,7 +127,7 @@ func (d *davStorage) ListBlocks(prefix string) (chan blob.BlockMetadata, blob.Ca select { case <-cancelled: return - case result <- blob.BlockMetadata{ + case result <- storage.BlockMetadata{ BlockID: fullID, Length: e.length, TimeStamp: e.modTime, @@ -156,7 +156,7 @@ func (d *davStorage) makeCollectionAll(urlStr string) error { case nil: return nil - case blob.ErrBlockNotFound: + case storage.ErrBlockNotFound: parent := getParentURL(urlStr) if parent == "" { return fmt.Errorf("can't create %q", urlStr) @@ -186,7 +186,7 @@ func (d *davStorage) makeCollection(urlStr string) error { defer resp.Body.Close() switch resp.StatusCode { case http.StatusConflict: - return blob.ErrBlockNotFound + return storage.ErrBlockNotFound case http.StatusOK, http.StatusCreated: return nil default: @@ -263,7 +263,7 @@ func (d *davStorage) putBlockInternal(urlStr string, data []byte) error { return nil case http.StatusNotFound: - return blob.ErrBlockNotFound + return storage.ErrBlockNotFound default: return fmt.Errorf("invalid response from webdav server: %v", resp.StatusCode) @@ -276,7 +276,7 @@ func (d *davStorage) PutBlock(blockID string, data []byte) error { tmpURL := url + "-" + makeClientNonce() err := d.putBlockInternal(tmpURL, data) - if err == blob.ErrBlockNotFound { + if err == storage.ErrBlockNotFound { if err := d.makeCollectionAll(shardPath); err != nil { return err } @@ -343,8 +343,8 @@ func parseShardString(shardString string) ([]int, error) { return result, nil } -func (d *davStorage) ConnectionInfo() blob.ConnectionInfo { - return blob.ConnectionInfo{ +func (d *davStorage) ConnectionInfo() storage.ConnectionInfo { + return storage.ConnectionInfo{ Type: davStorageType, Config: &d.Options, } @@ -355,7 +355,7 @@ func (d *davStorage) Close() error { } // New creates new WebDAV-backed storage in a specified URL. -func New(ctx context.Context, opts *Options) (blob.Storage, error) { +func New(ctx context.Context, opts *Options) (storage.Storage, error) { r := &davStorage{ Options: *opts, Client: http.DefaultClient, @@ -372,10 +372,10 @@ func New(ctx context.Context, opts *Options) (blob.Storage, error) { } func init() { - blob.AddSupportedStorage( + storage.AddSupportedStorage( davStorageType, func() interface{} { return &Options{} }, - func(ctx context.Context, o interface{}) (blob.Storage, error) { + func(ctx context.Context, o interface{}) (storage.Storage, error) { return New(ctx, o.(*Options)) }) } diff --git a/blob/webdav/webdav_storage_test.go b/storage/webdav/webdav_storage_test.go similarity index 100% rename from blob/webdav/webdav_storage_test.go rename to storage/webdav/webdav_storage_test.go