diff --git a/Makefile b/Makefile index 323b5d5f9..3d8d25694 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,7 @@ lint: golint github.com/kopia/kopia/... vet: - go vet github.com/kopia/kopia/... + go tool vet -all . deps: go get -u -t -v github.com/kopia/kopia/... diff --git a/blob/doc.go b/blob/doc.go deleted file mode 100644 index 6716e1290..000000000 --- a/blob/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package blob implements simple storage of immutable, unstructured binary large objects (BLOBs). -package blob diff --git a/cmd/kopia/command_create.go b/cmd/kopia/command_create.go index 1a0e36057..409d738ba 100644 --- a/cmd/kopia/command_create.go +++ b/cmd/kopia/command_create.go @@ -7,8 +7,8 @@ import ( "gopkg.in/alecthomas/kingpin.v2" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage" "github.com/kopia/kopia/vault" ) @@ -59,7 +59,7 @@ func repositoryFormat() (*repo.Format, error) { return f, nil } -func openStorageAndEnsureEmpty(url string) (blob.Storage, error) { +func openStorageAndEnsureEmpty(url string) (storage.Storage, error) { s, err := newStorageFromURL(url) if err != nil { return nil, err @@ -83,7 +83,7 @@ func runCreateCommand(context *kingpin.ParseContext) error { return fmt.Errorf("unable to get vault storage: %v", err) } - var repositoryStorage blob.Storage + var repositoryStorage storage.Storage if *createCommandRepository == "colocated" { repositoryStorage = vaultStorage diff --git a/cmd/kopia/urls.go b/cmd/kopia/urls.go index 13f4e5906..0eb498821 100644 --- a/cmd/kopia/urls.go +++ b/cmd/kopia/urls.go @@ -7,10 +7,12 @@ import ( "strconv" "strings" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" + fsstorage "github.com/kopia/kopia/storage/filesystem" + gcsstorage "github.com/kopia/kopia/storage/gcs" ) -func newStorageFromURL(urlString string) (blob.Storage, error) { +func newStorageFromURL(urlString string) (storage.Storage, error) { if strings.HasPrefix(urlString, "/") { urlString = "file://" + urlString } @@ -22,26 +24,26 @@ func newStorageFromURL(urlString string) (blob.Storage, error) { switch u.Scheme { case "file": - var fso blob.FSStorageOptions + var fso fsstorage.Options if err := parseFilesystemURL(&fso, u); err != nil { return nil, err } - return blob.NewFSStorage(&fso) + return fsstorage.New(&fso) case "gs", "gcs": - var gcso blob.GCSStorageOptions + var gcso gcsstorage.Options if err := parseGoogleCloudStorageURL(&gcso, u); err != nil { return nil, err } - return blob.NewGCSStorage(&gcso) + return gcsstorage.New(&gcso) default: return nil, fmt.Errorf("unrecognized storage type: %v", u.Scheme) } } -func parseFilesystemURL(fso *blob.FSStorageOptions, u *url.URL) error { +func parseFilesystemURL(fso *fsstorage.Options, u *url.URL) error { if u.Opaque != "" { fso.Path = u.Opaque } else { @@ -66,7 +68,7 @@ func parseFilesystemURL(fso *blob.FSStorageOptions, u *url.URL) error { return nil } -func parseGoogleCloudStorageURL(gcso *blob.GCSStorageOptions, u *url.URL) error { +func parseGoogleCloudStorageURL(gcso *gcsstorage.Options, u *url.URL) error { gcso.BucketName = u.Host gcso.Prefix = u.Path return nil diff --git a/fs/upload_test.go b/fs/upload_test.go index 2771f6479..6be6521a1 100644 --- a/fs/upload_test.go +++ b/fs/upload_test.go @@ -6,8 +6,9 @@ import ( "os" "path/filepath" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/repo" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/filesystem" "testing" ) @@ -16,7 +17,7 @@ type uploadTestHarness struct { sourceDir string repoDir string repo repo.Repository - storage blob.Storage + storage storage.Storage lister *perPathLister uploader Uploader } @@ -61,7 +62,7 @@ func newUploadTestHarness() *uploadTestHarness { panic("cannot create temp directory: " + err.Error()) } - storage, err := blob.NewFSStorage(&blob.FSStorageOptions{ + storage, err := filesystem.New(&filesystem.Options{ Path: repoDir, }) diff --git a/repo/buffer_manager.go b/repo/buffer_manager.go index b485653d7..ee19a24b8 100644 --- a/repo/buffer_manager.go +++ b/repo/buffer_manager.go @@ -7,7 +7,7 @@ import ( "sync" "sync/atomic" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // bufferManager manages pool of reusable bytes.Buffer objects. @@ -35,7 +35,7 @@ func (mgr *bufferManager) returnBuffer(b *bytes.Buffer) { mgr.pool.Put(b) } -func (mgr *bufferManager) returnBufferOnClose(b *bytes.Buffer) blob.ReaderWithLength { +func (mgr *bufferManager) returnBufferOnClose(b *bytes.Buffer) storage.ReaderWithLength { return &returnOnCloser{ buffer: b, mgr: mgr, diff --git a/repo/counting_reader.go b/repo/counting_reader.go index 86064204d..81fb0cdf3 100644 --- a/repo/counting_reader.go +++ b/repo/counting_reader.go @@ -5,12 +5,12 @@ import ( "sync/atomic" "unsafe" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // countingReader wraps an io.ReadCloser and counts bytes read. type countingReader struct { - blob.ReaderWithLength + storage.ReaderWithLength counter *int64 } @@ -25,7 +25,7 @@ func (cr *countingReader) String() string { return fmt.Sprintf("countingReader(%v)", cr.ReaderWithLength) } -func newCountingReader(source blob.ReaderWithLength, counter *int64) blob.ReaderWithLength { +func newCountingReader(source storage.ReaderWithLength, counter *int64) storage.ReaderWithLength { if uintptr(unsafe.Pointer(counter))&7 != 0 { panic("counter address must be 64-bit aligned") } diff --git a/repo/crypto_stream.go b/repo/crypto_stream.go index 44fa73876..808612408 100644 --- a/repo/crypto_stream.go +++ b/repo/crypto_stream.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // encryptingReader wraps an io.Reader and returns data encrypted using a stream cipher @@ -55,7 +55,7 @@ func (er *encryptingReader) String() string { return fmt.Sprintf("encryptingReader(%v)", er.source) } -func newEncryptingReader(source blob.ReaderWithLength, c cipher.Stream) blob.ReaderWithLength { +func newEncryptingReader(source storage.ReaderWithLength, c cipher.Stream) storage.ReaderWithLength { return &encryptingReader{ source: source, closer: source, diff --git a/repo/crypto_stream_test.go b/repo/crypto_stream_test.go index a178c413b..0ac000c54 100644 --- a/repo/crypto_stream_test.go +++ b/repo/crypto_stream_test.go @@ -7,7 +7,7 @@ import ( "fmt" "io/ioutil" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" "testing" ) @@ -45,7 +45,7 @@ func TestCryptoStream(t *testing.T) { continue } enc := newEncryptingReader( - blob.NewReader(bytes.NewBuffer(data)), + storage.NewReader(bytes.NewBuffer(data)), s.cipher) v, err := ioutil.ReadAll(enc) actual := fmt.Sprintf("%x", v) diff --git a/repo/object_reader.go b/repo/object_reader.go index de7050002..60e995baa 100644 --- a/repo/object_reader.go +++ b/repo/object_reader.go @@ -4,7 +4,7 @@ import ( "fmt" "io" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) type seekTableEntry struct { @@ -26,7 +26,7 @@ func (r *seekTableEntry) String() string { } type objectReader struct { - storage blob.Storage + storage storage.Storage seekTable []seekTableEntry diff --git a/repo/object_writer.go b/repo/object_writer.go index b6d4de4a2..02e2db438 100644 --- a/repo/object_writer.go +++ b/repo/object_writer.go @@ -5,7 +5,7 @@ import ( "fmt" "io" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" ) // ObjectWriter allows writing content to the storage and supports automatic deduplication and encryption @@ -101,7 +101,7 @@ func (w *objectWriter) flushBuffer(force bool) error { return err } - if err := w.repo.storage.PutBlock(objectID.BlockID(), blockReader, blob.PutOptionsDefault); err != nil { + if err := w.repo.storage.PutBlock(objectID.BlockID(), blockReader, storage.PutOptionsDefault); err != nil { return fmt.Errorf( "error when flushing chunk %d of %s to %#v: %#v", w.flushedObjectCount, diff --git a/repo/repository.go b/repo/repository.go index 23809cd2f..2710e4320 100644 --- a/repo/repository.go +++ b/repo/repository.go @@ -14,7 +14,8 @@ import ( "strings" "sync/atomic" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/logging" ) const ( @@ -34,14 +35,14 @@ var constantIV = []byte("kopiakopiakopiakopiakopiakopiakopiakopiakopiakopiakopia // Repository objects addressed by their content allows reading and writing them. type Repository interface { - // NewWriter opens an ObjectWriter for writing new content to the blob. + // NewWriter opens an ObjectWriter for writing new content to the storage. NewWriter(options ...WriterOption) ObjectWriter // Open creates an io.ReadSeeker for reading object with a specified ID. Open(objectID ObjectID) (ObjectReader, error) Flush() error - Storage() blob.Storage + Storage() storage.Storage Close() ResetStats() @@ -68,7 +69,7 @@ type RepositoryStats struct { type keygenFunc func([]byte) (blockIDBytes []byte, key []byte) type repository struct { - storage blob.Storage + storage storage.Storage verbose bool bufferManager *bufferManager stats *RepositoryStats @@ -83,7 +84,7 @@ func (repo *repository) Close() { } func (repo *repository) Flush() error { - if f, ok := repo.storage.(blob.Flusher); ok { + if f, ok := repo.storage.(storage.Flusher); ok { return f.Flush() } @@ -98,7 +99,7 @@ func (repo *repository) Stats() RepositoryStats { return *repo.stats } -func (repo *repository) Storage() blob.Storage { +func (repo *repository) Storage() storage.Storage { return repo.storage } @@ -145,7 +146,7 @@ type RepositoryOption func(o *repository) error // of goroutines. func WriteBack(workerCount int) RepositoryOption { return func(o *repository) error { - o.storage = blob.NewWriteBackWrapper(o.storage, workerCount) + o.storage = storage.NewWriteBackWrapper(o.storage, workerCount) return nil } } @@ -153,7 +154,7 @@ func WriteBack(workerCount int) RepositoryOption { // EnableLogging is an RepositoryOption that causes all storage access to be logged. func EnableLogging() RepositoryOption { return func(o *repository) error { - o.storage = blob.NewLoggingWrapper(o.storage) + o.storage = logging.NewWrapper(o.storage) return nil } } @@ -166,7 +167,7 @@ func hmacFunc(key []byte, hf func() hash.Hash) func() hash.Hash { // NewRepository creates new Repository with the specified storage, options, and key provider. func NewRepository( - r blob.Storage, + r storage.Storage, f *Format, options ...RepositoryOption, ) (Repository, error) { @@ -206,7 +207,7 @@ func (repo *repository) hashBuffer(data []byte) ([]byte, []byte) { return repo.idFormat.hashBuffer(data, repo.format.Secret) } -func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, blob.ReaderWithLength, error) { +func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string) (ObjectID, storage.ReaderWithLength, error) { var data []byte if buffer != nil { data = buffer.Bytes() @@ -234,7 +235,7 @@ func (repo *repository) hashBufferForWriting(buffer *bytes.Buffer, prefix string atomic.AddInt64(&repo.stats.HashedBytes, int64(len(data))) if buffer == nil { - return objectID, blob.NewReader(bytes.NewBuffer(nil)), nil + return objectID, storage.NewReader(bytes.NewBuffer(nil)), nil } blockReader := repo.bufferManager.returnBufferOnClose(buffer) diff --git a/repo/repository_test.go b/repo/repository_test.go index 219f55111..bb884714b 100644 --- a/repo/repository_test.go +++ b/repo/repository_test.go @@ -12,7 +12,8 @@ import ( "strings" "testing" - "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/storage" + "github.com/kopia/kopia/storage/storagetesting" ) func testFormat() *Format { @@ -40,7 +41,7 @@ func getMd5LObjectID(data []byte) string { func setupTest(t *testing.T) (data map[string][]byte, repo Repository) { data = map[string][]byte{} - st := blob.NewMapStorage(data) + st := storagetesting.NewMapStorage(data) repo, err := NewRepository(st, testFormat()) if err != nil { @@ -238,7 +239,7 @@ func TestHMAC(t *testing.T) { s.ObjectFormat = "md5" s.Secret = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25} - repo, err := NewRepository(blob.NewMapStorage(data), s) + repo, err := NewRepository(storagetesting.NewMapStorage(data), s) if err != nil { t.Errorf("cannot create manager: %v", err) } @@ -323,7 +324,7 @@ func TestReaderStoredBlockNotFound(t *testing.T) { t.Errorf("cannot parse object ID: %v", err) } reader, err := repo.Open(objectID) - if err != blob.ErrBlockNotFound || reader != nil { + if err != storage.ErrBlockNotFound || reader != nil { t.Errorf("unexpected result: reader: %v err: %v", reader, err) } } @@ -478,7 +479,7 @@ func TestFormats(t *testing.T) { for caseIndex, c := range cases { data := map[string][]byte{} - st := blob.NewMapStorage(data) + st := storagetesting.NewMapStorage(data) t.Logf("verifying %#v", c.format) repo, err := NewRepository(st, c.format) @@ -517,7 +518,7 @@ func TestFormats(t *testing.T) { func TestInvalidEncryptionKey(t *testing.T) { data := map[string][]byte{} - st := blob.NewMapStorage(data) + st := storagetesting.NewMapStorage(data) format := Format{ Version: "1", ObjectFormat: "sha512-aes256", diff --git a/blob/config.go b/storage/config.go similarity index 98% rename from blob/config.go rename to storage/config.go index 882de9eed..96652436e 100644 --- a/blob/config.go +++ b/storage/config.go @@ -1,4 +1,4 @@ -package blob +package storage import ( "encoding/json" diff --git a/storage/doc.go b/storage/doc.go new file mode 100644 index 000000000..a2c193b6f --- /dev/null +++ b/storage/doc.go @@ -0,0 +1,2 @@ +// Package storage implements simple storage of immutable, unstructured binary large objects (BLOBs). +package storage diff --git a/blob/errors.go b/storage/errors.go similarity index 92% rename from blob/errors.go rename to storage/errors.go index d69a67a4a..f95525ad1 100644 --- a/blob/errors.go +++ b/storage/errors.go @@ -1,9 +1,9 @@ -package blob +package storage import "errors" var ( - // ErrBlockNotFound is returned when a block cannot be found in blob. + // ErrBlockNotFound is returned when a block cannot be found in storage. ErrBlockNotFound = errors.New("block not found") // ErrInvalidChecksum is returned when a storage block is invalid, which may indicate diff --git a/blob/filesystem.go b/storage/filesystem/filesystem.go similarity index 75% rename from blob/filesystem.go rename to storage/filesystem/filesystem.go index 6daaf6009..151a21974 100644 --- a/blob/filesystem.go +++ b/storage/filesystem/filesystem.go @@ -1,4 +1,4 @@ -package blob +package filesystem import ( "fmt" @@ -9,6 +9,8 @@ import ( "path/filepath" "strconv" "strings" + + "github.com/kopia/kopia/storage" ) const ( @@ -23,45 +25,9 @@ var ( ) type fsStorage struct { - FSStorageOptions + Options } -// FSStorageOptions defines options for Filesystem-backed blob. -type FSStorageOptions struct { - Path string `json:"path"` - - DirectoryShards []int `json:"dirShards,omitempty"` - - FileMode os.FileMode `json:"fileMode,omitempty"` - DirectoryMode os.FileMode `json:"dirMode,omitempty"` - - FileUID *int `json:"uid,omitempty"` - FileGID *int `json:"gid,omitempty"` -} - -func (fso *FSStorageOptions) fileMode() os.FileMode { - if fso.FileMode == 0 { - return fsDefaultFileMode - } - - return fso.FileMode -} - -func (fso *FSStorageOptions) dirMode() os.FileMode { - if fso.DirectoryMode == 0 { - return fsDefaultDirMode - } - - return fso.DirectoryMode -} - -func (fso *FSStorageOptions) shards() []int { - if fso.DirectoryShards == nil { - return fsDefaultShards - } - - return fso.DirectoryShards -} func (fs *fsStorage) BlockExists(blockID string) (bool, error) { _, path := fs.getShardedPathAndFilePath(blockID) _, err := os.Stat(path) @@ -84,7 +50,7 @@ func (fs *fsStorage) GetBlock(blockID string) ([]byte, error) { } if os.IsNotExist(err) { - return nil, ErrBlockNotFound + return nil, storage.ErrBlockNotFound } return nil, err @@ -102,8 +68,8 @@ func makeFileName(blockID string) string { return string(blockID) + fsStorageChunkSuffix } -func (fs *fsStorage) ListBlocks(prefix string) chan (BlockMetadata) { - result := make(chan (BlockMetadata)) +func (fs *fsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { + result := make(chan (storage.BlockMetadata)) prefixString := string(prefix) @@ -129,7 +95,7 @@ func (fs *fsStorage) ListBlocks(prefix string) chan (BlockMetadata) { } } else if fullID, ok := getstringFromFileName(currentPrefix + e.Name()); ok { if strings.HasPrefix(string(fullID), prefixString) { - result <- BlockMetadata{ + result <- storage.BlockMetadata{ BlockID: fullID, Length: uint64(e.Size()), TimeStamp: e.ModTime(), @@ -149,7 +115,7 @@ func (fs *fsStorage) ListBlocks(prefix string) chan (BlockMetadata) { return result } -func (fs *fsStorage) PutBlock(blockID string, data ReaderWithLength, options PutOptions) error { +func (fs *fsStorage) PutBlock(blockID string, data storage.ReaderWithLength, options storage.PutOptions) error { // Close the data reader regardless of whether we use it or not. defer data.Close() @@ -238,10 +204,10 @@ func parseShardString(shardString string) ([]int, error) { return result, nil } -func (fs *fsStorage) ConnectionInfo() ConnectionInfo { - return ConnectionInfo{ - fsStorageType, - &fs.FSStorageOptions, +func (fs *fsStorage) ConnectionInfo() storage.ConnectionInfo { + return storage.ConnectionInfo{ + Type: fsStorageType, + Config: &fs.Options, } } @@ -249,8 +215,8 @@ func (fs *fsStorage) Close() error { return nil } -// NewFSStorage creates new fs-backed storage in a specified directory. -func NewFSStorage(options *FSStorageOptions) (Storage, error) { +// New creates new filesystem-backed storage in a specified directory. +func New(options *Options) (storage.Storage, error) { var err error if _, err = os.Stat(options.Path); err != nil { @@ -258,17 +224,17 @@ func NewFSStorage(options *FSStorageOptions) (Storage, error) { } r := &fsStorage{ - FSStorageOptions: *options, + Options: *options, } return r, nil } func init() { - AddSupportedStorage( + storage.AddSupportedStorage( fsStorageType, - func() interface{} { return &FSStorageOptions{} }, - func(o interface{}) (Storage, error) { - return NewFSStorage(o.(*FSStorageOptions)) + func() interface{} { return &Options{} }, + func(o interface{}) (storage.Storage, error) { + return New(o.(*Options)) }) } diff --git a/storage/filesystem/filesystem_test.go b/storage/filesystem/filesystem_test.go new file mode 100644 index 000000000..74dac36c7 --- /dev/null +++ b/storage/filesystem/filesystem_test.go @@ -0,0 +1,36 @@ +package filesystem + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/kopia/kopia/storage/storagetesting" +) + +func TestFileStorage(t *testing.T) { + // Test varioush shard configurations. + for _, shardSpec := range [][]int{ + []int{0}, + []int{1}, + []int{3, 3}, + []int{2}, + []int{1, 1}, + []int{1, 2}, + []int{2, 2, 2}, + } { + path, _ := ioutil.TempDir("", "r-fs") + defer os.RemoveAll(path) + + r, err := New(&Options{ + Path: path, + DirectoryShards: shardSpec, + }) + + if r == nil || err != nil { + t.Errorf("unexpected result: %v %v", r, err) + } + + storagetesting.VerifyStorage(t, r) + } +} diff --git a/storage/filesystem/options.go b/storage/filesystem/options.go new file mode 100644 index 000000000..4d067474c --- /dev/null +++ b/storage/filesystem/options.go @@ -0,0 +1,40 @@ +package filesystem + +import "os" + +// Options defines options for Filesystem-backed storage. +type Options struct { + Path string `json:"path"` + + DirectoryShards []int `json:"dirShards,omitempty"` + + FileMode os.FileMode `json:"fileMode,omitempty"` + DirectoryMode os.FileMode `json:"dirMode,omitempty"` + + FileUID *int `json:"uid,omitempty"` + FileGID *int `json:"gid,omitempty"` +} + +func (fso *Options) fileMode() os.FileMode { + if fso.FileMode == 0 { + return fsDefaultFileMode + } + + return fso.FileMode +} + +func (fso *Options) dirMode() os.FileMode { + if fso.DirectoryMode == 0 { + return fsDefaultDirMode + } + + return fso.DirectoryMode +} + +func (fso *Options) shards() []int { + if fso.DirectoryShards == nil { + return fsDefaultShards + } + + return fso.DirectoryShards +} diff --git a/blob/gcs.go b/storage/gcs/gcs.go similarity index 71% rename from blob/gcs.go rename to storage/gcs/gcs.go index e0796427d..b9a17e987 100644 --- a/blob/gcs.go +++ b/storage/gcs/gcs.go @@ -1,4 +1,4 @@ -package blob +package gcs import ( "encoding/json" @@ -7,10 +7,11 @@ import ( "io/ioutil" "log" "net/http" - "net/http/httptest" "os" "time" + "github.com/kopia/kopia/storage" + "golang.org/x/net/context" "golang.org/x/oauth2" "golang.org/x/oauth2/google" @@ -22,36 +23,13 @@ import ( const ( gcsStorageType = "gcs" - // Those are not really secret, since the app is installed. + // Those are not really set, since the app is installed. googleCloudClientID = "194841383482-nmn10h4mnllnsvou7qr55tfh5jsmtkap.apps.googleusercontent.com" googleCloudClientSecret = "ZL52E96Q7iRCD9YXVA7U6UaI" ) -// GCSStorageOptions defines options Google Cloud Storage-backed blob. -type GCSStorageOptions struct { - // BucketName is the name of the GCS bucket where data is stored. - BucketName string `json:"bucket"` - - // Prefix specifies additional string to prepend to all objects. - Prefix string `json:"prefix,omitempty"` - - // TokenCacheFile is the name of the file that will persist the OAuth2 token. - // If not specified, the token will be persisted in GCSStorageOptions. - TokenCacheFile string `json:"tokenCacheFile,omitempty"` - - // Token is the OAuth2 token (when TokenCacheFile is empty) - Token *oauth2.Token `json:"token,omitempty"` - - // ReadOnly causes the storage to be configured without write permissions, to prevent accidental - // modifications to the data. - ReadOnly bool `json:"readonly"` - - // IgnoreDefaultCredentials disables the use of credentials managed by Google Cloud SDK (gcloud). - IgnoreDefaultCredentials bool `json:"ignoreDefaultCredentials"` -} - type gcsStorage struct { - GCSStorageOptions + Options objectsService *gcsclient.ObjectsService } @@ -70,7 +48,7 @@ func (gcs *gcsStorage) GetBlock(b string) ([]byte, error) { if err != nil { if err, ok := err.(*googleapi.Error); ok { if err.Code == http.StatusNotFound { - return nil, ErrBlockNotFound + return nil, storage.ErrBlockNotFound } } @@ -82,7 +60,7 @@ func (gcs *gcsStorage) GetBlock(b string) ([]byte, error) { return ioutil.ReadAll(v.Body) } -func (gcs *gcsStorage) PutBlock(b string, data ReaderWithLength, options PutOptions) error { +func (gcs *gcsStorage) PutBlock(b string, data storage.ReaderWithLength, options storage.PutOptions) error { defer data.Close() object := gcsclient.Object{ Name: gcs.getObjectNameString(b), @@ -94,7 +72,7 @@ func (gcs *gcsStorage) PutBlock(b string, data ReaderWithLength, options PutOpti // Specify exact chunk size to ensure data is uploaded in one shot or not at all. googleapi.ChunkSize(data.Len()), ) - if options&PutOptionsOverwrite == 0 { + if options&storage.PutOptionsOverwrite == 0 { if ok, _ := gcs.BlockExists(b); ok { return nil } @@ -129,8 +107,8 @@ func (gcs *gcsStorage) getObjectNameString(b string) string { return gcs.Prefix + string(b) } -func (gcs *gcsStorage) ListBlocks(prefix string) chan (BlockMetadata) { - ch := make(chan BlockMetadata, 100) +func (gcs *gcsStorage) ListBlocks(prefix string) chan (storage.BlockMetadata) { + ch := make(chan storage.BlockMetadata, 100) go func() { ps := gcs.getObjectNameString(prefix) @@ -143,11 +121,11 @@ func (gcs *gcsStorage) ListBlocks(prefix string) chan (BlockMetadata) { for _, o := range page.Items { t, e := time.Parse(time.RFC3339, o.TimeCreated) if e != nil { - ch <- BlockMetadata{ + ch <- storage.BlockMetadata{ Error: e, } } else { - ch <- BlockMetadata{ + ch <- storage.BlockMetadata{ BlockID: string(o.Name)[len(gcs.Prefix):], Length: o.Size, TimeStamp: t, @@ -173,10 +151,10 @@ func (gcs *gcsStorage) Flush() error { return nil } -func (gcs *gcsStorage) ConnectionInfo() ConnectionInfo { - return ConnectionInfo{ - gcsStorageType, - &gcs.GCSStorageOptions, +func (gcs *gcsStorage) ConnectionInfo() storage.ConnectionInfo { + return storage.ConnectionInfo{ + Type: gcsStorageType, + Config: &gcs.Options, } } @@ -210,19 +188,19 @@ func saveToken(file string, token *oauth2.Token) { json.NewEncoder(f).Encode(token) } -// NewGCSStorage creates new Google Cloud Storage-backed storage with specified options: +// New creates new Google Cloud Storage-backed storage with specified options: // // - the 'BucketName' field is required and all other parameters are optional. // // By default the connection reuses credentials managed by (https://cloud.google.com/sdk/), // but this can be disabled by setting IgnoreDefaultCredentials to true. -func NewGCSStorage(options *GCSStorageOptions) (Storage, error) { +func New(options *Options) (storage.Storage, error) { ctx := context.TODO() //ctx = httptrace.WithClientTrace(ctx, &httptrace.ClientTrace{}) gcs := &gcsStorage{ - GCSStorageOptions: *options, + Options: *options, } if gcs.BucketName == "" { @@ -346,48 +324,13 @@ func tokenFromWeb(ctx context.Context, config *oauth2.Config) (*oauth2.Token, er return token, nil } -func authPrompt(url string, state string) (authenticationCode string, err error) { - ch := make(chan string) - - ts := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) { - if req.URL.Path == "/favicon.ico" { - http.Error(rw, "", 404) - return - } - if req.FormValue("state") != state { - log.Printf("State doesn't match: req = %#v", req) - http.Error(rw, "", 500) - return - } - if code := req.FormValue("code"); code != "" { - fmt.Fprintf(rw, "