diff --git a/blob/config.go b/blob/config.go index 46b2393ed..7a47378a3 100644 --- a/blob/config.go +++ b/blob/config.go @@ -4,10 +4,10 @@ import ( "encoding/json" ) -// StorageConfiguration is a JSON-serializable description of Storage and its configuration. +// StorageConfiguration is a JSON-serializable description of Storage and its options. type StorageConfiguration struct { Type string - Config interface{} + Config StorageOptions } // UnmarshalJSON parses the JSON-encoded data into StorageConfiguration. diff --git a/blob/filesystem.go b/blob/filesystem.go index 80a21322c..fa5bd3ab5 100644 --- a/blob/filesystem.go +++ b/blob/filesystem.go @@ -5,6 +5,7 @@ import ( "io" "io/ioutil" "math/rand" + "net/url" "os" "path/filepath" "strconv" @@ -12,8 +13,7 @@ import ( ) const ( - fsStorageType = "fs" - + fsStorageType = "file" fsStorageChunkSuffix = ".f" ) @@ -40,6 +40,96 @@ type FSStorageOptions struct { FileGID *int `json:"gid,omitempty"` } +func (fso *FSStorageOptions) fileMode() os.FileMode { + if fso.FileMode == 0 { + return fsDefaultFileMode + } + + return fso.FileMode +} + +func (fso *FSStorageOptions) dirMode() os.FileMode { + if fso.DirectoryMode == 0 { + return fsDefaultDirMode + } + + return fso.DirectoryMode +} + +func (fso *FSStorageOptions) shards() []int { + if fso.DirectoryShards == nil { + return fsDefaultShards + } + + return fso.DirectoryShards +} + +// ParseURL parses the given URL into FSStorageOptions. +func (fso *FSStorageOptions) ParseURL(u *url.URL) error { + if u.Scheme != "file" { + return fmt.Errorf("invalid scheme, expected 'file'") + } + + if u.Opaque != "" { + fso.Path = u.Opaque + } else { + fso.Path = u.Path + } + fso.FileUID = getIntPtrValue(u, "uid", 10) + fso.FileGID = getIntPtrValue(u, "gid", 10) + fso.FileMode = getFileModeValue(u, "filemode", 0) + fso.DirectoryMode = getFileModeValue(u, "dirmode", 0) + if s := u.Query().Get("shards"); s != "" { + parts := strings.Split(s, ".") + shards := make([]int, len(parts)) + for i, p := range parts { + var err error + shards[i], err = strconv.Atoi(p) + if err != nil { + return err + } + } + fso.DirectoryShards = shards + } + return nil +} + +// ToURL converts the FSStorageOptions to URL. +func (fso *FSStorageOptions) ToURL() *url.URL { + u := &url.URL{} + u.Scheme = "file" + if fso.Path[0] == '/' { + u.Path = fso.Path + } else { + u.Opaque = fso.Path + } + q := u.Query() + if fso.FileUID != nil { + q.Add("uid", strconv.Itoa(*fso.FileUID)) + } + if fso.FileGID != nil { + q.Add("gid", strconv.Itoa(*fso.FileGID)) + } + if fso.FileMode != 0 { + q.Add("filemode", strconv.FormatUint(uint64(fso.FileMode), 8)) + } + if fso.DirectoryMode != 0 { + q.Add("dirmode", strconv.FormatUint(uint64(fso.DirectoryMode), 8)) + } + if fso.DirectoryShards != nil { + shardsString := "" + for i, s := range fso.DirectoryShards { + if i > 0 { + shardsString += "." + } + shardsString += fmt.Sprintf("%v", s) + } + q.Add("shards", shardsString) + } + u.RawQuery = q.Encode() + return u +} + func (fs *fsStorage) BlockExists(blockID string) (bool, error) { _, path := fs.getShardedPathAndFilePath(blockID) _, err := os.Stat(path) @@ -136,12 +226,12 @@ func (fs *fsStorage) PutBlock(blockID string, data io.ReadCloser, options PutOpt // Open temporary file, create dir if required. tempFile := fmt.Sprintf("%s.tmp.%d", path, rand.Int()) flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL - f, err := os.OpenFile(tempFile, flags, fs.FileMode) + f, err := os.OpenFile(tempFile, flags, fs.fileMode()) if os.IsNotExist(err) { - if err = os.MkdirAll(shardPath, fs.DirectoryMode); err != nil { + if err = os.MkdirAll(shardPath, fs.dirMode()); err != nil { return fmt.Errorf("cannot create directory: %v", err) } - f, err = os.OpenFile(tempFile, flags, fs.FileMode) + f, err = os.OpenFile(tempFile, flags, fs.fileMode()) } if err != nil { @@ -185,7 +275,7 @@ func (fs *fsStorage) getShardDirectory(blockID string) (string, string) { if len(blockIDString) < 20 { return shardPath, blockID } - for _, size := range fs.DirectoryShards { + for _, size := range fs.shards() { shardPath = filepath.Join(shardPath, blockIDString[0:size]) blockIDString = blockIDString[size:] } @@ -235,26 +325,14 @@ func NewFSStorage(options *FSStorageOptions) (Storage, error) { FSStorageOptions: *options, } - if r.DirectoryShards == nil { - r.DirectoryShards = fsDefaultShards - } - - if r.DirectoryMode == 0 { - r.DirectoryMode = fsDefaultDirMode - } - - if r.FileMode == 0 { - r.FileMode = fsDefaultFileMode - } - return r, nil } func init() { AddSupportedStorage( fsStorageType, - func() interface{} { return &FSStorageOptions{} }, - func(cfg interface{}) (Storage, error) { - return NewFSStorage(cfg.(*FSStorageOptions)) + func() StorageOptions { return &FSStorageOptions{} }, + func(o StorageOptions) (Storage, error) { + return NewFSStorage(o.(*FSStorageOptions)) }) } diff --git a/blob/gcs.go b/blob/gcs.go index eda871d79..b4f3536c7 100644 --- a/blob/gcs.go +++ b/blob/gcs.go @@ -9,6 +9,7 @@ import ( "log" "net/http" "net/http/httptest" + "net/url" "os" "os/exec" "time" @@ -53,6 +54,22 @@ type GCSStorageOptions struct { IgnoreDefaultCredentials bool `json:"ignoreDefaultCredentials"` } +func (gso *GCSStorageOptions) ParseURL(u *url.URL) error { + if u.Scheme != "gcs" { + return fmt.Errorf("invalid scheme, expected 'file'") + } + + gso.BucketName = u.Host + return nil +} + +func (gso *GCSStorageOptions) ToURL() *url.URL { + u := &url.URL{} + u.Scheme = "gcs" + u.Host = gso.BucketName + return u +} + type gcsStorage struct { GCSStorageOptions objectsService *gcsclient.ObjectsService @@ -375,10 +392,10 @@ func authPrompt(url string, state string) (authenticationCode string, err error) func init() { AddSupportedStorage( gcsStorageType, - func() interface{} { + func() StorageOptions { return &GCSStorageOptions{} }, - func(cfg interface{}) (Storage, error) { - return NewGCSStorage(cfg.(*GCSStorageOptions)) + func(o StorageOptions) (Storage, error) { + return NewGCSStorage(o.(*GCSStorageOptions)) }) } diff --git a/blob/registry.go b/blob/registry.go index 4c9d153f6..24ead6413 100644 --- a/blob/registry.go +++ b/blob/registry.go @@ -1,27 +1,31 @@ package blob -import "fmt" +import ( + "fmt" + "net/url" +) var ( - factories = map[string]storageFactory{} + factories = map[string]*storageFactory{} ) // StorageFactory allows creation of repositories in a generic way. type storageFactory struct { - defaultConfigFunc func() interface{} - createStorageFunc func(interface{}) (Storage, error) + defaultConfigFunc func() StorageOptions + createStorageFunc func(StorageOptions) (Storage, error) } // AddSupportedStorage registers factory function to create storage with a given type name. func AddSupportedStorage( - storageType string, - defaultConfigFunc func() interface{}, - createStorageFunc func(interface{}) (Storage, error)) { + urlScheme string, + defaultConfigFunc func() StorageOptions, + createStorageFunc func(StorageOptions) (Storage, error)) { - factories[storageType] = storageFactory{ + f := &storageFactory{ defaultConfigFunc: defaultConfigFunc, createStorageFunc: createStorageFunc, } + factories[urlScheme] = f } // NewStorage creates new storage based on StorageConfiguration. @@ -33,3 +37,17 @@ func NewStorage(cfg StorageConfiguration) (Storage, error) { return nil, fmt.Errorf("unknown storage type: %s", cfg.Type) } + +// NewStorageFromURL creates new storage based on a URL. +// The storage type must be previously registered using AddSupportedStorage. +func NewStorageFromURL(u *url.URL) (Storage, error) { + if factory, ok := factories[u.Scheme]; ok { + o := factory.defaultConfigFunc() + if err := o.ParseURL(u); err != nil { + return nil, err + } + + return factory.createStorageFunc(o) + } + return nil, fmt.Errorf("unknown storage type: %s", u.Scheme) +} diff --git a/blob/storage.go b/blob/storage.go index d08510193..a42b071bd 100644 --- a/blob/storage.go +++ b/blob/storage.go @@ -2,6 +2,7 @@ package blob import ( "io" + "net/url" "time" ) @@ -31,3 +32,8 @@ type BlockMetadata struct { TimeStamp time.Time Error error } + +type StorageOptions interface { + ParseURL(u *url.URL) error + ToURL() *url.URL +} diff --git a/blob/storage_test.go b/blob/storage_test.go index 5a54dd495..611db86cc 100644 --- a/blob/storage_test.go +++ b/blob/storage_test.go @@ -43,13 +43,59 @@ func TestFileStorage(t *testing.T) { Path: path, DirectoryShards: shardSpec, }) + if r == nil || err != nil { t.Errorf("unexpected result: %v %v", r, err) } + + verifyStorageConfiguration(t, r) verifyStorage(t, r) } } +func TestFileStorageOptions(t *testing.T) { + cases := []struct { + o FSStorageOptions + url string + }{ + {FSStorageOptions{Path: "/blah"}, "file:///blah"}, + {FSStorageOptions{Path: "/blah", FileMode: 0123}, "file:///blah?filemode=123"}, + {FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "file:///blah?dirmode=123"}, + {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "file:///blah?dirmode=123&filemode=432"}, + {FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "file:///blah?dirmode=123&filemode=432&shards=1.2.3"}, + {FSStorageOptions{Path: "c:\\winpath"}, "file:c:\\winpath"}, + } + + for i, c := range cases { + u := c.o.ToURL().String() + if u != c.url { + t.Errorf("case #%v, expected url: %v, got %v", i, c.url, u) + } + + o2 := FSStorageOptions{} + err := o2.ParseURL(c.o.ToURL()) + if err != nil { + t.Errorf("URL does not round-trip: %v", err) + } + if o2.ToURL().String() != c.o.ToURL().String() { + t.Errorf("URL does not round-trip: %v vs %v", o2.ToURL().String(), c.o.ToURL().String()) + } + } +} + +func verifyStorageConfiguration(t *testing.T, r Storage) { + r2, err := NewStorageFromURL(r.Configuration().Config.ToURL()) + if err != nil { + t.Errorf("cannot create storage from URL: %v", err) + } else { + url1 := r.Configuration().Config.ToURL().String() + url2 := r2.Configuration().Config.ToURL().String() + if url1 != url2 { + t.Errorf("URL does not round-trip: %v vs %v", url1, url2) + } + } +} + func verifyStorage(t *testing.T, r Storage) { blocks := []struct { blk string diff --git a/blob/urlutils.go b/blob/urlutils.go new file mode 100644 index 000000000..3e7eafcf9 --- /dev/null +++ b/blob/urlutils.go @@ -0,0 +1,36 @@ +package blob + +import ( + "net/url" + "os" + "strconv" +) + +func getIntPtrValue(u *url.URL, name string, base int) *int { + if value := u.Query().Get(name); value != "" { + if int64Val, err := strconv.ParseInt(value, base, 32); err == nil { + intVal := int(int64Val) + return &intVal + } + } + + return nil +} + +func getFileModeValue(u *url.URL, name string, def os.FileMode) os.FileMode { + if value := u.Query().Get(name); value != "" { + if uint32Val, err := strconv.ParseUint(value, 8, 32); err == nil { + return os.FileMode(uint32Val) + } + } + + return def +} + +func getStringValue(u *url.URL, name string, def string) string { + if value := u.Query().Get(name); value != "" { + return value + } + + return def +} diff --git a/cmd/kopia/command_init.go b/cmd/kopia/command_init.go index 35dc2b2de..420b3f91d 100644 --- a/cmd/kopia/command_init.go +++ b/cmd/kopia/command_init.go @@ -7,13 +7,11 @@ import ( "os" "strings" - "github.com/kopia/kopia/auth" - - "github.com/kopia/kopia/config" - "github.com/kopia/kopia/blob" "github.com/kopia/kopia/cas" + "github.com/kopia/kopia/config" "github.com/kopia/kopia/session" + "github.com/kopia/kopia/vault" ) var ( @@ -30,7 +28,7 @@ var ( ) func runInitCommandForRepository(s blob.Storage, defaultSalt string) error { - var creds auth.Credentials + var creds vault.Credentials sess, err := session.New(s, creds) if err != nil { diff --git a/cmd/kopia/config.go b/cmd/kopia/config.go index aac1eec9d..d3c89eaa4 100644 --- a/cmd/kopia/config.go +++ b/cmd/kopia/config.go @@ -6,10 +6,10 @@ import ( "os/user" "path/filepath" - "github.com/kopia/kopia/auth" "github.com/kopia/kopia/blob" "github.com/kopia/kopia/config" "github.com/kopia/kopia/session" + "github.com/kopia/kopia/vault" ) var ( @@ -88,7 +88,7 @@ func openSession() (session.Session, error) { storage = blob.NewLoggingWrapper(storage) } - var creds auth.Credentials + var creds vault.Credentials return session.New(storage, creds) } diff --git a/cmd/playground/main.go b/cmd/playground/main.go index d3ce95ea7..4a68bd4f1 100644 --- a/cmd/playground/main.go +++ b/cmd/playground/main.go @@ -205,8 +205,8 @@ func readCached(repo cas.Repository, manifestOID cas.ObjectID) { } func main() { - u, err := url.Parse("local://c:/ala") - log.Printf("u: %v o: %v", u.Path, u.Opaque) + u, err := url.Parse("file:d:\\path") + log.Printf("path: %v opaque: %v scheme:%v", u.Path, u.Opaque, u.Scheme) //u.Query().Get("a") os.Exit(0) for i := 0; i < 10; i++ { diff --git a/session/session.go b/session/session.go index de9b096ad..83480a66e 100644 --- a/session/session.go +++ b/session/session.go @@ -8,9 +8,9 @@ import ( "io" "io/ioutil" - "github.com/kopia/kopia/auth" "github.com/kopia/kopia/blob" "github.com/kopia/kopia/cas" + "github.com/kopia/kopia/vault" ) // ErrConfigNotFound indicates that configuration is not found. @@ -25,7 +25,7 @@ type Session interface { type session struct { storage blob.Storage - creds auth.Credentials + creds vault.Credentials format cas.Format } @@ -98,7 +98,7 @@ func (s *session) OpenRepository() (cas.Repository, error) { return cas.NewRepository(s.storage, format) } -func New(storage blob.Storage, creds auth.Credentials) (Session, error) { +func New(storage blob.Storage, creds vault.Credentials) (Session, error) { sess := &session{ storage: storage, creds: creds, diff --git a/session/session_test.go b/session/session_test.go index 1f9c74bd4..d4ec71a65 100644 --- a/session/session_test.go +++ b/session/session_test.go @@ -19,7 +19,7 @@ func TestA(t *testing.T) { // cfg := LoadConfig("kopia.config") sc := blob.StorageConfiguration{ - Type: "fs", + Type: "file", Config: &blob.FSStorageOptions{ Path: tmpDir, }, diff --git a/auth/credentials.go b/vault/credentials.go similarity index 99% rename from auth/credentials.go rename to vault/credentials.go index 5f7564406..ac3b83386 100644 --- a/auth/credentials.go +++ b/vault/credentials.go @@ -1,4 +1,4 @@ -package auth +package vault import ( "crypto/sha256" diff --git a/auth/credentials_test.go b/vault/credentials_test.go similarity index 98% rename from auth/credentials_test.go rename to vault/credentials_test.go index 5bd6e807b..78c2dfbee 100644 --- a/auth/credentials_test.go +++ b/vault/credentials_test.go @@ -1,4 +1,4 @@ -package auth +package vault import ( "bytes" diff --git a/vault/doc.go b/vault/doc.go new file mode 100644 index 000000000..f7e34c8a3 --- /dev/null +++ b/vault/doc.go @@ -0,0 +1 @@ +package vault diff --git a/auth/keys.go b/vault/keys.go similarity index 98% rename from auth/keys.go rename to vault/keys.go index 524b3d94e..a6e5dd6c8 100644 --- a/auth/keys.go +++ b/vault/keys.go @@ -1,4 +1,4 @@ -package auth +package vault import ( "errors" diff --git a/vault/vault.go b/vault/vault.go new file mode 100644 index 000000000..32153048f --- /dev/null +++ b/vault/vault.go @@ -0,0 +1,42 @@ +package vault + +import ( + "net/url" + + "github.com/kopia/kopia/blob" + "github.com/kopia/kopia/cas" +) + +type Vault interface { +} + +type vault struct { + storage blob.Storage + repo cas.Repository +} + +func Open(vaultPath string, creds Credentials) (Vault, error) { + var v vault + var err error + + v.storage, err = openStorage(vaultPath) + if err != nil { + return nil, err + } + + return &v, nil +} + +func Create(vaultPath string, repositoryPath string, creds Credentials) (Vault, error) { + var v vault + return &v, nil +} + +func openStorage(vaultPath string) (blob.Storage, error) { + u, err := url.Parse(vaultPath) + if err != nil { + return nil, err + } + + return blob.NewStorageFromURL(u) +}