added url serialization and moved auth to vault

blob/config.go

@@ -4,10 +4,10 @@ import (
 	"encoding/json"
 )
 
-// StorageConfiguration is a JSON-serializable description of Storage and its configuration.
+// StorageConfiguration is a JSON-serializable description of Storage and its options.
 type StorageConfiguration struct {
 	Type   string
-	Config interface{}
+	Config StorageOptions
 }
 
 // UnmarshalJSON parses the JSON-encoded data into StorageConfiguration.

blob/filesystem.go

@@ -5,6 +5,7 @@ import (
 	"io"
 	"io/ioutil"
 	"math/rand"
+	"net/url"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -12,8 +13,7 @@ import (
 )
 
 const (
-	fsStorageType = "fs"
-
+	fsStorageType        = "file"
 	fsStorageChunkSuffix = ".f"
 )
 
@@ -40,6 +40,96 @@ type FSStorageOptions struct {
 	FileGID *int `json:"gid,omitempty"`
 }
 
+func (fso *FSStorageOptions) fileMode() os.FileMode {
+	if fso.FileMode == 0 {
+		return fsDefaultFileMode
+	}
+
+	return fso.FileMode
+}
+
+func (fso *FSStorageOptions) dirMode() os.FileMode {
+	if fso.DirectoryMode == 0 {
+		return fsDefaultDirMode
+	}
+
+	return fso.DirectoryMode
+}
+
+func (fso *FSStorageOptions) shards() []int {
+	if fso.DirectoryShards == nil {
+		return fsDefaultShards
+	}
+
+	return fso.DirectoryShards
+}
+
+// ParseURL parses the given URL into FSStorageOptions.
+func (fso *FSStorageOptions) ParseURL(u *url.URL) error {
+	if u.Scheme != "file" {
+		return fmt.Errorf("invalid scheme, expected 'file'")
+	}
+
+	if u.Opaque != "" {
+		fso.Path = u.Opaque
+	} else {
+		fso.Path = u.Path
+	}
+	fso.FileUID = getIntPtrValue(u, "uid", 10)
+	fso.FileGID = getIntPtrValue(u, "gid", 10)
+	fso.FileMode = getFileModeValue(u, "filemode", 0)
+	fso.DirectoryMode = getFileModeValue(u, "dirmode", 0)
+	if s := u.Query().Get("shards"); s != "" {
+		parts := strings.Split(s, ".")
+		shards := make([]int, len(parts))
+		for i, p := range parts {
+			var err error
+			shards[i], err = strconv.Atoi(p)
+			if err != nil {
+				return err
+			}
+		}
+		fso.DirectoryShards = shards
+	}
+	return nil
+}
+
+// ToURL converts the FSStorageOptions to URL.
+func (fso *FSStorageOptions) ToURL() *url.URL {
+	u := &url.URL{}
+	u.Scheme = "file"
+	if fso.Path[0] == '/' {
+		u.Path = fso.Path
+	} else {
+		u.Opaque = fso.Path
+	}
+	q := u.Query()
+	if fso.FileUID != nil {
+		q.Add("uid", strconv.Itoa(*fso.FileUID))
+	}
+	if fso.FileGID != nil {
+		q.Add("gid", strconv.Itoa(*fso.FileGID))
+	}
+	if fso.FileMode != 0 {
+		q.Add("filemode", strconv.FormatUint(uint64(fso.FileMode), 8))
+	}
+	if fso.DirectoryMode != 0 {
+		q.Add("dirmode", strconv.FormatUint(uint64(fso.DirectoryMode), 8))
+	}
+	if fso.DirectoryShards != nil {
+		shardsString := ""
+		for i, s := range fso.DirectoryShards {
+			if i > 0 {
+				shardsString += "."
+			}
+			shardsString += fmt.Sprintf("%v", s)
+		}
+		q.Add("shards", shardsString)
+	}
+	u.RawQuery = q.Encode()
+	return u
+}
+
 func (fs *fsStorage) BlockExists(blockID string) (bool, error) {
 	_, path := fs.getShardedPathAndFilePath(blockID)
 	_, err := os.Stat(path)
@@ -136,12 +226,12 @@ func (fs *fsStorage) PutBlock(blockID string, data io.ReadCloser, options PutOpt
 	// Open temporary file, create dir if required.
 	tempFile := fmt.Sprintf("%s.tmp.%d", path, rand.Int())
 	flags := os.O_CREATE | os.O_WRONLY | os.O_EXCL
-	f, err := os.OpenFile(tempFile, flags, fs.FileMode)
+	f, err := os.OpenFile(tempFile, flags, fs.fileMode())
 	if os.IsNotExist(err) {
-		if err = os.MkdirAll(shardPath, fs.DirectoryMode); err != nil {
+		if err = os.MkdirAll(shardPath, fs.dirMode()); err != nil {
 			return fmt.Errorf("cannot create directory: %v", err)
 		}
-		f, err = os.OpenFile(tempFile, flags, fs.FileMode)
+		f, err = os.OpenFile(tempFile, flags, fs.fileMode())
 	}
 
 	if err != nil {
@@ -185,7 +275,7 @@ func (fs *fsStorage) getShardDirectory(blockID string) (string, string) {
 	if len(blockIDString) < 20 {
 		return shardPath, blockID
 	}
-	for _, size := range fs.DirectoryShards {
+	for _, size := range fs.shards() {
 		shardPath = filepath.Join(shardPath, blockIDString[0:size])
 		blockIDString = blockIDString[size:]
 	}
@@ -235,26 +325,14 @@ func NewFSStorage(options *FSStorageOptions) (Storage, error) {
 		FSStorageOptions: *options,
 	}
 
-	if r.DirectoryShards == nil {
-		r.DirectoryShards = fsDefaultShards
-	}
-
-	if r.DirectoryMode == 0 {
-		r.DirectoryMode = fsDefaultDirMode
-	}
-
-	if r.FileMode == 0 {
-		r.FileMode = fsDefaultFileMode
-	}
-
 	return r, nil
 }
 
 func init() {
 	AddSupportedStorage(
 		fsStorageType,
-		func() interface{} { return &FSStorageOptions{} },
-		func(cfg interface{}) (Storage, error) {
-			return NewFSStorage(cfg.(*FSStorageOptions))
+		func() StorageOptions { return &FSStorageOptions{} },
+		func(o StorageOptions) (Storage, error) {
+			return NewFSStorage(o.(*FSStorageOptions))
 		})
 }

blob/gcs.go

@@ -9,6 +9,7 @@ import (
 	"log"
 	"net/http"
 	"net/http/httptest"
+	"net/url"
 	"os"
 	"os/exec"
 	"time"
@@ -53,6 +54,22 @@ type GCSStorageOptions struct {
 	IgnoreDefaultCredentials bool `json:"ignoreDefaultCredentials"`
 }
 
+func (gso *GCSStorageOptions) ParseURL(u *url.URL) error {
+	if u.Scheme != "gcs" {
+		return fmt.Errorf("invalid scheme, expected 'file'")
+	}
+
+	gso.BucketName = u.Host
+	return nil
+}
+
+func (gso *GCSStorageOptions) ToURL() *url.URL {
+	u := &url.URL{}
+	u.Scheme = "gcs"
+	u.Host = gso.BucketName
+	return u
+}
+
 type gcsStorage struct {
 	GCSStorageOptions
 	objectsService *gcsclient.ObjectsService
@@ -375,10 +392,10 @@ func authPrompt(url string, state string) (authenticationCode string, err error)
 func init() {
 	AddSupportedStorage(
 		gcsStorageType,
-		func() interface{} {
+		func() StorageOptions {
 			return &GCSStorageOptions{}
 		},
-		func(cfg interface{}) (Storage, error) {
-			return NewGCSStorage(cfg.(*GCSStorageOptions))
+		func(o StorageOptions) (Storage, error) {
+			return NewGCSStorage(o.(*GCSStorageOptions))
 		})
 }

blob/registry.go

@@ -1,27 +1,31 @@
 package blob
 
-import "fmt"
+import (
+	"fmt"
+	"net/url"
+)
 
 var (
-	factories = map[string]storageFactory{}
+	factories = map[string]*storageFactory{}
 )
 
 // StorageFactory allows creation of repositories in a generic way.
 type storageFactory struct {
-	defaultConfigFunc func() interface{}
-	createStorageFunc func(interface{}) (Storage, error)
+	defaultConfigFunc func() StorageOptions
+	createStorageFunc func(StorageOptions) (Storage, error)
 }
 
 // AddSupportedStorage registers factory function to create storage with a given type name.
 func AddSupportedStorage(
-	storageType string,
-	defaultConfigFunc func() interface{},
-	createStorageFunc func(interface{}) (Storage, error)) {
+	urlScheme string,
+	defaultConfigFunc func() StorageOptions,
+	createStorageFunc func(StorageOptions) (Storage, error)) {
 
-	factories[storageType] = storageFactory{
+	f := &storageFactory{
 		defaultConfigFunc: defaultConfigFunc,
 		createStorageFunc: createStorageFunc,
 	}
+	factories[urlScheme] = f
 }
 
 // NewStorage creates new storage based on StorageConfiguration.
@@ -33,3 +37,17 @@ func NewStorage(cfg StorageConfiguration) (Storage, error) {
 
 	return nil, fmt.Errorf("unknown storage type: %s", cfg.Type)
 }
+
+// NewStorageFromURL creates new storage based on a URL.
+// The storage type must be previously registered using AddSupportedStorage.
+func NewStorageFromURL(u *url.URL) (Storage, error) {
+	if factory, ok := factories[u.Scheme]; ok {
+		o := factory.defaultConfigFunc()
+		if err := o.ParseURL(u); err != nil {
+			return nil, err
+		}
+
+		return factory.createStorageFunc(o)
+	}
+	return nil, fmt.Errorf("unknown storage type: %s", u.Scheme)
+}

blob/storage.go

@@ -2,6 +2,7 @@ package blob
 
 import (
 	"io"
+	"net/url"
 	"time"
 )
 
@@ -31,3 +32,8 @@ type BlockMetadata struct {
 	TimeStamp time.Time
 	Error     error
 }
+
+type StorageOptions interface {
+	ParseURL(u *url.URL) error
+	ToURL() *url.URL
+}

blob/storage_test.go

@@ -43,13 +43,59 @@ func TestFileStorage(t *testing.T) {
 			Path:            path,
 			DirectoryShards: shardSpec,
 		})
+
 		if r == nil || err != nil {
 			t.Errorf("unexpected result: %v %v", r, err)
 		}
+
+		verifyStorageConfiguration(t, r)
 		verifyStorage(t, r)
 	}
 }
 
+func TestFileStorageOptions(t *testing.T) {
+	cases := []struct {
+		o   FSStorageOptions
+		url string
+	}{
+		{FSStorageOptions{Path: "/blah"}, "file:///blah"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0123}, "file:///blah?filemode=123"},
+		{FSStorageOptions{Path: "/blah", DirectoryMode: 0123}, "file:///blah?dirmode=123"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123}, "file:///blah?dirmode=123&filemode=432"},
+		{FSStorageOptions{Path: "/blah", FileMode: 0432, DirectoryMode: 0123, DirectoryShards: []int{1, 2, 3}}, "file:///blah?dirmode=123&filemode=432&shards=1.2.3"},
+		{FSStorageOptions{Path: "c:\\winpath"}, "file:c:\\winpath"},
+	}
+
+	for i, c := range cases {
+		u := c.o.ToURL().String()
+		if u != c.url {
+			t.Errorf("case #%v, expected url: %v, got %v", i, c.url, u)
+		}
+
+		o2 := FSStorageOptions{}
+		err := o2.ParseURL(c.o.ToURL())
+		if err != nil {
+			t.Errorf("URL does not round-trip: %v", err)
+		}
+		if o2.ToURL().String() != c.o.ToURL().String() {
+			t.Errorf("URL does not round-trip: %v vs %v", o2.ToURL().String(), c.o.ToURL().String())
+		}
+	}
+}
+
+func verifyStorageConfiguration(t *testing.T, r Storage) {
+	r2, err := NewStorageFromURL(r.Configuration().Config.ToURL())
+	if err != nil {
+		t.Errorf("cannot create storage from URL: %v", err)
+	} else {
+		url1 := r.Configuration().Config.ToURL().String()
+		url2 := r2.Configuration().Config.ToURL().String()
+		if url1 != url2 {
+			t.Errorf("URL does not round-trip: %v vs %v", url1, url2)
+		}
+	}
+}
+
 func verifyStorage(t *testing.T, r Storage) {
 	blocks := []struct {
 		blk      string

blob/urlutils.go

@@ -0,0 +1,36 @@
+package blob
+
+import (
+	"net/url"
+	"os"
+	"strconv"
+)
+
+func getIntPtrValue(u *url.URL, name string, base int) *int {
+	if value := u.Query().Get(name); value != "" {
+		if int64Val, err := strconv.ParseInt(value, base, 32); err == nil {
+			intVal := int(int64Val)
+			return &intVal
+		}
+	}
+
+	return nil
+}
+
+func getFileModeValue(u *url.URL, name string, def os.FileMode) os.FileMode {
+	if value := u.Query().Get(name); value != "" {
+		if uint32Val, err := strconv.ParseUint(value, 8, 32); err == nil {
+			return os.FileMode(uint32Val)
+		}
+	}
+
+	return def
+}
+
+func getStringValue(u *url.URL, name string, def string) string {
+	if value := u.Query().Get(name); value != "" {
+		return value
+	}
+
+	return def
+}

cmd/kopia/command_init.go

@@ -7,13 +7,11 @@ import (
 	"os"
 	"strings"
 
-	"github.com/kopia/kopia/auth"
-
-	"github.com/kopia/kopia/config"
-
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/cas"
+	"github.com/kopia/kopia/config"
 	"github.com/kopia/kopia/session"
+	"github.com/kopia/kopia/vault"
 )
 
 var (
@@ -30,7 +28,7 @@ var (
 )
 
 func runInitCommandForRepository(s blob.Storage, defaultSalt string) error {
-	var creds auth.Credentials
+	var creds vault.Credentials
 
 	sess, err := session.New(s, creds)
 	if err != nil {

cmd/kopia/config.go

@@ -6,10 +6,10 @@ import (
 	"os/user"
 	"path/filepath"
 
-	"github.com/kopia/kopia/auth"
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/config"
 	"github.com/kopia/kopia/session"
+	"github.com/kopia/kopia/vault"
 )
 
 var (
@@ -88,7 +88,7 @@ func openSession() (session.Session, error) {
 		storage = blob.NewLoggingWrapper(storage)
 	}
 
-	var creds auth.Credentials
+	var creds vault.Credentials
 
 	return session.New(storage, creds)
 }

cmd/playground/main.go

@@ -205,8 +205,8 @@ func readCached(repo cas.Repository, manifestOID cas.ObjectID) {
 }
 
 func main() {
-	u, err := url.Parse("local://c:/ala")
-	log.Printf("u: %v o: %v", u.Path, u.Opaque)
+	u, err := url.Parse("file:d:\\path")
+	log.Printf("path: %v opaque: %v scheme:%v", u.Path, u.Opaque, u.Scheme)
 	//u.Query().Get("a")
 	os.Exit(0)
 	for i := 0; i < 10; i++ {

session/session.go

@@ -8,9 +8,9 @@ import (
 	"io"
 	"io/ioutil"
 
-	"github.com/kopia/kopia/auth"
 	"github.com/kopia/kopia/blob"
 	"github.com/kopia/kopia/cas"
+	"github.com/kopia/kopia/vault"
 )
 
 // ErrConfigNotFound indicates that configuration is not found.
@@ -25,7 +25,7 @@ type Session interface {
 
 type session struct {
 	storage blob.Storage
-	creds   auth.Credentials
+	creds   vault.Credentials
 	format  cas.Format
 }
 
@@ -98,7 +98,7 @@ func (s *session) OpenRepository() (cas.Repository, error) {
 	return cas.NewRepository(s.storage, format)
 }
 
-func New(storage blob.Storage, creds auth.Credentials) (Session, error) {
+func New(storage blob.Storage, creds vault.Credentials) (Session, error) {
 	sess := &session{
 		storage: storage,
 		creds:   creds,

session/session_test.go

@@ -19,7 +19,7 @@ func TestA(t *testing.T) {
 
 	// cfg := LoadConfig("kopia.config")
 	sc := blob.StorageConfiguration{
-		Type: "fs",
+		Type: "file",
 		Config: &blob.FSStorageOptions{
 			Path: tmpDir,
 		},

vault/credentials.go

@@ -1,4 +1,4 @@
-package auth
+package vault
 
 import (
 	"crypto/sha256"

vault/credentials_test.go

@@ -1,4 +1,4 @@
-package auth
+package vault
 
 import (
 	"bytes"

vault/doc.go

@@ -0,0 +1 @@
+package vault

vault/keys.go

@@ -1,4 +1,4 @@
-package auth
+package vault
 
 import (
 	"errors"

vault/vault.go

@@ -0,0 +1,42 @@
+package vault
+
+import (
+	"net/url"
+
+	"github.com/kopia/kopia/blob"
+	"github.com/kopia/kopia/cas"
+)
+
+type Vault interface {
+}
+
+type vault struct {
+	storage blob.Storage
+	repo    cas.Repository
+}
+
+func Open(vaultPath string, creds Credentials) (Vault, error) {
+	var v vault
+	var err error
+
+	v.storage, err = openStorage(vaultPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return &v, nil
+}
+
+func Create(vaultPath string, repositoryPath string, creds Credentials) (Vault, error) {
+	var v vault
+	return &v, nil
+}
+
+func openStorage(vaultPath string) (blob.Storage, error) {
+	u, err := url.Parse(vaultPath)
+	if err != nil {
+		return nil, err
+	}
+
+	return blob.NewStorageFromURL(u)
+}