more repofs.Upload() API tweaks

cmd/kopia/command_backup.go

@@ -14,7 +14,6 @@ import (
 	"path/filepath"
 	"runtime"
 	"strings"
-	"time"
 
 	"github.com/kopia/kopia/fs/repofs"
 	"github.com/kopia/kopia/repo"
@@ -58,20 +57,17 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			return fmt.Errorf("invalid source: '%s': %s", backupDirectory, err)
 		}
 
-		manifest := repofs.Snapshot{
-			StartTime: time.Now(),
-			Source:    filepath.Clean(dir),
-
-			HostName:    getBackupHostName(),
-			UserName:    getBackupUser(),
-			Description: *backupDescription,
+		sourceInfo := repofs.SnapshotSourceInfo{
+			Path:     filepath.Clean(dir),
+			Host:     getBackupHostName(),
+			UserName: getBackupUser(),
 		}
 
-		if len(manifest.Description) > backupMaxDescriptionLength {
+		if len(*backupDescription) > backupMaxDescriptionLength {
 			return fmt.Errorf("description too long")
 		}
 
-		previous, err := conn.Vault.List("B" + manifest.SourceID() + ".")
+		previous, err := conn.Vault.List("B" + sourceInfo.HashString() + ".")
 		if err != nil {
 			return fmt.Errorf("error listing previous backups")
 		}
@@ -82,12 +78,13 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 			oldManifest, err = loadBackupManifest(conn.Vault, previous[0])
 		}
 
-		localEntry := mustGetLocalFSEntry(manifest.Source)
+		localEntry := mustGetLocalFSEntry(sourceInfo.Path)
 		if err != nil {
 			return err
 		}
 
-		if err := repofs.Upload(localEntry, conn.Repository, &manifest, oldManifest); err != nil {
+		manifest, err := repofs.Upload(conn.Repository, localEntry, &sourceInfo, oldManifest)
+		if err != nil {
 			return err
 		}
 
@@ -98,10 +95,11 @@ func runBackupCommand(context *kingpin.ParseContext) error {
 
 		uniqueID := make([]byte, 8)
 		rand.Read(uniqueID)
-		fileID := fmt.Sprintf("B%v.%08x.%x", manifest.SourceID(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID)
+		fileID := fmt.Sprintf("B%v.%08x.%x", sourceInfo.HashString(), math.MaxInt64-manifest.StartTime.UnixNano(), uniqueID)
 		manifest.Handle = handleID
+		manifest.Description = *backupDescription
 
-		err = saveBackupManifest(conn.Vault, fileID, &manifest)
+		err = saveBackupManifest(conn.Vault, fileID, manifest)
 		if err != nil {
 			return fmt.Errorf("cannot save manifest: %v", err)
 		}

cmd/kopia/command_backups.go

@@ -21,13 +21,13 @@ func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
 	var relPath string
 
 	for len(path) > 0 {
-		manifest := repofs.Snapshot{
-			Source:   path,
-			HostName: getBackupHostName(),
+		sourceInfo := repofs.SnapshotSourceInfo{
+			Path:     path,
+			Host:     getBackupHostName(),
 			UserName: getBackupUser(),
 		}
 
-		prefix := manifest.SourceID() + "."
+		prefix := sourceInfo.HashString() + "."
 
 		list, err := vlt.List("B" + prefix)
 		if err != nil {
@@ -44,7 +44,7 @@ func findBackups(vlt *vault.Vault, path string) ([]string, string, error) {
 			relPath = filepath.Base(path)
 		}
 
-		log.Printf("No backups of %v@%v:%v", manifest.UserName, manifest.HostName, manifest.Source)
+		log.Printf("No backups of %v@%v:%v", sourceInfo.UserName, sourceInfo.Host, sourceInfo.Path)
 
 		parent := filepath.Dir(path)
 		if parent == path {
@@ -82,17 +82,13 @@ func runBackupsCommand(context *kingpin.ParseContext) error {
 		return fmt.Errorf("cannot list backups: %v", err)
 	}
 
-	var lastHost string
-	var lastUser string
-	var lastSource string
+	var lastSource repofs.SnapshotSourceInfo
 	var count int
 
 	for _, m := range loadBackupManifests(conn.Vault, previous) {
-		if m.HostName != lastHost || m.UserName != lastUser || m.Source != lastSource {
-			fmt.Printf("%v@%v:%v\n", m.UserName, m.HostName, m.Source)
+		if m.Source != lastSource {
+			fmt.Printf("%v@%v:%v\n", m.Source.UserName, m.Source.Host, m.Source.Path)
 			lastSource = m.Source
-			lastUser = m.UserName
-			lastHost = m.HostName
 			count = 0
 		}
 

fs/repofs/snapshot.go

@@ -11,17 +11,34 @@ import (
 
 var zeroByte = []byte{0}
 
+// SnapshotSourceInfo represents the information about snapshot source.
+type SnapshotSourceInfo struct {
+	Host     string `json:"host"`
+	UserName string `json:"userName"`
+	Path     string `json:"path"`
+}
+
+// HashString generates hash of SnapshotSourceInfo.
+func (r SnapshotSourceInfo) HashString() string {
+	h := sha1.New()
+	io.WriteString(h, r.Host)
+	h.Write(zeroByte)
+	io.WriteString(h, r.UserName)
+	h.Write(zeroByte)
+	io.WriteString(h, r.Path)
+	h.Write(zeroByte)
+	return hex.EncodeToString(h.Sum(nil))
+}
+
 // Snapshot stores information about a single filesystem snapshot.
 type Snapshot struct {
-	StartTime time.Time `json:"startTime"`
-	EndTime   time.Time `json:"endTime"`
+	Source SnapshotSourceInfo `json:"source"`
 
-	HostName    string `json:"host"`
-	UserName    string `json:"userName"`
-	Description string `json:"description"`
+	Description string    `json:"description"`
+	StartTime   time.Time `json:"startTime"`
+	EndTime     time.Time `json:"endTime"`
 
 	Handle       string        `json:"handle"`
-	Source       string        `json:"source"`
 	RootObjectID repo.ObjectID `json:"root"`
 	HashCacheID  repo.ObjectID `json:"hashCache"`
 
@@ -31,16 +48,3 @@ type Snapshot struct {
 
 	Stats *repo.Stats `json:"stats"`
 }
-
-// SourceID generates unique identifier of the backup source, which is a
-// SHA1 hash of the host name, username and source directory.
-func (r Snapshot) SourceID() string {
-	h := sha1.New()
-	io.WriteString(h, r.HostName)
-	h.Write(zeroByte)
-	io.WriteString(h, r.UserName)
-	h.Write(zeroByte)
-	io.WriteString(h, r.Source)
-	h.Write(zeroByte)
-	return hex.EncodeToString(h.Sum(nil))
-}

fs/repofs/upload.go

@@ -176,11 +176,16 @@ func (u *uploader) uploadBundleInternal(b *bundle) (*dirEntry, uint64, error) {
 
 // Upload uploads contents of the specified filesystem entry (file or directory) to the repository and updates given manifest with statistics.
 // Old snapshot manifest, when provided can be used to speed up backups by utilizing hash cache.
-func Upload(source fs.Entry, repository *repo.Repository, s *Snapshot, old *Snapshot) error {
+func Upload(repository *repo.Repository, source fs.Entry, sourceInfo *SnapshotSourceInfo, old *Snapshot) (*Snapshot, error) {
 	u := &uploader{
 		repo: repository,
 	}
-	s.StartTime = time.Now()
+
+	s := &Snapshot{
+		Source:    *sourceInfo,
+		StartTime: time.Now(),
+	}
+
 	var hashCacheID *repo.ObjectID
 
 	if old != nil {
@@ -195,18 +200,18 @@ func Upload(source fs.Entry, repository *repo.Repository, s *Snapshot, old *Snap
 	case fs.File:
 		r, err = u.uploadFile(entry)
 	default:
-		return fmt.Errorf("unsupported source: %v", s.Source)
+		return nil, fmt.Errorf("unsupported source: %v", s.Source)
 	}
 	s.EndTime = time.Now()
 	if err != nil {
-		return err
+		return nil, err
 	}
 	s.RootObjectID = r.ObjectID
 	s.HashCacheID = r.ManifestID
 	stats := u.repo.Stats
 	s.Stats = &stats
 
-	return nil
+	return s, nil
 }
 
 // uploadFile uploads the specified File to the repository.