kopia/cli/command_snapshot_create.go

package cli

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/rs/zerolog/log"

	"github.com/kopia/kopia/object"
	"github.com/kopia/kopia/repo"
	"github.com/kopia/kopia/snapshot"

	kingpin "gopkg.in/alecthomas/kingpin.v2"
)

const (
	maxSnapshotDescriptionLength = 1024
)

var (
	snapshotCreateCommand = snapshotCommands.Command("create", "Creates a snapshot of local directory or file.").Default()

	snapshotCreateSources                 = snapshotCreateCommand.Arg("source", "Files or directories to create snapshot(s) of.").ExistingFilesOrDirs()
	snapshotCreateAll                     = snapshotCreateCommand.Flag("all", "Create snapshots for files or directories previously backed up by this user on this computer").Bool()
	snapshotCreateCheckpointUploadLimitMB = snapshotCreateCommand.Flag("upload-limit-mb", "Stop the backup process after the specified amount of data (in MB) has been uploaded.").PlaceHolder("MB").Default("0").Int64()
	snapshotCreateDescription             = snapshotCreateCommand.Flag("description", "Free-form snapshot description.").String()
	snapshotCreateForceHash               = snapshotCreateCommand.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Int()
	snapshotCreateHashCacheMinAge         = snapshotCreateCommand.Flag("hash-cache-min-age", "Do not hash-cache files below certain age").Default("1h").Duration()
	snapshotCreateWriteBack               = snapshotCreateCommand.Flag("async-write", "Perform updates asynchronously.").PlaceHolder("N").Default("0").Int()
	snapshotCreateParallelUploads         = snapshotCreateCommand.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").Int()
)

func runBackupCommand(c *kingpin.ParseContext) error {
	rep := mustOpenRepository(&repo.Options{
		ObjectManagerOptions: object.ManagerOptions{
			WriteBack: *snapshotCreateWriteBack,
		},
	})
	defer rep.Close() //nolint: errcheck

	mgr := snapshot.NewManager(rep)
	pmgr := snapshot.NewPolicyManager(rep)

	sources := *snapshotCreateSources
	if *snapshotCreateAll {
		local, err := getLocalBackupPaths(mgr)
		if err != nil {
			return err
		}
		sources = append(sources, local...)
	}

	if len(sources) == 0 {
		return errors.New("no backup sources")
	}

	u := snapshot.NewUploader(rep)
	u.MaxUploadBytes = *snapshotCreateCheckpointUploadLimitMB * 1024 * 1024
	u.ForceHashPercentage = *snapshotCreateForceHash
	u.HashCacheMinAge = *snapshotCreateHashCacheMinAge
	u.ParallelUploads = *snapshotCreateParallelUploads
	onCtrlC(u.Cancel)

	u.Progress = &uploadProgress{}

	if len(*snapshotCreateDescription) > maxSnapshotDescriptionLength {
		return fmt.Errorf("description too long")
	}

	for _, snapshotDir := range sources {
		log.Printf("Backing up %v", snapshotDir)
		dir, err := filepath.Abs(snapshotDir)
		if err != nil {
			return fmt.Errorf("invalid source: '%s': %s", snapshotDir, err)
		}

		sourceInfo := snapshot.SourceInfo{Path: filepath.Clean(dir), Host: getHostName(), UserName: getUserName()}
		log.Info().Str("source", sourceInfo.String()).Msg("snapshotting")
		if err := snapshotSingleSource(rep, mgr, pmgr, u, sourceInfo); err != nil {
			return err
		}
	}

	return nil
}

func snapshotSingleSource(rep *repo.Repository, mgr *snapshot.Manager, pmgr *snapshot.PolicyManager, u *snapshot.Uploader, sourceInfo snapshot.SourceInfo) error {
	t0 := time.Now()
	rep.Blocks.ResetStats()
	policy, err := pmgr.GetEffectivePolicy(sourceInfo)
	if err != nil {
		return fmt.Errorf("unable to get backup policy for source %v: %v", sourceInfo, err)
	}

	localEntry := mustGetLocalFSEntry(sourceInfo.Path)
	if err != nil {
		return err
	}

	u.FilesPolicy = policy.FilesPolicy

	previousManifest, err := findPreviousSnapshotManifest(mgr, sourceInfo)
	if err != nil {
		return err
	}

	log.Debug().Msgf("uploading %v using previous manifest %v", sourceInfo, previousManifest)
	manifest, err := u.Upload(localEntry, sourceInfo, previousManifest)
	if err != nil {
		return err
	}

	manifest.Description = *snapshotCreateDescription

	snapID, err := mgr.SaveSnapshot(manifest)
	if err != nil {
		return fmt.Errorf("cannot save manifest: %v", err)
	}

	log.Info().Str("id", snapID).Str("oid", manifest.RootObjectID.String()).Dur("duration_ms", time.Since(t0)).Msg("uploaded")
	log.Printf("Hash Cache: %v", manifest.HashCacheID.String())

	b, _ := json.MarshalIndent(&manifest, "", "  ")
	log.Printf("%s", string(b))

	return nil
}

func findPreviousSnapshotManifest(mgr *snapshot.Manager, sourceInfo snapshot.SourceInfo) (*snapshot.Manifest, error) {
	previous, err := mgr.ListSnapshots(sourceInfo)
	if err != nil {
		return nil, fmt.Errorf("error listing previous backups: %v", err)
	}

	var previousManifest *snapshot.Manifest
	for _, p := range previous {
		if previousManifest == nil || p.StartTime.After(previousManifest.StartTime) {
			previousManifest = p
		}
	}

	if previousManifest != nil {
		log.Debug().Msgf("found previous manifest for %v with start time %v", sourceInfo, previousManifest.StartTime)
	} else {
		log.Debug().Msgf("no previous manifest for %v", sourceInfo)
	}

	return previousManifest, nil
}

func getLocalBackupPaths(mgr *snapshot.Manager) ([]string, error) {
	h := getHostName()
	u := getUserName()
	log.Printf("Looking for previous backups of '%v@%v'...", u, h)

	sources := mgr.ListSources()

	var result []string

	for _, src := range sources {
		if src.Host == h && src.UserName == u {
			result = append(result, src.Path)
		}
	}

	return result, nil
}

func getUserName() string {
	currentUser, err := user.Current()
	if err != nil {
		log.Warn().Msgf("Cannot determine current user: %s", err)
		return "nobody"
	}

	u := currentUser.Username
	if runtime.GOOS == "windows" {
		if p := strings.Index(u, "\\"); p >= 0 {
			// On Windows ignore domain name.
			u = u[p+1:]
		}
	}

	return u
}

func getHostName() string {
	hostname, err := os.Hostname()
	if err != nil {
		log.Warn().Msgf("Unable to determine hostname: %s", err)
		return "nohost"
	}

	// Normalize hostname.
	hostname = strings.ToLower(strings.Split(hostname, ".")[0])

	return hostname
}

func init() {
	snapshotCreateCommand.Action(runBackupCommand)
}