cli: changed 'kopia snapshot verify --verify-files-percent' to float (#1210)

* cli: changed 'kopia snapshot verify --verify-files-percent' to float

* cli: added 'kopia content verify --download-percent'

cli/command_content_verify.go

@@ -2,6 +2,7 @@
 
 import (
 	"context"
+	"math/rand"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -18,6 +19,7 @@ type commandContentVerify struct {
 	contentVerifyParallel       int
 	contentVerifyFull           bool
 	contentVerifyIncludeDeleted bool
+	contentVerifyPercent        float64
 	progressInterval            time.Duration
 
 	contentRange contentRangeFlags
@@ -29,6 +31,7 @@ func (c *commandContentVerify) setup(svc appServices, parent commandParent) {
 	cmd.Flag("parallel", "Parallelism").Default("16").IntVar(&c.contentVerifyParallel)
 	cmd.Flag("full", "Full verification (including download)").BoolVar(&c.contentVerifyFull)
 	cmd.Flag("include-deleted", "Include deleted contents").BoolVar(&c.contentVerifyIncludeDeleted)
+	cmd.Flag("download-percent", "Download a percentage of files [0.0 .. 100.0]").Float64Var(&c.contentVerifyPercent)
 	cmd.Flag("progress-interval", "Progress output interval").Default("3s").DurationVar(&c.progressInterval)
 	c.contentRange.setup(cmd)
 	cmd.Action(svc.directRepositoryReadAction(c.run))
@@ -56,14 +59,15 @@ func readBlobMap(ctx context.Context, br blob.Reader) (map[blob.ID]blob.Metadata
 
 func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepository) error {
 	blobMap := map[blob.ID]blob.Metadata{}
+	downloadPercent := c.contentVerifyPercent
 
-	if !c.contentVerifyFull {
-		m, err := readBlobMap(ctx, rep.BlobReader())
-		if err != nil {
-			return err
-		}
+	if c.contentVerifyFull {
+		downloadPercent = 100.0
+	}
 
-		blobMap = m
+	blobMap, err := readBlobMap(ctx, rep.BlobReader())
+	if err != nil {
+		return err
 	}
 
 	verifiedCount := new(int32)
@@ -93,12 +97,12 @@ func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepositor
 	throttle := new(timetrack.Throttle)
 	est := timetrack.Start()
 
-	err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{
+	if err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{
 		Range:          c.contentRange.contentIDRange(),
 		Parallel:       c.contentVerifyParallel,
 		IncludeDeleted: c.contentVerifyIncludeDeleted,
 	}, func(ci content.Info) error {
-		if err := c.contentVerify(ctx, rep.ContentReader(), ci, blobMap); err != nil {
+		if err := c.contentVerify(ctx, rep.ContentReader(), ci, blobMap, downloadPercent); err != nil {
 			log(ctx).Errorf("error %v", err)
 			atomic.AddInt32(errorCount, 1)
 		} else {
@@ -124,8 +128,7 @@ func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepositor
 		}
 
 		return nil
-	})
-	if err != nil {
+	}); err != nil {
 		return errors.Wrap(err, "iterate contents")
 	}
 
@@ -160,15 +163,7 @@ func (c *commandContentVerify) getTotalContentCount(ctx context.Context, rep rep
 	atomic.StoreInt32(totalCount, tc)
 }
 
-func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Reader, ci content.Info, blobMap map[blob.ID]blob.Metadata) error {
-	if c.contentVerifyFull {
-		if _, err := r.GetContent(ctx, ci.GetContentID()); err != nil {
-			return errors.Wrapf(err, "content %v is invalid", ci.GetContentID())
-		}
-
-		return nil
-	}
-
+func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Reader, ci content.Info, blobMap map[blob.ID]blob.Metadata, downloadPercent float64) error {
 	bi, ok := blobMap[ci.GetPackBlobID()]
 	if !ok {
 		return errors.Errorf("content %v depends on missing blob %v", ci.GetContentID(), ci.GetPackBlobID())
@@ -178,5 +173,14 @@ func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Read
 		return errors.Errorf("content %v out of bounds of its pack blob %v", ci.GetContentID(), ci.GetPackBlobID())
 	}
 
+	// nolint:gosec
+	if 100*rand.Float64() < downloadPercent {
+		if _, err := r.GetContent(ctx, ci.GetContentID()); err != nil {
+			return errors.Wrapf(err, "content %v is invalid", ci.GetContentID())
+		}
+
+		return nil
+	}
+
 	return nil
 }

cli/command_content_verify_test.go

@@ -22,7 +22,7 @@ func TestContentVerify(t *testing.T) {
 	env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir)
 	env.RunAndExpectSuccess(t, "content", "verify")
 	env.RunAndExpectSuccess(t, "snapshot", "create", dir)
-	env.RunAndExpectSuccess(t, "content", "verify")
+	env.RunAndExpectSuccess(t, "content", "verify", "--download-percent=30")
 
 	// delete one of 'p' blobs.
 	blobIDToDelete := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=p")[0], " ")[0]

cli/command_snapshot_create.go

@@ -52,7 +52,7 @@ func (c *commandSnapshotCreate) setup(svc appServices, parent commandParent) {
 	cmd.Flag("checkpoint-interval", "Frequency for creating periodic checkpoint.").DurationVar(&c.snapshotCreateCheckpointInterval)
 	cmd.Flag("description", "Free-form snapshot description.").StringVar(&c.snapshotCreateDescription)
 	cmd.Flag("fail-fast", "Fail fast when creating snapshot.").Envar("KOPIA_SNAPSHOT_FAIL_FAST").BoolVar(&c.snapshotCreateFailFast)
-	cmd.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Float64Var(&c.snapshotCreateForceHash)
+	cmd.Flag("force-hash", "Force hashing of source files for a given percentage of files [0.0 .. 100.0]").Default("0").Float64Var(&c.snapshotCreateForceHash)
 	cmd.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").IntVar(&c.snapshotCreateParallelUploads)
 	cmd.Flag("start-time", "Override snapshot start timestamp.").StringVar(&c.snapshotCreateStartTime)
 	cmd.Flag("end-time", "Override snapshot end timestamp.").StringVar(&c.snapshotCreateEndTime)

cli/command_snapshot_verify.go

@@ -27,7 +27,7 @@ type commandSnapshotVerify struct {
 	verifyCommandAllSources     bool
 	verifyCommandSources        []string
 	verifyCommandParallel       int
-	verifyCommandFilesPercent   int
+	verifyCommandFilesPercent   float64
 }
 
 func (c *commandSnapshotVerify) setup(svc appServices, parent commandParent) {
@@ -38,7 +38,7 @@ func (c *commandSnapshotVerify) setup(svc appServices, parent commandParent) {
 	cmd.Flag("all-sources", "Verify all snapshots (DEPRECATED)").Hidden().BoolVar(&c.verifyCommandAllSources)
 	cmd.Flag("sources", "Verify the provided sources").StringsVar(&c.verifyCommandSources)
 	cmd.Flag("parallel", "Parallelization").Default("16").IntVar(&c.verifyCommandParallel)
-	cmd.Flag("verify-files-percent", "Randomly verify a percentage of files").Default("0").IntVar(&c.verifyCommandFilesPercent)
+	cmd.Flag("verify-files-percent", "Randomly verify a percentage of files by downloading them [0.0 .. 100.0]").Default("0").Float64Var(&c.verifyCommandFilesPercent)
 	cmd.Action(svc.repositoryReaderAction(c.run))
 }
 
@@ -55,7 +55,7 @@ type verifier struct {
 	errors []error
 
 	errorsThreshold      int
-	downloadFilesPercent int
+	downloadFilesPercent float64
 }
 
 func (v *verifier) progressCallback(ctx context.Context, enqueued, active, completed int64) {
@@ -174,8 +174,8 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin
 		}
 	}
 
-	//nolint:gomnd,gosec
-	if rand.Intn(100) < v.downloadFilesPercent {
+	//nolint:gosec
+	if 100*rand.Float64() < v.downloadFilesPercent {
 		if err := v.readEntireObject(ctx, oid, path); err != nil {
 			v.reportError(ctx, path, errors.Wrapf(err, "error reading object %v", oid))
 		}