From 6c5bf42d2374bdeac0f5a976208bd51bd5576bfc Mon Sep 17 00:00:00 2001 From: Jarek Kowalski Date: Sat, 24 Jul 2021 13:04:50 -0700 Subject: [PATCH] cli: changed 'kopia snapshot verify --verify-files-percent' to float (#1210) * cli: changed 'kopia snapshot verify --verify-files-percent' to float * cli: added 'kopia content verify --download-percent' --- cli/command_content_verify.go | 42 ++++++++++++++++-------------- cli/command_content_verify_test.go | 2 +- cli/command_snapshot_create.go | 2 +- cli/command_snapshot_verify.go | 10 +++---- 4 files changed, 30 insertions(+), 26 deletions(-) diff --git a/cli/command_content_verify.go b/cli/command_content_verify.go index f0e4dff97..349592e5b 100644 --- a/cli/command_content_verify.go +++ b/cli/command_content_verify.go @@ -2,6 +2,7 @@ import ( "context" + "math/rand" "sync" "sync/atomic" "time" @@ -18,6 +19,7 @@ type commandContentVerify struct { contentVerifyParallel int contentVerifyFull bool contentVerifyIncludeDeleted bool + contentVerifyPercent float64 progressInterval time.Duration contentRange contentRangeFlags @@ -29,6 +31,7 @@ func (c *commandContentVerify) setup(svc appServices, parent commandParent) { cmd.Flag("parallel", "Parallelism").Default("16").IntVar(&c.contentVerifyParallel) cmd.Flag("full", "Full verification (including download)").BoolVar(&c.contentVerifyFull) cmd.Flag("include-deleted", "Include deleted contents").BoolVar(&c.contentVerifyIncludeDeleted) + cmd.Flag("download-percent", "Download a percentage of files [0.0 .. 100.0]").Float64Var(&c.contentVerifyPercent) cmd.Flag("progress-interval", "Progress output interval").Default("3s").DurationVar(&c.progressInterval) c.contentRange.setup(cmd) cmd.Action(svc.directRepositoryReadAction(c.run)) @@ -56,14 +59,15 @@ func readBlobMap(ctx context.Context, br blob.Reader) (map[blob.ID]blob.Metadata func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepository) error { blobMap := map[blob.ID]blob.Metadata{} + downloadPercent := c.contentVerifyPercent - if !c.contentVerifyFull { - m, err := readBlobMap(ctx, rep.BlobReader()) - if err != nil { - return err - } + if c.contentVerifyFull { + downloadPercent = 100.0 + } - blobMap = m + blobMap, err := readBlobMap(ctx, rep.BlobReader()) + if err != nil { + return err } verifiedCount := new(int32) @@ -93,12 +97,12 @@ func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepositor throttle := new(timetrack.Throttle) est := timetrack.Start() - err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{ + if err := rep.ContentReader().IterateContents(ctx, content.IterateOptions{ Range: c.contentRange.contentIDRange(), Parallel: c.contentVerifyParallel, IncludeDeleted: c.contentVerifyIncludeDeleted, }, func(ci content.Info) error { - if err := c.contentVerify(ctx, rep.ContentReader(), ci, blobMap); err != nil { + if err := c.contentVerify(ctx, rep.ContentReader(), ci, blobMap, downloadPercent); err != nil { log(ctx).Errorf("error %v", err) atomic.AddInt32(errorCount, 1) } else { @@ -124,8 +128,7 @@ func (c *commandContentVerify) run(ctx context.Context, rep repo.DirectRepositor } return nil - }) - if err != nil { + }); err != nil { return errors.Wrap(err, "iterate contents") } @@ -160,15 +163,7 @@ func (c *commandContentVerify) getTotalContentCount(ctx context.Context, rep rep atomic.StoreInt32(totalCount, tc) } -func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Reader, ci content.Info, blobMap map[blob.ID]blob.Metadata) error { - if c.contentVerifyFull { - if _, err := r.GetContent(ctx, ci.GetContentID()); err != nil { - return errors.Wrapf(err, "content %v is invalid", ci.GetContentID()) - } - - return nil - } - +func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Reader, ci content.Info, blobMap map[blob.ID]blob.Metadata, downloadPercent float64) error { bi, ok := blobMap[ci.GetPackBlobID()] if !ok { return errors.Errorf("content %v depends on missing blob %v", ci.GetContentID(), ci.GetPackBlobID()) @@ -178,5 +173,14 @@ func (c *commandContentVerify) contentVerify(ctx context.Context, r content.Read return errors.Errorf("content %v out of bounds of its pack blob %v", ci.GetContentID(), ci.GetPackBlobID()) } + // nolint:gosec + if 100*rand.Float64() < downloadPercent { + if _, err := r.GetContent(ctx, ci.GetContentID()); err != nil { + return errors.Wrapf(err, "content %v is invalid", ci.GetContentID()) + } + + return nil + } + return nil } diff --git a/cli/command_content_verify_test.go b/cli/command_content_verify_test.go index 91061f82b..faae5f3da 100644 --- a/cli/command_content_verify_test.go +++ b/cli/command_content_verify_test.go @@ -22,7 +22,7 @@ func TestContentVerify(t *testing.T) { env.RunAndExpectSuccess(t, "repo", "create", "filesystem", "--path", env.RepoDir) env.RunAndExpectSuccess(t, "content", "verify") env.RunAndExpectSuccess(t, "snapshot", "create", dir) - env.RunAndExpectSuccess(t, "content", "verify") + env.RunAndExpectSuccess(t, "content", "verify", "--download-percent=30") // delete one of 'p' blobs. blobIDToDelete := strings.Split(env.RunAndExpectSuccess(t, "blob", "list", "--prefix=p")[0], " ")[0] diff --git a/cli/command_snapshot_create.go b/cli/command_snapshot_create.go index f12344b8f..302a97be4 100644 --- a/cli/command_snapshot_create.go +++ b/cli/command_snapshot_create.go @@ -52,7 +52,7 @@ func (c *commandSnapshotCreate) setup(svc appServices, parent commandParent) { cmd.Flag("checkpoint-interval", "Frequency for creating periodic checkpoint.").DurationVar(&c.snapshotCreateCheckpointInterval) cmd.Flag("description", "Free-form snapshot description.").StringVar(&c.snapshotCreateDescription) cmd.Flag("fail-fast", "Fail fast when creating snapshot.").Envar("KOPIA_SNAPSHOT_FAIL_FAST").BoolVar(&c.snapshotCreateFailFast) - cmd.Flag("force-hash", "Force hashing of source files for a given percentage of files [0..100]").Default("0").Float64Var(&c.snapshotCreateForceHash) + cmd.Flag("force-hash", "Force hashing of source files for a given percentage of files [0.0 .. 100.0]").Default("0").Float64Var(&c.snapshotCreateForceHash) cmd.Flag("parallel", "Upload N files in parallel").PlaceHolder("N").Default("0").IntVar(&c.snapshotCreateParallelUploads) cmd.Flag("start-time", "Override snapshot start timestamp.").StringVar(&c.snapshotCreateStartTime) cmd.Flag("end-time", "Override snapshot end timestamp.").StringVar(&c.snapshotCreateEndTime) diff --git a/cli/command_snapshot_verify.go b/cli/command_snapshot_verify.go index ae6a220ff..8b0de0275 100644 --- a/cli/command_snapshot_verify.go +++ b/cli/command_snapshot_verify.go @@ -27,7 +27,7 @@ type commandSnapshotVerify struct { verifyCommandAllSources bool verifyCommandSources []string verifyCommandParallel int - verifyCommandFilesPercent int + verifyCommandFilesPercent float64 } func (c *commandSnapshotVerify) setup(svc appServices, parent commandParent) { @@ -38,7 +38,7 @@ func (c *commandSnapshotVerify) setup(svc appServices, parent commandParent) { cmd.Flag("all-sources", "Verify all snapshots (DEPRECATED)").Hidden().BoolVar(&c.verifyCommandAllSources) cmd.Flag("sources", "Verify the provided sources").StringsVar(&c.verifyCommandSources) cmd.Flag("parallel", "Parallelization").Default("16").IntVar(&c.verifyCommandParallel) - cmd.Flag("verify-files-percent", "Randomly verify a percentage of files").Default("0").IntVar(&c.verifyCommandFilesPercent) + cmd.Flag("verify-files-percent", "Randomly verify a percentage of files by downloading them [0.0 .. 100.0]").Default("0").Float64Var(&c.verifyCommandFilesPercent) cmd.Action(svc.repositoryReaderAction(c.run)) } @@ -55,7 +55,7 @@ type verifier struct { errors []error errorsThreshold int - downloadFilesPercent int + downloadFilesPercent float64 } func (v *verifier) progressCallback(ctx context.Context, enqueued, active, completed int64) { @@ -174,8 +174,8 @@ func (v *verifier) doVerifyObject(ctx context.Context, oid object.ID, path strin } } - //nolint:gomnd,gosec - if rand.Intn(100) < v.downloadFilesPercent { + //nolint:gosec + if 100*rand.Float64() < v.downloadFilesPercent { if err := v.readEntireObject(ctx, oid, path); err != nil { v.reportError(ctx, path, errors.Wrapf(err, "error reading object %v", oid)) }